tesseract::FullyConnected Class Reference

#include <fullyconnected.h>

Inheritance diagram for tesseract::FullyConnected:

Public Member Functions
	FullyConnected (const STRING &name, int ni, int no, NetworkType type)
virtual	~FullyConnected ()
virtual StaticShape	OutputShape (const StaticShape &input_shape) const
virtual STRING	spec () const
void	ChangeType (NetworkType type)
virtual void	SetEnableTraining (TrainingState state)
virtual int	InitWeights (float range, TRand *randomizer)
virtual void	ConvertToInt ()
virtual void	DebugWeights ()
virtual bool	Serialize (TFile *fp) const
virtual bool	DeSerialize (TFile *fp)
virtual void	Forward (bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output)
void	SetupForward (const NetworkIO &input, const TransposedArray *input_transpose)
void	ForwardTimeStep (const double *d_input, const inT8 *i_input, int t, double *output_line)
virtual bool	Backward (bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas)
void	BackwardTimeStep (const NetworkIO &fwd_deltas, int t, double *curr_errors, TransposedArray *errors_t, double *backprop)
void	FinishBackward (const TransposedArray &errors_t)
virtual void	Update (float learning_rate, float momentum, int num_samples)
virtual void	CountAlternators (const Network &other, double *same, double *changed) const
Public Member Functions inherited from tesseract::Network
	Network ()
	Network (NetworkType type, const STRING &name, int ni, int no)
virtual	~Network ()
NetworkType	type () const
bool	IsTraining () const
bool	needs_to_backprop () const
int	num_weights () const
int	NumInputs () const
int	NumOutputs () const
virtual StaticShape	InputShape () const
const STRING &	name () const
bool	TestFlag (NetworkFlags flag) const
virtual bool	IsPlumbingType () const
virtual void	SetNetworkFlags (uinT32 flags)
virtual void	SetRandomizer (TRand *randomizer)
virtual bool	SetupNeedsBackprop (bool needs_backprop)
virtual int	XScaleFactor () const
virtual void	CacheXScaleFactor (int factor)
void	DisplayForward (const NetworkIO &matrix)
void	DisplayBackward (const NetworkIO &matrix)

Protected Attributes
WeightMatrix	weights_
TransposedArray	source_t_
const TransposedArray *	external_source_
NetworkIO	acts_
bool	int_mode_
Protected Attributes inherited from tesseract::Network
NetworkType	type_
TrainingState	training_
bool	needs_to_backprop_
inT32	network_flags_
inT32	ni_
inT32	no_
inT32	num_weights_
STRING	name_
ScrollView *	forward_win_
ScrollView *	backward_win_
TRand *	randomizer_

Additional Inherited Members
Static Public Member Functions inherited from tesseract::Network
static Network *	CreateFromFile (TFile *fp)
static void	ClearWindow (bool tess_coords, const char *window_name, int width, int height, ScrollView **window)
static int	DisplayImage (Pix *pix, ScrollView *window)
Protected Member Functions inherited from tesseract::Network
double	Random (double range)
Static Protected Attributes inherited from tesseract::Network
static char const *const	kTypeNames [NT_COUNT]

Detailed Description

Definition at line 28 of file fullyconnected.h.

Constructor & Destructor Documentation

FullyConnected()

tesseract::FullyConnected::FullyConnected	(	const STRING &	name,
		int	ni,
		int	no,
		NetworkType	type
	)

Definition at line 35 of file fullyconnected.cpp.

  : Network(type, name, ni, no), external_source_(NULL), int_mode_(false) {
}

~FullyConnected()

tesseract::FullyConnected::~FullyConnected ( )

virtual

Definition at line 40 of file fullyconnected.cpp.

                                {
}

Member Function Documentation

Backward()

bool tesseract::FullyConnected::Backward ( bool  debug, const NetworkIO &  fwd_deltas, NetworkScratch *  scratch, NetworkIO *  back_deltas  )

virtual

Reimplemented from tesseract::Network.

Definition at line 204 of file fullyconnected.cpp.

                                                      {
  if (debug) DisplayBackward(fwd_deltas);
  back_deltas->Resize(fwd_deltas, ni_);
  GenericVector<NetworkScratch::FloatVec> errors;
  errors.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < errors.size(); ++i) errors[i].Init(no_, scratch);
  GenericVector<NetworkScratch::FloatVec> temp_backprops;
  if (needs_to_backprop_) {
    temp_backprops.init_to_size(kNumThreads, NetworkScratch::FloatVec());
    for (int i = 0; i < kNumThreads; ++i) temp_backprops[i].Init(ni_, scratch);
  }
  int width = fwd_deltas.Width();
  NetworkScratch::GradientStore errors_t;
  errors_t.Init(no_, width, scratch);
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    int thread_id = 0;
#endif
    double* backprop = NULL;
    if (needs_to_backprop_) backprop = temp_backprops[thread_id];
    double* curr_errors = errors[thread_id];
    BackwardTimeStep(fwd_deltas, t, curr_errors, errors_t.get(), backprop);
    if (backprop != NULL) {
      back_deltas->WriteTimeStep(t, backprop);
    }
  }
  FinishBackward(*errors_t.get());
  if (needs_to_backprop_) {
    back_deltas->ZeroInvalidElements();
    back_deltas->CopyWithNormalization(*back_deltas, fwd_deltas);
#if DEBUG_DETAIL > 0
    tprintf("F Backprop:%s\n", name_.string());
    back_deltas->Print(10);
#endif
    return true;
  }
  return false;  // No point going further back.
}

BackwardTimeStep()

void tesseract::FullyConnected::BackwardTimeStep	(	const NetworkIO &	fwd_deltas,
		int	t,
		double *	curr_errors,
		TransposedArray *	errors_t,
		double *	backprop
	)

Definition at line 249 of file fullyconnected.cpp.

                                                        {
  if (type_ == NT_TANH)
    acts_.FuncMultiply<GPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_LOGISTIC)
    acts_.FuncMultiply<FPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_POSCLIP)
    acts_.FuncMultiply<ClipFPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SYMCLIP)
    acts_.FuncMultiply<ClipGPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_RELU)
    acts_.FuncMultiply<ReluPrime>(fwd_deltas, t, curr_errors);
  else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC ||
           type_ == NT_LINEAR)
    fwd_deltas.ReadTimeStep(t, curr_errors);  // fwd_deltas are the errors.
  else
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  // Generate backprop only if needed by the lower layer.
  if (backprop != NULL) weights_.VectorDotMatrix(curr_errors, backprop);
  errors_t->WriteStrided(t, curr_errors);
}

ChangeType()

void tesseract::FullyConnected::ChangeType ( NetworkType  type )

inline

Definition at line 60 of file fullyconnected.h.

                                    {
    type_ = type;
  }

ConvertToInt()

void tesseract::FullyConnected::ConvertToInt ( )

virtual

Reimplemented from tesseract::Network.

Definition at line 84 of file fullyconnected.cpp.

                                  {
  weights_.ConvertToInt();
}

CountAlternators()

void tesseract::FullyConnected::CountAlternators ( const Network &  other, double *  same, double *  changed  ) const

virtual

Reimplemented from tesseract::Network.

Definition at line 291 of file fullyconnected.cpp.

                                                             {
  ASSERT_HOST(other.type() == type_);
  const FullyConnected* fc = static_cast<const FullyConnected*>(&other);
  weights_.CountAlternators(fc->weights_, same, changed);
}

DebugWeights()

void tesseract::FullyConnected::DebugWeights ( )

virtual

Reimplemented from tesseract::Network.

Definition at line 89 of file fullyconnected.cpp.

                                  {
  weights_.Debug2D(name_.string());
}

DeSerialize()

bool tesseract::FullyConnected::DeSerialize ( TFile *  fp )

virtual

Reimplemented from tesseract::Network.

Definition at line 101 of file fullyconnected.cpp.

                                          {
  return weights_.DeSerialize(IsTraining(), fp);
}

FinishBackward()

void tesseract::FullyConnected::FinishBackward

(

const TransposedArray &

errors_t

)

Definition at line 273 of file fullyconnected.cpp.

                                                                   {
  if (external_source_ == NULL)
    weights_.SumOuterTransposed(errors_t, source_t_, true);
  else
    weights_.SumOuterTransposed(errors_t, *external_source_, true);
}

Forward()

void tesseract::FullyConnected::Forward ( bool  debug, const NetworkIO &  input, const TransposedArray *  input_transpose, NetworkScratch *  scratch, NetworkIO *  output  )

virtual

Reimplemented from tesseract::Network.

Definition at line 107 of file fullyconnected.cpp.

                                                                         {
  int width = input.Width();
  if (type_ == NT_SOFTMAX)
    output->ResizeFloat(input, no_);
  else
    output->Resize(input, no_);
  SetupForward(input, input_transpose);
  GenericVector<NetworkScratch::FloatVec> temp_lines;
  temp_lines.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  GenericVector<NetworkScratch::FloatVec> curr_input;
  curr_input.init_to_size(kNumThreads, NetworkScratch::FloatVec());
  for (int i = 0; i < temp_lines.size(); ++i) {
    temp_lines[i].Init(no_, scratch);
    curr_input[i].Init(ni_, scratch);
  }
#ifdef _OPENMP
#pragma omp parallel for num_threads(kNumThreads)
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = omp_get_thread_num();
#else
  for (int t = 0; t < width; ++t) {
    // Thread-local pointer to temporary storage.
    int thread_id = 0;
#endif
    double* temp_line = temp_lines[thread_id];
    const double* d_input = NULL;
    const inT8* i_input = NULL;
    if (input.int_mode()) {
      i_input = input.i(t);
    } else {
      input.ReadTimeStep(t, curr_input[thread_id]);
      d_input = curr_input[thread_id];
    }
    ForwardTimeStep(d_input, i_input, t, temp_line);
    output->WriteTimeStep(t, temp_line);
    if (IsTraining() && type_ != NT_SOFTMAX) {
      acts_.CopyTimeStepFrom(t, *output, t);
    }
  }
  // Zero all the elements that are in the padding around images that allows
  // multiple different-sized images to exist in a single array.
  // acts_ is only used if this is not a softmax op.
  if (IsTraining() && type_ != NT_SOFTMAX) {
    acts_.ZeroInvalidElements();
  }
  output->ZeroInvalidElements();
#if DEBUG_DETAIL > 0
  tprintf("F Output:%s\n", name_.string());
  output->Print(10);
#endif
  if (debug) DisplayForward(*output);
}

ForwardTimeStep()

void tesseract::FullyConnected::ForwardTimeStep	(	const double *	d_input,
		const inT8 *	i_input,
		int	t,
		double *	output_line
	)

Definition at line 176 of file fullyconnected.cpp.

                                                                 {
  // input is copied to source_ line-by-line for cache coherency.
  if (IsTraining() && external_source_ == NULL && d_input != NULL)
    source_t_.WriteStrided(t, d_input);
  if (d_input != NULL)
    weights_.MatrixDotVector(d_input, output_line);
  else
    weights_.MatrixDotVector(i_input, output_line);
  if (type_ == NT_TANH) {
    FuncInplace<GFunc>(no_, output_line);
  } else if (type_ == NT_LOGISTIC) {
    FuncInplace<FFunc>(no_, output_line);
  } else if (type_ == NT_POSCLIP) {
    FuncInplace<ClipFFunc>(no_, output_line);
  } else if (type_ == NT_SYMCLIP) {
    FuncInplace<ClipGFunc>(no_, output_line);
  } else if (type_ == NT_RELU) {
    FuncInplace<Relu>(no_, output_line);
  } else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC) {
    SoftmaxInPlace(no_, output_line);
  } else if (type_ != NT_LINEAR) {
    ASSERT_HOST("Invalid fully-connected type!" == NULL);
  }
}

InitWeights()

int tesseract::FullyConnected::InitWeights ( float  range, TRand *  randomizer  )

virtual

Reimplemented from tesseract::Network.

Definition at line 76 of file fullyconnected.cpp.

                                                              {
  Network::SetRandomizer(randomizer);
  num_weights_ = weights_.InitWeightsFloat(no_, ni_ + 1, TestFlag(NF_ADA_GRAD),
                                           range, randomizer);
  return num_weights_;
}

OutputShape()

StaticShape tesseract::FullyConnected::OutputShape ( const StaticShape &  input_shape ) const

virtual

Reimplemented from tesseract::Network.

Definition at line 45 of file fullyconnected.cpp.

                                                                            {
  LossType loss_type = LT_NONE;
  if (type_ == NT_SOFTMAX)
    loss_type = LT_CTC;
  else if (type_ == NT_SOFTMAX_NO_CTC)
    loss_type = LT_SOFTMAX;
  else if (type_ == NT_LOGISTIC)
    loss_type = LT_LOGISTIC;
  StaticShape result(input_shape);
  result.set_depth(no_);
  result.set_loss_type(loss_type);
  return result;
}

Serialize()

bool tesseract::FullyConnected::Serialize ( TFile *  fp ) const

virtual

Reimplemented from tesseract::Network.

Definition at line 94 of file fullyconnected.cpp.

                                              {
  if (!Network::Serialize(fp)) return false;
  if (!weights_.Serialize(IsTraining(), fp)) return false;
  return true;
}

SetEnableTraining()

void tesseract::FullyConnected::SetEnableTraining ( TrainingState  state )

virtual

Reimplemented from tesseract::Network.

Definition at line 60 of file fullyconnected.cpp.

                                                          {
  if (state == TS_RE_ENABLE) {
    // Enable only from temp disabled.
    if (training_ == TS_TEMP_DISABLE) training_ = TS_ENABLED;
  } else if (state == TS_TEMP_DISABLE) {
    // Temp disable only from enabled.
    if (training_ == TS_ENABLED) training_ = state;
  } else {
    if (state == TS_ENABLED && training_ != TS_ENABLED)
      weights_.InitBackward();
    training_ = state;
  }
}

SetupForward()

void tesseract::FullyConnected::SetupForward	(	const NetworkIO &	input,
		const TransposedArray *	input_transpose
	)

Definition at line 164 of file fullyconnected.cpp.

                                                                          {
  // Softmax output is always float, so save the input type.
  int_mode_ = input.int_mode();
  if (IsTraining()) {
    acts_.Resize(input, no_);
    // Source_ is a transposed copy of input. It isn't needed if provided.
    external_source_ = input_transpose;
    if (external_source_ == NULL) source_t_.ResizeNoInit(ni_, input.Width());
  }
}

spec()

virtual STRING tesseract::FullyConnected::spec ( ) const

inlinevirtual

Reimplemented from tesseract::Network.

Definition at line 37 of file fullyconnected.h.

                              {
    STRING spec;
    if (type_ == NT_TANH)
      spec.add_str_int("Ft", no_);
    else if (type_ == NT_LOGISTIC)
      spec.add_str_int("Fs", no_);
    else if (type_ == NT_RELU)
      spec.add_str_int("Fr", no_);
    else if (type_ == NT_LINEAR)
      spec.add_str_int("Fl", no_);
    else if (type_ == NT_POSCLIP)
      spec.add_str_int("Fp", no_);
    else if (type_ == NT_SYMCLIP)
      spec.add_str_int("Fs", no_);
    else if (type_ == NT_SOFTMAX)
      spec.add_str_int("Fc", no_);
    else
      spec.add_str_int("Fm", no_);
    return spec;
  }

Update()

void tesseract::FullyConnected::Update ( float  learning_rate, float  momentum, int  num_samples  )

virtual

Reimplemented from tesseract::Network.

Definition at line 283 of file fullyconnected.cpp.

                                             {
  weights_.Update(learning_rate, momentum, num_samples);
}

Member Data Documentation

acts_

NetworkIO tesseract::FullyConnected::acts_

protected

Definition at line 123 of file fullyconnected.h.

external_source_

const TransposedArray* tesseract::FullyConnected::external_source_

protected

Definition at line 121 of file fullyconnected.h.

int_mode_

bool tesseract::FullyConnected::int_mode_

protected

Definition at line 126 of file fullyconnected.h.

source_t_

TransposedArray tesseract::FullyConnected::source_t_

protected

Definition at line 118 of file fullyconnected.h.

weights_

WeightMatrix tesseract::FullyConnected::weights_

protected

Definition at line 116 of file fullyconnected.h.

---

The documentation for this class was generated from the following files:

• /home/stefan/src/github/tesseract-ocr/tesseract/lstm/fullyconnected.h
• /home/stefan/src/github/tesseract-ocr/tesseract/lstm/fullyconnected.cpp