16 #include "allheaders.h" 65 "Min number of samples per proto as % of total");
67 "Max percentage of samples in a cluster which have more" 68 " than 1 feature in that cluster");
70 "Desired independence between dimensions");
72 "Desired confidence in prototypes created");
91 usage +=
" [.tr files ...]";
98 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_min_samples_fraction)));
100 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_max_illegal)));
102 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_independence)));
104 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_confidence)));
106 if (!FLAGS_configfile.empty()) {
108 FLAGS_configfile.c_str(),
118 STRING shape_table_file = file_prefix;
119 shape_table_file += kShapeTableFileSuffix;
121 if (shape_fp.
Open(shape_table_file.
string(),
nullptr)) {
125 shape_table =
nullptr;
126 tprintf(
"Error: Failed to read shape table %s\n",
127 shape_table_file.
string());
129 int num_shapes = shape_table->
NumShapes();
130 tprintf(
"Read shape table %s of %d shapes\n",
131 shape_table_file.
string(), num_shapes);
134 tprintf(
"Warning: No shape table file present: %s\n",
135 shape_table_file.
string());
142 STRING shape_table_file = file_prefix;
143 shape_table_file += kShapeTableFileSuffix;
144 FILE* fp = fopen(shape_table_file.
string(),
"wb");
147 fprintf(stderr,
"Error writing shape table: %s\n",
148 shape_table_file.
string());
152 fprintf(stderr,
"Error creating shape table: %s\n",
153 shape_table_file.
string());
179 if (!FLAGS_D.empty()) {
180 *file_prefix += FLAGS_D.
c_str();
187 bool shape_analysis =
false;
188 if (shape_table !=
nullptr) {
190 if (*shape_table !=
nullptr) shape_analysis =
true;
192 shape_analysis =
true;
202 if (!FLAGS_F.empty()) {
208 if (!FLAGS_X.empty()) {
215 const char* page_name;
218 tprintf(
"Reading %s ...\n", page_name);
223 int pagename_len = strlen(page_name);
224 char* fontinfo_file_name =
new char[pagename_len + 7];
225 strncpy(fontinfo_file_name, page_name, pagename_len - 2);
226 strcpy(fontinfo_file_name + pagename_len - 2,
"fontinfo");
228 delete[] fontinfo_file_name;
231 if (FLAGS_load_images) {
232 STRING image_name = page_name;
241 if (!FLAGS_output_trainer.empty()) {
242 FILE* fp = fopen(FLAGS_output_trainer.c_str(),
"wb");
244 tprintf(
"Can't create saved trainer data!\n");
251 if (!FLAGS_O.empty() &&
253 fprintf(stderr,
"Failed to save unicharset to file %s\n", FLAGS_O.c_str());
257 if (shape_table !=
nullptr) {
260 if (*shape_table ==
nullptr) {
263 tprintf(
"Flat shape table summary: %s\n",
264 (*shape_table)->SummaryStr().string());
266 (*shape_table)->set_unicharset(trainer->
unicharset());
311 if (strcmp (LabeledList->
Label, Label) == 0)
312 return (LabeledList);
333 strcpy (LabeledList->
Label, Label);
337 return (LabeledList);
364 const char *feature_name,
int max_samples,
366 FILE* file,
LIST* training_samples) {
373 assert(0 <= ShortNameToFeatureType_res);
374 unsigned int feature_type =
static_cast<unsigned int>(ShortNameToFeatureType_res);
376 LIST it = *training_samples;
382 while (fgets(buffer, 2048, file) !=
nullptr) {
383 if (buffer[0] ==
'\n')
386 sscanf(buffer,
"%*s %s", unichar);
390 tprintf(
"Error: Size of unicharset in training is " 391 "greater than MAX_NUM_CLASSES\n");
395 char_sample =
FindList(*training_samples, unichar);
396 if (char_sample ==
nullptr) {
398 *training_samples =
push(*training_samples, char_sample);
401 feature_samples = char_desc->
FeatureSets[feature_type];
403 char_sample->
List =
push(char_sample->
List, feature_samples);
410 if (feature_type != i)
433 LIST nodes = CharList;
436 FeatureList = char_sample->
List;
459 free(LabeledList->
Label);
479 const char* program_feature_type) {
485 LIST FeatureList =
nullptr;
492 FeatureList = char_sample->
List;
498 for (j = 0; j < N; j++)
513 bool debug = strcmp(FLAGS_test_ch.c_str(), label) == 0;
515 LIST pProtoList = ProtoList;
523 LIST list_it = ProtoList;
526 if (test_p != Prototype && !test_p->
Merged) {
530 if (dist < best_dist) {
536 if (best_match !=
nullptr && !best_match->
Significant) {
538 tprintf(
"Merging red clusters (%d+%d) at %g,%g and %g,%g\n",
540 best_match->
Mean[0], best_match->
Mean[1],
541 Prototype->
Mean[0], Prototype->
Mean[1]);
550 }
else if (best_match !=
nullptr) {
552 tprintf(
"Red proto at %g,%g matched a green one at %g,%g\n",
553 Prototype->
Mean[0], Prototype->
Mean[1],
554 best_match->
Mean[0], best_match->
Mean[1]);
560 pProtoList = ProtoList;
567 tprintf(
"Red proto at %g,%g becoming green\n",
568 Prototype->
Mean[0], Prototype->
Mean[1]);
596 BOOL8 KeepInsigProtos,
606 pProtoList = ProtoList;
622 for (i=0; i < N; i++)
626 for (i=0; i < N; i++)
634 for (i=0; i < N; i++)
642 for (i=0; i < N; i++)
650 NewProtoList =
push_last(NewProtoList, NewProto);
654 return (NewProtoList);
664 if (strcmp (MergeClass->
Label, Label) == 0)
677 strcpy (MergeClass->
Label, Label);
696 LIST nodes = ClassList;
700 free (MergeClass->
Label);
710 LIST LabeledClassList) {
737 for(i=0; i < NumProtos; i++)
741 Values[0] = OldProto->
X;
742 Values[1] = OldProto->
Y;
743 Values[2] = OldProto->
Angle;
745 NewProto->
X = OldProto->
X;
746 NewProto->
Y = OldProto->
Y;
749 NewProto->
A = Values[0];
750 NewProto->
B = Values[1];
751 NewProto->
C = Values[2];
759 for(i=0; i < NumConfigs; i++)
763 for(j=0; j < NumWords; j++)
764 NewConfig[j] = OldConfig[j];
768 return float_classes;
779 Slope = tan (Values [2] * 2 *
PI);
780 Intercept = Values [1] - Slope * Values [0];
781 Normalizer = 1 / sqrt (Slope * Slope + 1.0);
783 Values [0] = Slope * Normalizer;
784 Values [1] = - Normalizer;
785 Values [2] = Intercept * Normalizer;
794 LIST nodes = CharList;
817 LabeledProtoList->
List =
push(LabeledProtoList->
List, Proto);
819 *NormProtoList =
push(*NormProtoList, LabeledProtoList);
824 BOOL8 CountInsigProtos) {
LABELEDLIST FindList(LIST List, char *Label)
void CleanUpUnusedData(LIST ProtoList)
bool Serialize(FILE *fp) const
void InitFeatureDefs(FEATURE_DEFS_STRUCT *featuredefs)
bool LoadFontInfo(const char *filename)
void truncate_at(inT32 index)
void AddToNormProtosList(LIST *NormProtoList, LIST ProtoList, char *CharName)
STRING_PARAM_FLAG(configfile, "", "File to load more configs from")
void SetFeatureSpace(const IntFeatureSpace &fs)
MERGE_CLASS_NODE * MERGE_CLASS
bool contains_unichar(const char *const unichar_repr) const
void Normalize(float *Values)
DOUBLE_PARAM_FLAG(clusterconfig_min_samples_fraction, Config.MinSamples, "Min number of samples per proto as % of total")
void FreeFeatureSet(FEATURE_SET FeatureSet)
void WriteShapeTable(const STRING &file_prefix, const ShapeTable &shape_table)
#define WordsInVectorOfSize(NumBits)
void FreeNormProtoList(LIST CharList)
const PARAM_DESC * ParamDesc
const UNICHARSET & unicharset() const
const char * string() const
bool DeSerialize(TFile *fp)
LIST push_last(LIST list, void *item)
struct LABELEDLISTNODE * LABELEDLIST
void ParseArguments(int *argc, char ***argv)
CHAR_DESC ReadCharDescription(const FEATURE_DEFS_STRUCT &FeatureDefs, FILE *File)
void Init(uinT8 xbuckets, uinT8 ybuckets, uinT8 thetabuckets)
MERGE_CLASS FindClass(LIST List, const char *Label)
BIT_VECTOR NewBitVector(int NumBits)
FEATURE_SET FeatureSets[NUM_FEATURE_TYPES]
CLUSTERER * MakeClusterer(inT16 SampleSize, const PARAM_DESC ParamDesc[])
void LoadPageImages(const char *filename)
const int kBoostDirBuckets
MasterTrainer * LoadTrainingData(int argc, const char *const *argv, bool replication, ShapeTable **shape_table, STRING *file_prefix)
void move(UnicityTable< T > *from)
CLUSTERER * SetUpForClustering(const FEATURE_DEFS_STRUCT &FeatureDefs, LABELEDLIST char_sample, const char *program_feature_type)
void FreeProtoList(LIST *ProtoList)
void ReadTrainingSamples(const char *page_name, const FEATURE_DEFS_STRUCT &feature_defs, bool verification)
const char * GetNextFilename(int argc, const char *const *argv)
CLASS_STRUCT * SetUpForFloat2Int(const UNICHARSET &unicharset, LIST LabeledClassList)
static bool ReadParamsFile(const char *file, SetParamConstraint constraint, ParamsVectors *member_params)
const FEATURE_DESC_STRUCT * FeatureDesc[NUM_FEATURE_TYPES]
FEATURE_SET_STRUCT * FEATURE_SET
void FreeClass(CLASS_TYPE Class)
LIST RemoveInsignificantProtos(LIST ProtoList, BOOL8 KeepSigProtos, BOOL8 KeepInsigProtos, int N)
CLASS_TYPE NewClass(int NumProtos, int NumConfigs)
int ShortNameToFeatureType(const FEATURE_DEFS_STRUCT &FeatureDefs, const char *ShortName)
ShapeTable * LoadShapeTable(const STRING &file_prefix)
void MergeInsignificantProtos(LIST ProtoList, const char *label, CLUSTERER *Clusterer, CLUSTERCONFIG *Config)
void FreeLabeledList(LABELEDLIST LabeledList)
void SetupFlatShapeTable(ShapeTable *shape_table)
UnicityTableEqEq< int > font_set
const int kBoostXYBuckets
bool save_to_file(const char *const filename) const
FLOAT32 ComputeDistance(int k, PARAM_DESC *dim, FLOAT32 p1[], FLOAT32 p2[])
const char * c_str() const
bool Open(const STRING &filename, FileReader reader)
void ReadTrainingSamples(const FEATURE_DEFS_STRUCT &feature_defs, const char *feature_name, int max_samples, UNICHARSET *unicharset, FILE *file, LIST *training_samples)
FEATURE_DEFS_STRUCT feature_defs
void FreeLabeledClassList(LIST ClassList)
INT_PARAM_FLAG(debug_level, 0, "Level of Trainer debugging")
int NumberOfProtos(LIST ProtoList, BOOL8 CountSigProtos, BOOL8 CountInsigProtos)
inT32 MergeClusters(inT16 N, register PARAM_DESC ParamDesc[], register inT32 n1, register inT32 n2, register FLOAT32 m[], register FLOAT32 m1[], register FLOAT32 m2[])
LABELEDLIST NewLabeledList(const char *Label)
bool Serialize(FILE *fp) const
bool AddSpacingInfo(const char *filename)
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
void LoadUnicharset(const char *filename)
bool LoadXHeights(const char *filename)
MERGE_CLASS NewLabeledClass(const char *Label)
LIST push(LIST list, void *element)
#define ProtoIn(Class, Pid)
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
void unichar_insert(const char *const unichar_repr)
void FreeTrainingSamples(LIST CharList)
SAMPLE * MakeSample(CLUSTERER *Clusterer, const FLOAT32 *Feature, inT32 CharID)