24 #include "config_auto.h" 30 #include "allheaders.h" 56 bool replicate_samples,
58 : norm_mode_(norm_mode), samples_(fontinfo_table_),
59 junk_samples_(fontinfo_table_), verify_samples_(fontinfo_table_),
61 enable_shape_anaylsis_(shape_analysis),
62 enable_replication_(replicate_samples),
63 fragments_(NULL), prev_unichar_id_(-1), debug_level_(debug_level) {
68 for (
int p = 0; p < page_images_.
size(); ++p)
69 pixDestroy(&page_images_[p]);
76 if (fwrite(&norm_mode_,
sizeof(norm_mode_), 1, fp) != 1)
return false;
78 if (!feature_space_.
Serialize(fp))
return false;
79 if (!samples_.
Serialize(fp))
return false;
80 if (!junk_samples_.
Serialize(fp))
return false;
81 if (!verify_samples_.
Serialize(fp))
return false;
82 if (!master_shapes_.
Serialize(fp))
return false;
83 if (!flat_shapes_.
Serialize(fp))
return false;
84 if (!fontinfo_table_.
Serialize(fp))
return false;
85 if (!xheights_.
Serialize(fp))
return false;
92 tprintf(
"Failed to load unicharset from file %s\n" 93 "Building unicharset for training from scratch...\n",
101 charsetsize_ = unicharset_.
size();
102 delete [] fragments_;
103 fragments_ =
new int[charsetsize_];
104 memset(fragments_, 0,
sizeof(*fragments_) * charsetsize_);
124 FILE* fp =
Efopen(page_name,
"rb");
126 tprintf(
"Failed to open tr file: %s\n", page_name);
130 while (fgets(buffer,
sizeof(buffer), fp) != NULL) {
131 if (buffer[0] ==
'\n')
134 char* space = strchr(buffer,
' ');
136 tprintf(
"Bad format in tr file, reading fontname, unichar\n");
141 if (font_id < 0) font_id = 0;
146 tprintf(
"Bad format in tr file, reading box coords\n");
155 cn_feature_type, geo_feature_type, char_desc);
159 charsetsize_ = unicharset_.
size();
168 verify_samples_.
AddSample(unichar, sample);
169 prev_unichar_id_ = -1;
171 if (prev_unichar_id_ >= 0)
172 fragments_[prev_unichar_id_] = -1;
173 prev_unichar_id_ = samples_.
AddSample(unichar, sample);
177 int junk_id = junk_samples_.
AddSample(unichar, sample);
178 if (prev_unichar_id_ >= 0) {
181 if (fragments_[prev_unichar_id_] == 0)
182 fragments_[prev_unichar_id_] = junk_id;
183 else if (fragments_[prev_unichar_id_] != junk_id)
184 fragments_[prev_unichar_id_] = -1;
188 prev_unichar_id_ = -1;
199 for (page = 0;; page++) {
200 pix = pixReadFromMultipageTiff(filename, &offset);
205 tprintf(
"Loaded %d page images from %s\n", page, filename);
214 if (debug_level_ > 0)
215 tprintf(
"PostLoadCleanup...\n");
216 if (enable_shape_anaylsis_)
217 ReplaceFragmentedSamples();
219 sample_it.
Init(NULL, NULL,
true, &verify_samples_);
228 if (debug_level_ > 0)
229 tprintf(
"ComputeCanonicalSamples...\n");
237 if (debug_level_ > 0)
238 tprintf(
"PreTrainingSetup...\n");
241 if (debug_level_ > 0)
242 tprintf(
"ComputeCloudFeatures...\n");
249 tprintf(
"Building master shape table\n");
250 int num_fonts = samples_.
NumFonts();
257 for (
int f = 0; f < num_fonts; ++f) {
261 ClusterShapes(kMinClusteredShapes, 1, kFontMergeDistance, &shapes);
265 if (fragment == NULL)
266 char_shapes.AppendMasterShapes(shapes, NULL);
268 char_shapes_begin_fragment.AppendMasterShapes(shapes, NULL);
270 char_shapes_end_fragment.AppendMasterShapes(shapes, NULL);
272 char_shapes.AppendMasterShapes(shapes, NULL);
274 ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
275 kFontMergeDistance, &char_shapes_begin_fragment);
276 char_shapes.AppendMasterShapes(char_shapes_begin_fragment, NULL);
277 ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
278 kFontMergeDistance, &char_shapes_end_fragment);
279 char_shapes.AppendMasterShapes(char_shapes_end_fragment, NULL);
280 ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
281 kFontMergeDistance, &char_shapes);
302 tprintf(
"Moving %d junk samples to master sample set.\n", num_junks);
303 for (
int s = 0; s < num_junks; ++s) {
308 if (sample_id == INVALID_UNICHAR_ID)
324 if (enable_replication_) {
325 if (debug_level_ > 0)
326 tprintf(
"ReplicateAndRandomize...\n");
336 FILE* fp = fopen(filename,
"rb");
338 fprintf(stderr,
"Failed to load font_properties from %s\n", filename);
341 int italic, bold, fixed, serif, fraktur;
344 char* font_name =
new char[1024];
345 fontinfo.
name = font_name;
348 if (
tfscanf(fp,
"%1024s %i %i %i %i %i\n", font_name, &italic, &bold,
349 &fixed, &serif, &fraktur) != 6) {
359 if (!fontinfo_table_.
contains(fontinfo)) {
372 tprintf(
"fontinfo table is of size %d\n", fontinfo_table_.
size());
374 if (filename == NULL)
return true;
375 FILE *f = fopen(filename,
"rb");
377 fprintf(stderr,
"Failed to load font xheights from %s\n", filename);
380 tprintf(
"Reading x-heights from %s ...\n", filename);
386 int total_xheight = 0;
387 int xheight_count = 0;
389 if (
tfscanf(f,
"%1023s %d\n", buffer, &xht) != 2)
392 fontinfo.
name = buffer;
393 if (!fontinfo_table_.
contains(fontinfo))
continue;
394 int fontinfo_id = fontinfo_table_.
get_index(fontinfo);
395 xheights_[fontinfo_id] = xht;
396 total_xheight += xht;
399 if (xheight_count == 0) {
400 fprintf(stderr,
"No valid xheights in %s!\n", filename);
404 int mean_xheight =
DivRounded(total_xheight, xheight_count);
405 for (
int i = 0; i < fontinfo_table_.
size(); ++i) {
406 if (xheights_[i] < 0)
407 xheights_[i] = mean_xheight;
415 FILE* fontinfo_file = fopen(filename,
"rb");
416 if (fontinfo_file == NULL)
420 if (fontinfo_id < 0) {
421 tprintf(
"No font found matching fontinfo filename %s\n", filename);
422 fclose(fontinfo_file);
425 tprintf(
"Reading spacing from %s for font %d...\n", filename, fontinfo_id);
432 int x_gap, x_gap_before, x_gap_after, num_kerned;
437 for (
int l = 0; l < num_unichars; ++l) {
438 if (
tfscanf(fontinfo_file,
"%s %d %d %d",
439 uch, &x_gap_before, &x_gap_after, &num_kerned) != 4) {
440 tprintf(
"Bad format of font spacing file %s\n", filename);
441 fclose(fontinfo_file);
450 for (
int k = 0; k < num_kerned; ++k) {
451 if (
tfscanf(fontinfo_file,
"%s %d", kerned_uch, &x_gap) != 2) {
452 tprintf(
"Bad format of font spacing file %s\n", filename);
453 fclose(fontinfo_file);
464 fclose(fontinfo_file);
473 fontinfo.
name =
const_cast<char*
>(font_name);
476 return fontinfo_table_.
get_index(fontinfo);
482 int fontinfo_id = -1;
484 for (
int f = 0; f < fontinfo_table_.
size(); ++f) {
485 if (strstr(filename, fontinfo_table_.
get(f).name) != NULL) {
486 int len = strlen(fontinfo_table_.
get(f).name);
488 if (len > best_len) {
504 int num_shapes = flat_shapes_.
NumShapes();
505 for (
int s = 0; s < num_shapes; ++s) {
506 int font = flat_shapes_.
GetShape(s)[0].font_ids[0];
508 for (f = 0; f < active_fonts.
size(); ++f) {
509 if (active_fonts[f] == font)
512 if (f == active_fonts.
size())
516 int num_fonts = active_fonts.
size();
517 for (
int f = 0; f < num_fonts; ++f) {
518 for (
int s = num_shapes - 1; s >= 0; --s) {
519 int font = flat_shapes_.
GetShape(s)[0].font_ids[0];
520 if (font == active_fonts[f]) {
544 shape_map.
SetMap(shape_id,
true);
549 it.
Init(&shape_map, &shape_table,
false, &samples_);
554 for (
int i = sample_ptrs.
size() - 1; i >= 0; --i) {
557 for (
int f = 0; f < num_features; ++f)
561 *num_samples = sample_id;
573 const char* inttemp_file,
574 const char* pffmtable_file) {
580 FILE* fp = fopen(inttemp_file,
"wb");
590 for (
int c = 0; c < unicharset.
size(); ++c)
593 for (
int i = 0; i < int_templates->
NumClasses; ++i) {
598 for (
int config_id = 0; config_id < Class->
NumConfigs; config_id++) {
602 if (length > max_length)
604 int shape_id = float_classes[i].
font_set.
get(config_id);
606 for (
int c = 0; c < shape.
size(); ++c) {
607 int unichar_id = shape[c].unichar_id;
608 if (length > unichar_cutoffs[unichar_id])
609 unichar_cutoffs[unichar_id] = length;
612 shapetable_cutoffs.
push_back(max_length);
614 fp = fopen(pffmtable_file,
"wb");
616 for (
int c = 0; c < unicharset.
size(); ++c) {
618 if (strcmp(unichar,
" ") == 0) {
621 fprintf(fp,
"%s %d\n", unichar, unichar_cutoffs[c]);
631 const char* unichar_str2) {
634 if (class_id2 == INVALID_UNICHAR_ID)
635 class_id2 = class_id1;
636 if (class_id1 == INVALID_UNICHAR_ID) {
637 tprintf(
"No unicharset entry found for %s\n", unichar_str1);
640 tprintf(
"Font ambiguities for unichar %d = %s and %d = %s\n",
641 class_id1, unichar_str1, class_id2, unichar_str2);
643 int num_fonts = samples_.
NumFonts();
648 for (
int f = 0; f < num_fonts; ++f) {
654 for (
int f1 = 0; f1 < num_fonts; ++f1) {
659 for (
int f2 = 0; f2 < num_fonts; ++f2) {
670 for (
int f = 0; f < num_fonts; ++f) {
673 if (class_id1 != class_id2 &&
679 #ifndef GRAPHICS_DISABLED 691 const char* unichar_str2,
692 int canonical_font) {
699 if (class_id2 != INVALID_UNICHAR_ID && canonical_font >= 0) {
707 if (class_id1 != INVALID_UNICHAR_ID && cloud_font >= 0) {
709 for (
int f = 0; f < cloud.
size(); ++f) {
727 if (feature_index >= 0) {
741 #endif // GRAPHICS_DISABLED 747 sample_it.
Init(NULL, NULL, replicate_samples, &samples_);
750 page_images_, &sample_it);
757 bool replicate_samples,
760 TestClassifier(error_mode, report_level, replicate_samples, &samples_,
761 test_classifier, report_string);
779 bool replicate_samples,
784 sample_it.
Init(NULL, NULL, replicate_samples, samples);
785 if (report_level > 0) {
789 tprintf(
"Iterator has charset size of %d/%d, %d shapes, %d samples\n",
792 tprintf(
"Testing %sREPLICATED:\n", replicate_samples ?
"" :
"NON-");
794 double unichar_error = 0.0;
796 error_mode, fontinfo_table_,
797 page_images_, &sample_it, &unichar_error,
798 NULL, report_string);
799 return unichar_error;
808 int num_chars1 = shape1.
size();
809 int num_chars2 = shape2.
size();
810 float dist_sum = 0.0f;
812 if (num_chars1 > 1 || num_chars2 > 1) {
815 for (
int c1 = 0; c1 < num_chars1; ++c1) {
816 for (
int c2 = 0; c2 < num_chars2; ++c2) {
829 return dist_sum / dist_count;
834 void MasterTrainer::ReplaceFragmentedSamples() {
835 if (fragments_ == NULL)
return;
839 for (
int s = 0; s < num_samples; ++s) {
841 if (fragments_[sample->
class_id()] > 0)
853 bool* good_junk =
new bool[frag_set.
size()];
854 memset(good_junk, 0,
sizeof(*good_junk) * frag_set.
size());
855 for (
int dead_ch = 1; dead_ch < unicharset_.
size(); ++dead_ch) {
856 int frag_ch = fragments_[dead_ch];
857 if (frag_ch <= 0)
continue;
861 for (
int part = 0; part < frag->
get_total(); ++part) {
864 if (good_ch != INVALID_UNICHAR_ID)
865 good_junk[good_ch] =
true;
873 for (
int s = 0; s < num_junks; ++s) {
891 delete [] fragments_;
901 void MasterTrainer::ClusterShapes(
int min_shapes,
int max_shape_unichars,
904 int max_merges = num_shapes - min_shapes;
910 tprintf(
"Computing shape distances...");
911 for (
int s1 = 0; s1 < num_shapes; ++s1) {
912 for (
int s2 = s1 + 1; s2 < num_shapes; ++s2) {
925 while (num_merged < max_merges && min_dist < max_dist) {
926 tprintf(
"Distance = %f: ", min_dist);
928 shape_dists[min_s1][min_s2 - min_s1 - 1].distance =
kInfiniteDist;
929 if (num_unichars > max_shape_unichars) {
930 tprintf(
"Merge of %d and %d with %d would exceed max of %d unichars\n",
931 min_s1, min_s2, num_unichars, max_shape_unichars);
934 shape_dists[min_s2].
clear();
937 for (
int s = 0; s < min_s1; ++s) {
938 if (!shape_dists[s].empty()) {
939 shape_dists[s][min_s1 - s - 1].distance =
944 for (
int s2 = min_s1 + 1; s2 < num_shapes; ++s2) {
945 if (shape_dists[min_s1][s2 - min_s1 - 1].distance < kInfiniteDist)
946 shape_dists[min_s1][s2 - min_s1 - 1].distance =
949 for (
int s = min_s1 + 1; s < min_s2; ++s) {
950 if (!shape_dists[s].empty()) {
956 for (
int s1 = 0; s1 < num_shapes; ++s1) {
957 for (
int i = 0; i < shape_dists[s1].
size(); ++i) {
958 if (shape_dists[s1][i].distance < min_dist) {
959 min_dist = shape_dists[s1][i].distance;
966 tprintf(
"Stopped with %d merged, min dist %f\n", num_merged, min_dist);
967 delete [] shape_dists;
968 if (debug_level_ > 1) {
969 for (
int s1 = 0; s1 < num_shapes; ++s1) {
int tfscanf(FILE *stream, const char *format,...)
const int kMaxUnicharsPerCluster
void MoveTo(UnicityTable< FontInfo > *target)
void ClearFeatureSpaceWindow(NORM_METHOD norm_method, ScrollView *window)
static double ComputeErrorRate(ShapeClassifier *classifier, int report_level, CountTypes boosting_mode, const FontInfoTable &fontinfo_table, const GenericVector< Pix *> &page_images, SampleIterator *it, double *unichar_error, double *scaled_error, STRING *fonts_report)
void add_spacing(UNICHAR_ID uch_id, FontSpacingInfo *spacing_info)
ScrollView * CreateFeatureSpaceWindow(const char *name, int xpos, int ypos)
void AppendMasterShapes(const ShapeTable &other, GenericVector< int > *shape_map)
int CompactCharsetSize() const
void Init(int size, bool all_mapped)
CLUSTERER * SetupForClustering(const ShapeTable &shape_table, const FEATURE_DEFS_STRUCT &feature_defs, int shape_id, int *num_samples)
bool Serialize(FILE *fp) const
int get_index(T object) const
void ComputeCloudFeatures(int feature_space_size)
MasterTrainer(NormalizationMode norm_mode, bool shape_analysis, bool replicate_samples, int debug_level)
bool LoadFontInfo(const char *filename)
const UNICHARSET & unicharset() const
UNICHAR_ID class_id() const
bool Serialize(FILE *fp) const
void WriteIntTemplates(FILE *File, INT_TEMPLATES Templates, const UNICHARSET &target_unicharset)
void init_to_size(int size, T t)
bool contains_unichar(const char *const unichar_repr) const
bool is_beginning() const
void AddSample(bool verification, const char *unichar_str, TrainingSample *sample)
STRING SummaryStr() const
const CHAR_FRAGMENT * get_fragment(UNICHAR_ID unichar_id) const
int MergedUnicharCount(int shape_id1, int shape_id2) const
void set_bounding_box(const TBOX &box)
STRING DebugStr(int shape_id) const
void MergeShapes(int shape_id1, int shape_id2)
const char * kMicroFeatureType
const PARAM_DESC * ParamDesc
void AppendOtherUnicharset(const UNICHARSET &src)
static void DebugNewErrors(ShapeClassifier *new_classifier, ShapeClassifier *old_classifier, CountTypes boosting_mode, const FontInfoTable &fontinfo_table, const GenericVector< Pix *> &page_images, SampleIterator *it)
const UNICHARSET & unicharset() const
const BitVector & GetCloudFeatures(int font_id, int class_id) const
void LoadUnicharset(const char *filename)
const char * string() const
int GetFontInfoId(const char *font_name)
void KillSample(TrainingSample *sample)
void Init(const IndexMapBiDi *charset_map, const ShapeTable *shape_table, bool randomize, TrainingSampleSet *sample_set)
GenericVector< UNICHAR_ID > kerned_unichar_ids
const TrainingSample * GetCanonicalSample(int font_id, int class_id) const
virtual const ShapeTable * GetShapeTable() const =0
UnicityTable< FontInfo > & get_fontinfo_table()
CHAR_DESC ReadCharDescription(const FEATURE_DEFS_STRUCT &FeatureDefs, FILE *File)
void ReplicateAndRandomizeSamplesIfRequired()
TrainingSample * extract_sample(int index)
void set_class_id(int id)
static STRING to_string(const char *unichar, int pos, int total, bool natural)
void set_page_num(int page)
int SparseCharsetSize() const
void TestClassifierVOld(bool replicate_samples, ShapeClassifier *test_classifier, ShapeClassifier *old_classifier)
void init_spacing(int unicharset_size)
const char * id_to_unichar(UNICHAR_ID id) const
CLUSTERER * MakeClusterer(inT16 SampleSize, const PARAM_DESC ParamDesc[])
int NumClassSamples(int font_id, int class_id, bool randomize) const
void DisplaySamplesWithFeature(int f_index, const Shape &shape, const IntFeatureSpace &feature_space, ScrollView::Color color, ScrollView *window) const
int XYToFeatureIndex(int x, int y) const
void IndexFeatures(const IntFeatureSpace &feature_space)
void ComputeCanonicalSamples(const IntFeatureMap &map, bool debug)
int MasterDestinationIndex(int shape_id) const
int num_micro_features() const
void WriteInttempAndPFFMTable(const UNICHARSET &unicharset, const UNICHARSET &shape_set, const ShapeTable &shape_table, CLASS_STRUCT *float_classes, const char *inttemp_file, const char *pffmtable_file)
#define ClassForClassId(T, c)
const Shape & GetShape(int shape_id) const
void RenderIntFeature(ScrollView *window, const INT_FEATURE_STRUCT *Feature, ScrollView::Color color)
void DisplaySamples(const char *unichar_str1, int cloud_font, const char *unichar_str2, int canonical_font)
void LoadPageImages(const char *filename)
bool ParseBoxFileStr(const char *boxfile_str, int *page_number, STRING *utf8_str, TBOX *bounding_box)
uinT16 ConfigLengths[MAX_NUM_CONFIGS]
void FreeCharDescription(CHAR_DESC CharDesc)
bool Serialize(FILE *fp) const
bool contains(T object) const
void ReadTrainingSamples(const char *page_name, const FEATURE_DEFS_STRUCT &feature_defs, bool verification)
float ShapeDistance(const ShapeTable &shapes, int s1, int s2)
const FEATURE_DESC_STRUCT * FeatureDesc[NUM_FEATURE_TYPES]
static CHAR_FRAGMENT * parse_from_string(const char *str)
GenericVector< inT16 > kerned_x_gaps
int ShortNameToFeatureType(const FEATURE_DEFS_STRUCT &FeatureDefs, const char *ShortName)
void AddToShape(int unichar_id, int font_id)
const char * kCNFeatureType
double NormalizeSamples()
bool load_from_file(const char *const filename, bool skip_fragments)
FILE * Efopen(const char *Name, const char *Mode)
void SetupFlatShapeTable(ShapeTable *shape_table)
void OrganizeByFontAndClass()
void ExtractCharDesc(int feature_type, int micro_type, int cn_type, int geo_type, CHAR_DESC_STRUCT *char_desc)
const float kFontMergeDistance
UnicityTableEqEq< int > font_set
bool save_to_file(const char *const filename) const
INT_FEATURE_STRUCT InverseIndexFeature(int index_feature) const
const char * kIntFeatureType
bool Serialize(FILE *fp) const
const TrainingSample & GetSample() const
const INT_FEATURE_STRUCT * features() const
float UnicharDistance(const UnicharAndFonts &uf1, const UnicharAndFonts &uf2, bool matched_fonts, const IntFeatureMap &feature_map)
FEATURE_DEFS_STRUCT feature_defs
int GetBestMatchingFontInfoId(const char *filename)
bool Serialize(FILE *fp) const
int AddShape(int unichar_id, int font_id)
const int kMinClusteredShapes
const T & get(int id) const
Return the object from an id.
void ReplicateAndRandomizeSamples()
double TestClassifier(CountTypes error_mode, int report_level, bool replicate_samples, TrainingSampleSet *samples, ShapeClassifier *test_classifier, STRING *report_string)
bool Serialize(FILE *fp) const
void free_int_templates(INT_TEMPLATES templates)
INT_TEMPLATES CreateIntTemplates(CLASSES FloatProtos, const UNICHARSET &target_unicharset)
void DebugCanonical(const char *unichar_str1, const char *unichar_str2)
bool AddSpacingInfo(const char *filename)
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
void LoadUnicharset(const char *filename)
bool LoadXHeights(const char *filename)
float ClusterDistance(int font_id1, int class_id1, int font_id2, int class_id2, const IntFeatureMap &feature_map)
const char * kGeoFeatureType
int FindShape(int unichar_id, int font_id) const
int AddSample(const char *unichar, TrainingSample *sample)
void TestClassifierOnSamples(CountTypes error_mode, int report_level, bool replicate_samples, ShapeClassifier *test_classifier, STRING *report_string)
int DivRounded(int a, int b)
TrainingSample * mutable_sample(int index)
const IntFeatureSpace & feature_space() const
SVEvent * AwaitEvent(SVEventType type)
void ComputeCanonicalFeatures()
SAMPLE * MakeSample(CLUSTERER *Clusterer, const FLOAT32 *Feature, inT32 CharID)
void SetMap(int sparse_index, bool mapped)
const MicroFeature * micro_features() const
const float kInfiniteDist