17 #include "allheaders.h" 39 TrainingSampleSet::FontClassInfo::FontClassInfo()
40 : num_raw_samples(0), canonical_sample(-1), canonical_dist(0.0f) {
44 bool TrainingSampleSet::FontClassInfo::Serialize(FILE* fp)
const {
45 if (fwrite(&num_raw_samples,
sizeof(num_raw_samples), 1, fp) != 1)
47 if (fwrite(&canonical_sample,
sizeof(canonical_sample), 1, fp) != 1)
49 if (fwrite(&canonical_dist,
sizeof(canonical_dist), 1, fp) != 1)
return false;
50 if (!samples.Serialize(fp))
return false;
55 bool TrainingSampleSet::FontClassInfo::DeSerialize(
bool swap, FILE* fp) {
56 if (fread(&num_raw_samples,
sizeof(num_raw_samples), 1, fp) != 1)
58 if (fread(&canonical_sample,
sizeof(canonical_sample), 1, fp) != 1)
60 if (fread(&canonical_dist,
sizeof(canonical_dist), 1, fp) != 1)
return false;
61 if (!samples.DeSerialize(swap, fp))
return false;
63 ReverseN(&num_raw_samples,
sizeof(num_raw_samples));
64 ReverseN(&canonical_sample,
sizeof(canonical_sample));
65 ReverseN(&canonical_dist,
sizeof(canonical_dist));
71 : num_raw_samples_(0), unicharset_size_(0),
72 font_class_array_(NULL), fontinfo_table_(font_table) {
76 delete font_class_array_;
81 if (!samples_.Serialize(fp))
return false;
83 if (!font_id_map_.
Serialize(fp))
return false;
84 inT8 not_null = font_class_array_ != NULL;
85 if (fwrite(¬_null,
sizeof(not_null), 1, fp) != 1)
return false;
95 if (!samples_.DeSerialize(swap, fp))
return false;
96 num_raw_samples_ = samples_.size();
98 if (!font_id_map_.
DeSerialize(swap, fp))
return false;
99 delete font_class_array_;
100 font_class_array_ = NULL;
102 if (fread(¬_null,
sizeof(not_null), 1, fp) != 1)
return false;
108 unicharset_size_ = unicharset_.
size();
115 tprintf(
"Failed to load unicharset from file %s\n" 116 "Building unicharset from scratch...\n",
123 unicharset_size_ = unicharset_.
size();
133 tprintf(
"Error: Size of unicharset in TrainingSampleSet::AddSample is " 134 "greater than MAX_NUM_CLASSES\n");
147 samples_.push_back(sample);
148 num_raw_samples_ = samples_.size();
149 unicharset_size_ = unicharset_.
size();
157 bool randomize)
const {
159 if (font_id < 0 || class_id < 0 ||
160 font_id >= font_id_map_.
SparseSize() || class_id >= unicharset_size_) {
168 return (*font_class_array_)(font_index, class_id).samples.size();
175 return samples_[index];
184 if (font_index < 0)
return NULL;
185 int sample_index = (*font_class_array_)(font_index, class_id).samples[index];
186 return samples_[sample_index];
195 if (font_index < 0)
return NULL;
196 int sample_index = (*font_class_array_)(font_index, class_id).samples[index];
197 return samples_[sample_index];
206 return STRING(fontinfo_table_.
get(sample.
font_id()).name) +
" " + boxfile_str;
212 int font_id,
int class_id)
const {
215 return (*font_class_array_)(font_index, class_id).cloud_features;
220 int font_id,
int class_id)
const {
223 return (*font_class_array_)(font_index, class_id).canonical_features;
238 double dist_sum = 0.0;
243 for (
int i = 0; i < num_fonts1; ++i) {
245 for (
int j = 0; j < num_fonts2; ++j) {
253 }
else if (num_fonts1 * num_fonts2 <= kSquareLimit) {
255 for (
int i = 0; i < num_fonts1; ++i) {
257 for (
int j = 0; j < num_fonts2; ++j) {
261 tprintf(
"Cluster dist %d %d %d %d = %g\n",
274 for (
int i = 0; i <
num_samples; ++i, index += increment) {
275 int f1 = uf1.
font_ids[i % num_fonts1];
276 int f2 = uf2.
font_ids[index % num_fonts2];
278 tprintf(
"Cluster dist %d %d %d %d = %g\n",
285 if (dist_count == 0) {
290 return dist_sum / dist_count;
297 int font_id2,
int class_id2,
302 if (font_index1 < 0 || font_index2 < 0)
304 FontClassInfo& fc_info = (*font_class_array_)(font_index1, class_id1);
305 if (font_id1 == font_id2) {
307 if (fc_info.unichar_distance_cache.size() == 0)
308 fc_info.unichar_distance_cache.init_to_size(unicharset_size_, -1.0f);
309 if (fc_info.unichar_distance_cache[class_id2] < 0) {
314 fc_info.unichar_distance_cache[class_id2] = result;
316 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
317 if (fc_info2.unichar_distance_cache.size() == 0)
318 fc_info2.unichar_distance_cache.init_to_size(unicharset_size_, -1.0f);
319 fc_info2.unichar_distance_cache[class_id1] = result;
321 return fc_info.unichar_distance_cache[class_id2];
322 }
else if (class_id1 == class_id2) {
324 if (fc_info.font_distance_cache.size() == 0)
325 fc_info.font_distance_cache.init_to_size(font_id_map_.
CompactSize(),
327 if (fc_info.font_distance_cache[font_index2] < 0) {
332 fc_info.font_distance_cache[font_index2] = result;
334 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
335 if (fc_info2.font_distance_cache.size() == 0)
336 fc_info2.font_distance_cache.init_to_size(font_id_map_.
CompactSize(),
338 fc_info2.font_distance_cache[font_index1] = result;
340 return fc_info.font_distance_cache[font_index2];
345 while (cache_index < fc_info.distance_cache.size() &&
346 (fc_info.distance_cache[cache_index].unichar_id != class_id2 ||
347 fc_info.distance_cache[cache_index].font_id != font_id2))
349 if (cache_index == fc_info.distance_cache.size()) {
354 FontClassDistance fc_dist = { class_id2, font_id2, result };
355 fc_info.distance_cache.push_back(fc_dist);
358 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
359 fc_dist.unichar_id = class_id1;
360 fc_dist.font_id = font_id1;
361 fc_info2.distance_cache.push_back(fc_dist);
363 return fc_info.distance_cache[cache_index].distance;
368 int font_id1,
int class_id1,
int font_id2,
int class_id2,
376 return static_cast<float>(dist) / denominator;
382 static void AddNearFeatures(
const IntFeatureMap& feature_map,
int f,
int levels,
384 int prev_num_features = 0;
386 int num_features = 1;
387 for (
int level = 0; level < levels; ++level) {
388 for (
int i = prev_num_features; i < num_features; ++i) {
389 int feature = (*good_features)[i];
390 for (
int dir = -kNumOffsetMaps; dir <= kNumOffsetMaps; ++dir) {
391 if (dir == 0)
continue;
398 prev_num_features = num_features;
399 num_features = good_features->
size();
414 int font_id2,
int class_id2,
416 bool thorough)
const {
424 if (cloud1.
size() == 0)
425 return canonical2.
size();
428 for (
int f = 0; f < canonical2.
size(); ++f) {
429 int feature = canonical2[f];
434 AddNearFeatures(feature_map, feature, 1, &good_features);
437 for (i = 0; i < good_features.
size(); ++i) {
438 int good_f = good_features[i];
439 if (cloud1[good_f]) {
443 if (i < good_features.
size())
456 if (font_index < 0)
return -1;
457 return (*font_class_array_)(font_index, class_id).samples[index];
463 int font_id,
int class_id)
const {
466 if (font_index < 0)
return NULL;
467 int sample_index = (*font_class_array_)(font_index,
468 class_id).canonical_sample;
469 return sample_index >= 0 ? samples_[sample_index] : NULL;
477 if (font_index < 0)
return 0.0f;
478 if ((*font_class_array_)(font_index, class_id).canonical_sample >= 0)
479 return (*font_class_array_)(font_index, class_id).canonical_dist;
486 for (
int s = 0; s < samples_.size(); ++s)
500 num_raw_samples_ = samples_.size();
507 return sample == NULL || sample->
class_id() < 0;
515 int compact_font_size = font_id_map_.
CompactSize();
517 if (font_class_array_ != NULL)
518 delete font_class_array_;
521 compact_font_size, unicharset_size_, empty);
522 for (
int s = 0; s < samples_.size(); ++s) {
523 int font_id = samples_[s]->font_id();
524 int class_id = samples_[s]->class_id();
525 if (font_id < 0 || font_id >= font_id_map_.
SparseSize()) {
526 tprintf(
"Font id = %d/%d, class id = %d/%d on sample %d\n",
527 font_id, font_id_map_.
SparseSize(), class_id, unicharset_size_,
531 ASSERT_HOST(class_id >= 0 && class_id < unicharset_size_);
533 (*font_class_array_)(font_index, class_id).samples.push_back(s);
537 for (
int f = 0; f < compact_font_size; ++f) {
538 for (
int c = 0; c < unicharset_size_; ++c)
540 (*font_class_array_)(f, c).samples.size();
544 num_raw_samples_ = samples_.size();
552 for (
int s = 0; s < samples_.size(); ++s) {
553 int font_id = samples_[s]->font_id();
554 while (font_id >= font_counts.
size())
556 ++font_counts[font_id];
558 font_id_map_.
Init(font_counts.
size(),
false);
559 for (
int f = 0; f < font_counts.
size(); ++f) {
560 font_id_map_.
SetMap(f, font_counts[f] > 0);
562 font_id_map_.
Setup();
577 double global_worst_dist = 0.0;
580 for (
int font_index = 0; font_index < font_size; ++font_index) {
582 for (
int c = 0; c < unicharset_size_; ++c) {
583 int samples_found = 0;
584 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
585 if (fcinfo.samples.size() == 0 ||
587 fcinfo.canonical_sample = -1;
588 fcinfo.canonical_dist = 0.0f;
589 if (debug)
tprintf(
"Skipping class %d\n", c);
594 double min_max_dist = 2.0;
597 double max_max_dist = 0.0;
600 fcinfo.canonical_sample = fcinfo.samples[0];
601 fcinfo.canonical_dist = 0.0f;
602 for (
int i = 0; i < fcinfo.samples.size(); ++i) {
603 int s1 = fcinfo.samples[i];
605 f_table.
Set(features1, features1.
size(),
true);
606 double max_dist = 0.0;
611 for (
int j = 0; j < fcinfo.samples.size(); ++j) {
612 int s2 = fcinfo.samples[j];
613 if (samples_[s2]->class_id() != c ||
614 samples_[s2]->font_id() != font_id ||
619 if (dist > max_dist) {
621 if (dist > max_max_dist) {
629 f_table.
Set(features1, features1.
size(),
false);
630 samples_[s1]->set_max_dist(max_dist);
632 if (max_dist < min_max_dist) {
633 fcinfo.canonical_sample = s1;
634 fcinfo.canonical_dist = max_dist;
636 UpdateRange(max_dist, &min_max_dist, &max_max_dist);
638 if (max_max_dist > global_worst_dist) {
640 global_worst_dist = max_max_dist;
645 tprintf(
"Found %d samples of class %d=%s, font %d, " 646 "dist range [%g, %g], worst pair= %s, %s\n",
648 font_index, min_max_dist, max_max_dist,
655 tprintf(
"Global worst dist = %g, between sample %d and %d\n",
656 global_worst_dist, worst_s1, worst_s2);
668 for (
int font_index = 0; font_index < font_size; ++font_index) {
669 for (
int c = 0; c < unicharset_size_; ++c) {
670 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
671 int sample_count = fcinfo.samples.size();
672 int min_samples = 2 *
MAX(kSampleRandomSize, sample_count);
673 if (sample_count > 0 && sample_count < min_samples) {
674 int base_count = sample_count;
675 for (
int base_index = 0; sample_count < min_samples; ++sample_count) {
676 int src_index = fcinfo.samples[base_index++];
677 if (base_index >= base_count) base_index = 0;
679 sample_count % kSampleRandomSize);
680 int sample_index = samples_.size();
682 samples_.push_back(sample);
683 fcinfo.samples.push_back(sample_index);
697 for (
int font_index = 0; font_index < font_size; ++font_index) {
699 for (
int c = 0; c < unicharset_size_; ++c) {
701 if (num_samples == 0)
704 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
715 for (
int font_index = 0; font_index < font_size; ++font_index) {
717 for (
int c = 0; c < unicharset_size_; ++c) {
719 if (num_samples == 0)
721 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
722 fcinfo.cloud_features.Init(feature_space_size);
726 for (
int i = 0; i < sample_features.
size(); ++i)
727 fcinfo.cloud_features.SetBit(sample_features[i]);
735 for (
int f = 0; f < font_id_map_.
CompactSize(); ++f) {
754 for (
int f = 0; f < indexed_features.
size(); ++f) {
755 if (indexed_features[f] == f_index) {
int GlobalSampleIndex(int font_id, int class_id, int index) const
const int kMinOutlierSamples
void Init(int size, bool all_mapped)
void ComputeCloudFeatures(int feature_space_size)
UNICHAR_ID class_id() const
void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, STRING *box_str)
_ConstTessMemberResultCallback_0_0< false, R, T1 >::base * NewPermanentTessCallback(const T1 *obj, R(T2::*member)() const)
bool contains_unichar(const char *const unichar_repr) const
void AddAllFontsForClass(int class_id, Shape *shape) const
void set_sample_index(int value)
const TrainingSample * GetSample(int index) const
int num_raw_samples() const
int CompactToSparse(int compact_index) const
bool DeSerializeClasses(bool swap, FILE *fp)
const GenericVector< int > & GetCanonicalFeatures(int font_id, int class_id) const
void AppendOtherUnicharset(const UNICHARSET &src)
const BitVector & GetCloudFeatures(int font_id, int class_id) const
void LoadUnicharset(const char *filename)
const char * string() const
void KillSample(TrainingSample *sample)
const TrainingSample * GetCanonicalSample(int font_id, int class_id) const
void set_class_id(int id)
const char * id_to_unichar(UNICHAR_ID id) const
int NumClassSamples(int font_id, int class_id, bool randomize) const
void DisplaySamplesWithFeature(int f_index, const Shape &shape, const IntFeatureSpace &feature_space, ScrollView::Color color, ScrollView *window) const
void IndexFeatures(const IntFeatureSpace &feature_space)
bool SerializeClasses(FILE *fp) const
void ComputeCanonicalSamples(const IntFeatureMap &map, bool debug)
void Init(const IntFeatureMap *feature_map)
bool Serialize(FILE *fp) const
float ComputeClusterDistance(int font_id1, int class_id1, int font_id2, int class_id2, const IntFeatureMap &feature_map) const
int OffsetFeature(int index_feature, int dir) const
STRING SampleToString(const TrainingSample &sample) const
void IndexAndSortFeatures(const INT_FEATURE_STRUCT *features, int num_features, GenericVector< int > *sorted_features) const
float GetCanonicalDist(int font_id, int class_id) const
bool DeleteableSample(const TrainingSample *sample)
bool Serialize(FILE *fp) const
int ReliablySeparable(int font_id1, int class_id1, int font_id2, int class_id2, const IntFeatureMap &feature_map, bool thorough) const
virtual int SparseToCompact(int sparse_index) const
TrainingSample * MutableSample(int font_id, int class_id, int index)
void AddToShape(int unichar_id, int font_id)
bool load_from_file(const char *const filename, bool skip_fragments)
void OrganizeByFontAndClass()
GenericVector< inT32 > font_ids
bool DeSerialize(bool swap, FILE *fp)
bool save_to_file(const char *const filename) const
const INT_FEATURE_STRUCT * features() const
float UnicharDistance(const UnicharAndFonts &uf1, const UnicharAndFonts &uf2, bool matched_fonts, const IntFeatureMap &feature_map)
bool ContainsUnichar(int unichar_id) const
void ReplicateAndRandomizeSamples()
void DisplayFeatures(ScrollView::Color color, ScrollView *window) const
const GenericVector< int > & indexed_features() const
bool DeSerialize(bool swap, FILE *fp)
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
float ClusterDistance(int font_id1, int class_id1, int font_id2, int class_id2, const IntFeatureMap &feature_map)
virtual int SparseSize() const
double FeatureDistance(const GenericVector< int > &features) const
void UpdateRange(const T1 &x, T2 *lower_bound, T2 *upper_bound)
int AddSample(const char *unichar, TrainingSample *sample)
const TBOX & bounding_box() const
STRING debug_str(UNICHAR_ID id) const
void unichar_insert(const char *const unichar_repr)
void Set(const GenericVector< int > &indexed_features, int canonical_count, bool value)
void ComputeCanonicalFeatures()
void ReverseN(void *ptr, int num_bytes)
TrainingSampleSet(const FontInfoTable &fontinfo_table)
void SetMap(int sparse_index, bool mapped)