19 #ifndef TESSERACT_DICT_DICT_H_ 20 #define TESSERACT_DICT_DICT_H_ 35 #define MAX_WERD_LENGTH (inT64) 128 54 static const int kRatingPad = 4;
55 static const char kDictWildcard[] =
"\u2606";
56 static const int kDictMaxWildcards = 2;
59 static const char kHyphenSymbol[] =
"-";
60 static const char kSlashSymbol[] =
"/";
61 static const char kQuestionSymbol[] =
"?";
62 static const char kApostropheSymbol[] =
"'";
63 static const float kSimCertaintyScale = -10.0;
64 static const float kSimCertaintyOffset = -10.0;
65 static const float kSimilarityFloor = 100.0;
66 static const int kDocDictMaxRepChars = 4;
78 : active_dawgs(d), updated_dawgs(up), permuter(p), valid_end(false) {}
98 return getCCUtil()->unicharset;
101 return getCCUtil()->unicharset;
104 return getCCUtil()->unichar_ambigs;
110 getUnicharset().normed_ids(unichar_id);
111 return normed_ids.
size() == 1 &&
112 (normed_ids[0] == hyphen_unichar_id_ ||
113 normed_ids[0] == slash_unichar_id_);
119 getUnicharset().normed_ids(unichar_id);
120 return normed_ids.
size() == 1 && normed_ids[0] == apostrophe_unichar_id_;
127 !last_word_on_line_ && hyphen_word_;
131 return this->hyphenated() ? hyphen_word_->length() : 0;
137 if (this->hyphenated()) {
138 *word = *hyphen_word_;
139 if (hyphen_debug_level) word->
print(
"copy_hyphen_info: ");
144 if (!last_word_on_line_ || first_pos)
147 getUnicharset().normed_ids(unichar_id);
148 return normed_ids.
size() == 1 && normed_ids[0] == hyphen_unichar_id_;
152 int word_index = word.
length() - 1;
153 return has_hyphen_end(word.
unichar_id(word_index), word_index == 0);
158 void reset_hyphen_vars(
bool last_word_on_line);
180 bool ambigs_mode)
const;
183 bool suppress_patterns)
const;
196 void go_deeper_dawg_fxn(
199 bool word_ending,
WERD_CHOICE *word,
float certainties[],
200 float *limit,
WERD_CHOICE *best_choice,
int *attempts_left,
201 void *void_more_args);
204 void (
Dict::*go_deeper_fxn_)(
const char *debug,
206 int char_choice_index,
209 float certainties[],
float *limit,
211 void *void_more_args);
215 void permute_choices(
218 int char_choice_index,
231 int char_choice_index,
240 bool fragment_state_okay(
UNICHAR_ID curr_unichar_id,
241 float curr_rating,
float curr_certainty,
243 const char *debug,
int word_ending,
249 bool fix_replaceable,
258 void ReplaceAmbig(
int wrong_ngram_begin_index,
int wrong_ngram_size,
263 int LengthOfShortestAlphaRun(
const WERD_CHOICE &WordChoice)
const;
273 bool AcceptableChoice(
const WERD_CHOICE& best_choice,
278 bool AcceptableResult(
WERD_RES *word)
const;
279 void EndDangerousAmbigs();
281 void DebugWordChoices();
283 void SettupStopperPass1();
285 void SettupStopperPass2();
299 void SetupForLoad(
DawgCache *dawg_cache);
311 if (pending_words_ != NULL)
312 pending_words_->clear();
313 if (document_words_ != NULL)
314 document_words_->clear();
353 int def_letter_is_okay(
void* void_dawg_args,
356 int (
Dict::*letter_is_okay_)(
void* void_dawg_args,
361 return (this->*letter_is_okay_)(void_dawg_args,
unichar_id, word_end);
366 double (
Dict::*probability_in_context_)(
const char*
lang,
370 int character_bytes);
374 const char* character,
375 int character_bytes) {
376 return (this->*probability_in_context_)(
377 getCCUtil()->lang.string(),
378 context, context_bytes,
384 const char* lang,
const char* context,
int context_bytes,
385 const char* character,
int character_bytes) {
390 (void)character_bytes;
393 double ngram_probability_in_context(
const char* lang,
396 const char* character,
397 int character_bytes);
400 float (
Dict::*params_model_classify_)(
const char *
lang,
void *path);
401 float ParamsModelClassify(
const char *lang,
void *path);
405 return (this->*params_model_classify_)(
406 getCCUtil()->lang.string(), path);
412 inline int NumDawgs()
const {
return dawgs_.size(); }
414 inline const Dawg *
GetDawg(
int index)
const {
return dawgs_[index]; }
421 if (edge_ref == NO_EDGE)
return 0;
423 if (node == 0) node = NO_EDGE;
431 if (!dawg)
return ch;
432 switch (dawg->
type()) {
434 return getUnicharset().get_isdigit(ch) ? Dawg::kPatternUnicharID : ch;
461 int valid_word(
const WERD_CHOICE &word,
bool numbers_ok)
const;
463 return valid_word(word,
false);
466 return valid_word(word,
true);
471 return valid_word(word);
483 void add_document_word(
const WERD_CHOICE &best_choice);
487 float additional_adjust,
492 wordseg_rating_adjust_factor_ = f;
495 bool IsSpaceDelimitedLang()
const;
520 bool last_word_on_line_;
528 bool dawg_cache_is_ours_;
532 Trie *pending_words_;
546 Trie *document_words_;
549 float wordseg_rating_adjust_factor_;
551 FILE *output_ambig_words_file_;
557 STRING_VAR_H(user_words_file,
"",
"A filename of user-provided words.");
559 "A suffix of user-provided words located in tessdata.");
561 "A filename of user-provided patterns.");
563 "A suffix of user-provided patterns located in tessdata.");
564 BOOL_VAR_H(load_system_dawg,
true,
"Load system word dawg.");
565 BOOL_VAR_H(load_freq_dawg,
true,
"Load frequent word dawg.");
566 BOOL_VAR_H(load_unambig_dawg,
true,
"Load unambiguous word dawg.");
568 "Load dawg with punctuation patterns.");
569 BOOL_VAR_H(load_number_dawg,
true,
"Load dawg with number patterns.");
571 "Load dawg with special word bigrams.");
573 "Score penalty (0.1 = 10%) added if there are subscripts " 574 "or superscripts in a word, but it is otherwise OK.");
576 "Score penalty (0.1 = 10%) added if an xheight is " 579 "Score multiplier for word matches which have good case and" 580 "are frequent in the given language (lower is better).");
583 "Score multiplier for word matches that have good case " 584 "(lower is better).");
587 "Default score multiplier for word matches, which may have " 588 "case issues (lower is better).");
592 "Multipler to for the best choice from the ngram model.");
595 "Score multiplier for glyph fragment segmentations which " 596 "do not match a dictionary word (lower is better).");
599 "Score multiplier for poorly cased strings that are not in" 600 " the dictionary and generally look like garbage (lower is" 603 "Output file for ambiguities found in the dictionary");
604 INT_VAR_H(dawg_debug_level, 0,
"Set to 1 for general debug info" 605 ", to 2 for more details, to 3 to see all the debug messages");
606 INT_VAR_H(hyphen_debug_level, 0,
"Debug level for hyphenated words.");
607 INT_VAR_H(max_viterbi_list_size, 10,
"Maximum size of viterbi list.");
609 "Use only the first UTF8 step of the given string" 610 " when computing log probabilities.");
613 "Certainty threshold for non-dict words");
614 double_VAR_H(stopper_phase2_certainty_rejection_offset, 1.0,
615 "Reject certainty offset");
617 "Size of dict word to be treated as non-dict word");
619 "Certainty to add for each dict char above small word size.");
621 "Max certaintly variation allowed in a word (in sigma)");
622 INT_VAR_H(stopper_debug_level, 0,
"Stopper debug level");
623 BOOL_VAR_H(stopper_no_acceptable_choices,
false,
624 "Make AcceptableChoice() always return false. Useful" 625 " when there is a need to explore all segmentations");
627 "Deprecated- backward compatibility only");
628 INT_VAR_H(tessedit_truncate_wordchoice_log, 10,
"Max words to keep in list");
629 STRING_VAR_H(word_to_debug,
"",
"Word for which stopper debug information" 630 " should be printed to stdout");
632 "Lengths of unichars in word_to_debug");
633 INT_VAR_H(fragments_debug, 0,
"Debug character fragments");
634 BOOL_VAR_H(segment_nonalphabetic_script,
false,
635 "Don't use any alphabetic-specific tricks." 636 "Set to true in the traineddata config file for" 637 " scripts that are cursive or inherently fixed-pitch");
640 "Worst certainty for using pending dictionary");
641 double_VAR_H(doc_dict_certainty_threshold, -2.25,
"Worst certainty" 642 " for words that can be inserted into the document dictionary");
643 INT_VAR_H(max_permuter_attempts, 10000,
"Maximum number of different" 644 " character choices to consider during permutation." 645 " This limit is especially useful when user patterns" 646 " are specified, since overly generic patterns can result in" 647 " dawg search exploring an overly large number of options.");
651 #endif // THIRD_PARTY_TESSERACT_DICT_DICT_H_ void update_best_choice(const WERD_CHOICE &word, WERD_CHOICE *best_choice)
double def_probability_in_context(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)
Default (no-op) implementation of probability in context function.
double ProbabilityInContext(const char *context, int context_bytes, const char *character, int character_bytes)
Calls probability_in_context_ member function.
static NODE_REF GetStartingNode(const Dawg *dawg, EDGE_REF edge_ref)
Returns the appropriate next node given the EDGE_REF.
virtual NODE_REF next_node(EDGE_REF edge_ref) const =0
const UNICHARSET & getUnicharset() const
bool has_hyphen_end(const WERD_CHOICE &word) const
Same as above, but check the unichar at the end of the word.
UNICHAR_ID unichar_id(int index) const
UNICHAR_ID char_for_dawg(UNICHAR_ID ch, const Dawg *dawg) const
static bool valid_word_permuter(uinT8 perm, bool numbers_ok)
Check all the DAWGs to see if this word is in any of them.
const Dawg * GetUnambigDawg() const
Return the points to the unambiguous words dawg.
const CHAR_FRAGMENT * fragment
bool compound_marker(UNICHAR_ID unichar_id)
UNICHARSET & getUnicharset()
int valid_word(const WERD_CHOICE &word) const
int hyphen_base_size() const
Size of the base word (the part on the line before) of a hyphenated word.
float CallParamsModelClassify(void *path)
const Dawg * GetPuncDawg() const
Return the points to the punctuation dawg.
const CCUtil * getCCUtil() const
const Dawg * GetDawg(int index) const
Return i-th dawg pointer recorded in the dawgs_ vector.
void SetWildcardID(UNICHAR_ID id)
int NumDawgs() const
Return the number of dawgs in the dawgs_ vector.
#define double_VAR_H(name, val, comment)
DawgPositionVector * updated_dawgs
bool is_apostrophe(UNICHAR_ID unichar_id)
const UnicharAmbigs & getUnicharAmbigs() const
void copy_hyphen_info(WERD_CHOICE *word) const
bool hyphenated() const
Returns true if we've recorded the beginning of a hyphenated word.
void SetWordsegRatingAdjustFactor(float f)
Set wordseg_rating_adjust_factor_ to the given value.
void ResetDocumentDictionary()
int LetterIsOkay(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
Calls letter_is_okay_ member function.
#define STRING_VAR_H(name, val, comment)
DawgPositionVector * active_dawgs
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
GenericVector< Dawg * > DawgVector
#define INT_VAR_H(name, val, comment)
int valid_word_or_number(const WERD_CHOICE &word) const
#define BOOL_VAR_H(name, val, comment)
int valid_word(const char *string) const
This function is used by api/tesseract_cube_combiner.cpp.
DawgArgs(DawgPositionVector *d, DawgPositionVector *up, PermuterType p)
UNICHAR_ID WildcardID() const
bool has_hyphen_end(UNICHAR_ID unichar_id, bool first_pos) const
Check whether the word has a hyphen at the end.