tesseract  4.00.00dev
unicharset.h
Go to the documentation of this file.
1 // File: unicharset.h
3 // Description: Unicode character/ligature set class.
4 // Author: Thomas Kielbus
5 // Created: Wed Jun 28 17:05:01 PDT 2006
6 //
7 // (C) Copyright 2006, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
19 
20 #ifndef TESSERACT_CCUTIL_UNICHARSET_H_
21 #define TESSERACT_CCUTIL_UNICHARSET_H_
22 
23 #include "errcode.h"
24 #include "genericvector.h"
25 #include "helpers.h"
26 #include "serialis.h"
27 #include "strngs.h"
28 #include "tesscallback.h"
29 #include "unichar.h"
30 #include "unicharmap.h"
31 
32 // Enum holding special values of unichar_id. Every unicharset has these.
33 // Warning! Keep in sync with kSpecialUnicharCodes.
38 
40 };
41 
43  public:
44  // Minimum number of characters used for fragment representation.
45  static const int kMinLen = 6;
46  // Maximum number of characters used for fragment representation.
47  static const int kMaxLen = 3 + UNICHAR_LEN + 2;
48  // Maximum number of fragments per character.
49  static const int kMaxChunks = 5;
50 
51  // Setters and Getters.
52  inline void set_all(const char *unichar, int pos, int total, bool natural) {
53  set_unichar(unichar);
54  set_pos(pos);
55  set_total(total);
56  set_natural(natural);
57  }
58  inline void set_unichar(const char *uch) {
59  strncpy(this->unichar, uch, UNICHAR_LEN);
60  this->unichar[UNICHAR_LEN] = '\0';
61  }
62  inline void set_pos(int p) { this->pos = p; }
63  inline void set_total(int t) { this->total = t; }
64  inline const char* get_unichar() const { return this->unichar; }
65  inline int get_pos() const { return this->pos; }
66  inline int get_total() const { return this->total; }
67 
68  // Returns the string that represents a fragment
69  // with the given unichar, pos and total.
70  static STRING to_string(const char *unichar, int pos, int total,
71  bool natural);
72  // Returns the string that represents this fragment.
73  STRING to_string() const {
74  return to_string(unichar, pos, total, natural);
75  }
76 
77  // Checks whether a fragment has the same unichar,
78  // position and total as the given inputs.
79  inline bool equals(const char *other_unichar,
80  int other_pos, int other_total) const {
81  return (strcmp(this->unichar, other_unichar) == 0 &&
82  this->pos == other_pos && this->total == other_total);
83  }
84  inline bool equals(const CHAR_FRAGMENT *other) const {
85  return this->equals(other->get_unichar(),
86  other->get_pos(),
87  other->get_total());
88  }
89 
90  // Checks whether a given fragment is a continuation of this fragment.
91  // Assumes that the given fragment pointer is not NULL.
92  inline bool is_continuation_of(const CHAR_FRAGMENT *fragment) const {
93  return (strcmp(this->unichar, fragment->get_unichar()) == 0 &&
94  this->total == fragment->get_total() &&
95  this->pos == fragment->get_pos() + 1);
96  }
97 
98  // Returns true if this fragment is a beginning fragment.
99  inline bool is_beginning() const { return this->pos == 0; }
100 
101  // Returns true if this fragment is an ending fragment.
102  inline bool is_ending() const { return this->pos == this->total-1; }
103 
104  // Returns true if the fragment was a separate component to begin with,
105  // ie did not need chopping to be isolated, but may have been separated
106  // out from a multi-outline blob.
107  inline bool is_natural() const { return natural; }
108  void set_natural(bool value) { natural = value; }
109 
110  // Parses the string to see whether it represents a character fragment
111  // (rather than a regular character). If so, allocates memory for a new
112  // CHAR_FRAGMENT instance and fills it in with the corresponding fragment
113  // information. Fragments are of the form:
114  // |m|1|2, meaning chunk 1 of 2 of character m, or
115  // |:|1n2, meaning chunk 1 of 2 of character :, and no chopping was needed
116  // to divide the parts, as they were already separate connected components.
117  //
118  // If parsing succeeded returns the pointer to the allocated CHAR_FRAGMENT
119  // instance, otherwise (if the string does not represent a fragment or it
120  // looks like it does, but parsing it as a fragment fails) returns NULL.
121  //
122  // Note: The caller is responsible for deallocating memory
123  // associated with the returned pointer.
124  static CHAR_FRAGMENT *parse_from_string(const char *str);
125 
126  private:
127  char unichar[UNICHAR_LEN + 1];
128  // True if the fragment was a separate component to begin with,
129  // ie did not need chopping to be isolated, but may have been separated
130  // out from a multi-outline blob.
131  bool natural;
132  inT16 pos; // fragment position in the character
133  inT16 total; // total number of fragments in the character
134 };
135 
136 // The UNICHARSET class is an utility class for Tesseract that holds the
137 // set of characters that are used by the engine. Each character is identified
138 // by a unique number, from 0 to (size - 1).
139 class UNICHARSET {
140  public:
141  // Custom list of characters and their ligature forms (UTF8)
142  // These map to unicode values in the private use area (PUC) and are supported
143  // by only few font families (eg. Wyld, Adobe Caslon Pro).
144  static TESS_API const char* kCustomLigatures[][2];
145 
146  // List of strings for the SpecialUnicharCodes. Keep in sync with the enum.
147  static const char* kSpecialUnicharCodes[SPECIAL_UNICHAR_CODES_COUNT];
148 
149  // ICU 2.0 UCharDirection enum (from third_party/icu/include/unicode/uchar.h)
150  enum Direction {
151  U_LEFT_TO_RIGHT = 0,
152  U_RIGHT_TO_LEFT = 1,
153  U_EUROPEAN_NUMBER = 2,
154  U_EUROPEAN_NUMBER_SEPARATOR = 3,
155  U_EUROPEAN_NUMBER_TERMINATOR = 4,
156  U_ARABIC_NUMBER = 5,
157  U_COMMON_NUMBER_SEPARATOR = 6,
158  U_BLOCK_SEPARATOR = 7,
159  U_SEGMENT_SEPARATOR = 8,
160  U_WHITE_SPACE_NEUTRAL = 9,
161  U_OTHER_NEUTRAL = 10,
162  U_LEFT_TO_RIGHT_EMBEDDING = 11,
163  U_LEFT_TO_RIGHT_OVERRIDE = 12,
164  U_RIGHT_TO_LEFT_ARABIC = 13,
165  U_RIGHT_TO_LEFT_EMBEDDING = 14,
166  U_RIGHT_TO_LEFT_OVERRIDE = 15,
167  U_POP_DIRECTIONAL_FORMAT = 16,
168  U_DIR_NON_SPACING_MARK = 17,
169  U_BOUNDARY_NEUTRAL = 18,
170  U_CHAR_DIRECTION_COUNT
171  };
172 
173  // Create an empty UNICHARSET
174  UNICHARSET();
175 
176  ~UNICHARSET();
177 
178  // Return the UNICHAR_ID of a given unichar representation within the
179  // UNICHARSET.
180  UNICHAR_ID unichar_to_id(const char* const unichar_repr) const;
181 
182  // Return the UNICHAR_ID of a given unichar representation within the
183  // UNICHARSET. Only the first length characters from unichar_repr are used.
184  UNICHAR_ID unichar_to_id(const char* const unichar_repr, int length) const;
185 
186  // Return the minimum number of bytes that matches a legal UNICHAR_ID,
187  // while leaving the rest of the string encodable. Returns 0 if the
188  // beginning of the string is not encodable.
189  // WARNING: this function now encodes the whole string for precision.
190  // Use encode_string in preference to repeatedly calling step.
191  int step(const char* str) const;
192 
193  // Return whether the given UTF-8 string is encodable with this UNICHARSET.
194  // If not encodable, write the first byte offset which cannot be converted
195  // into the second (return) argument.
196  bool encodable_string(const char *str, int *first_bad_position) const;
197 
198  // Encodes the given UTF-8 string with this UNICHARSET.
199  // Any part of the string that cannot be encoded (because the utf8 can't
200  // be broken up into pieces that are in the unicharset) then:
201  // if give_up_on_failure, stops and returns a partial encoding,
202  // else continues and inserts an INVALID_UNICHAR_ID in the returned encoding.
203  // Returns true if the encoding succeeds completely, false if there is at
204  // least one failure.
205  // If lengths is not NULL, then it is filled with the corresponding
206  // byte length of each encoded UNICHAR_ID.
207  // If encoded_length is not NULL then on return it contains the length of
208  // str that was encoded. (if give_up_on_failure the location of the first
209  // failure, otherwise strlen(str).)
210  bool encode_string(const char* str, bool give_up_on_failure,
211  GenericVector<UNICHAR_ID>* encoding,
212  GenericVector<char>* lengths,
213  int* encoded_length) const;
214 
215  // Return the unichar representation corresponding to the given UNICHAR_ID
216  // within the UNICHARSET.
217  const char* id_to_unichar(UNICHAR_ID id) const;
218 
219  // Return the UTF8 representation corresponding to the given UNICHAR_ID after
220  // resolving any private encodings internal to Tesseract. This method is
221  // preferable to id_to_unichar for outputting text that will be visible to
222  // external applications.
223  const char* id_to_unichar_ext(UNICHAR_ID id) const;
224 
225  // Return a STRING that reformats the utf8 str into the str followed
226  // by its hex unicodes.
227  static STRING debug_utf8_str(const char* str);
228 
229  // Return a STRING containing debug information on the unichar, including
230  // the id_to_unichar, its hex unicodes and the properties.
231  STRING debug_str(UNICHAR_ID id) const;
232  STRING debug_str(const char * unichar_repr) const {
233  return debug_str(unichar_to_id(unichar_repr));
234  }
235 
236  // Add a unichar representation to the set.
237  void unichar_insert(const char* const unichar_repr);
238 
239  // Return true if the given unichar id exists within the set.
240  // Relies on the fact that unichar ids are contiguous in the unicharset.
241  bool contains_unichar_id(UNICHAR_ID unichar_id) const {
242  return unichar_id != INVALID_UNICHAR_ID && unichar_id < size_used &&
243  unichar_id >= 0;
244  }
245 
246  // Return true if the given unichar representation exists within the set.
247  bool contains_unichar(const char* const unichar_repr) const;
248  bool contains_unichar(const char* const unichar_repr, int length) const;
249 
250  // Return true if the given unichar representation corresponds to the given
251  // UNICHAR_ID within the set.
252  bool eq(UNICHAR_ID unichar_id, const char* const unichar_repr) const;
253 
254  // Delete CHAR_FRAGMENTs stored in properties of unichars array.
256  for (int i = 0; i < size_used; ++i) {
257  if (unichars[i].properties.fragment != NULL) {
258  delete unichars[i].properties.fragment;
259  unichars[i].properties.fragment = NULL;
260  }
261  }
262  }
263 
264  // Clear the UNICHARSET (all the previous data is lost).
265  void clear() {
266  if (script_table != NULL) {
267  for (int i = 0; i < script_table_size_used; ++i)
268  delete[] script_table[i];
269  delete[] script_table;
270  script_table = NULL;
271  script_table_size_used = 0;
272  }
273  if (unichars != NULL) {
274  delete_pointers_in_unichars();
275  delete[] unichars;
276  unichars = NULL;
277  }
278  script_table_size_reserved = 0;
279  size_reserved = 0;
280  size_used = 0;
281  ids.clear();
282  top_bottom_set_ = false;
283  script_has_upper_lower_ = false;
284  script_has_xheight_ = false;
285  null_sid_ = 0;
286  common_sid_ = 0;
287  latin_sid_ = 0;
288  cyrillic_sid_ = 0;
289  greek_sid_ = 0;
290  han_sid_ = 0;
291  hiragana_sid_ = 0;
292  katakana_sid_ = 0;
293  thai_sid_ = 0;
294  hangul_sid_ = 0;
295  default_sid_ = 0;
296  }
297 
298  // Return the size of the set (the number of different UNICHAR it holds).
299  int size() const {
300  return size_used;
301  }
302 
303  // Reserve enough memory space for the given number of UNICHARS
304  void reserve(int unichars_number);
305 
306  // Opens the file indicated by filename and saves unicharset to that file.
307  // Returns true if the operation is successful.
308  bool save_to_file(const char * const filename) const {
309  FILE* file = fopen(filename, "w+b");
310  if (file == NULL) return false;
311  bool result = save_to_file(file);
312  fclose(file);
313  return result;
314  }
315 
316  // Saves the content of the UNICHARSET to the given file.
317  // Returns true if the operation is successful.
318  bool save_to_file(FILE *file) const {
319  STRING str;
320  if (!save_to_string(&str)) return false;
321  if (fwrite(&str[0], str.length(), 1, file) != 1) return false;
322  return true;
323  }
324  bool save_to_file(tesseract::TFile *file) const {
325  STRING str;
326  if (!save_to_string(&str)) return false;
327  if (file->FWrite(&str[0], str.length(), 1) != 1) return false;
328  return true;
329  }
330 
331  // Saves the content of the UNICHARSET to the given STRING.
332  // Returns true if the operation is successful.
333  bool save_to_string(STRING *str) const;
334 
335  // Load a unicharset from a unicharset file that has been loaded into
336  // the given memory buffer.
337  // Returns true if the operation is successful.
338  bool load_from_inmemory_file(const char* const memory, int mem_size,
339  bool skip_fragments);
340  // Returns true if the operation is successful.
341  bool load_from_inmemory_file(const char* const memory, int mem_size) {
342  return load_from_inmemory_file(memory, mem_size, false);
343  }
344 
345  // Opens the file indicated by filename and loads the UNICHARSET
346  // from the given file. The previous data is lost.
347  // Returns true if the operation is successful.
348  bool load_from_file(const char* const filename, bool skip_fragments) {
349  FILE* file = fopen(filename, "rb");
350  if (file == NULL) return false;
351  bool result = load_from_file(file, skip_fragments);
352  fclose(file);
353  return result;
354  }
355  // returns true if the operation is successful.
356  bool load_from_file(const char* const filename) {
357  return load_from_file(filename, false);
358  }
359 
360  // Loads the UNICHARSET from the given file. The previous data is lost.
361  // Returns true if the operation is successful.
362  bool load_from_file(FILE *file, bool skip_fragments);
363  bool load_from_file(FILE *file) { return load_from_file(file, false); }
364  bool load_from_file(tesseract::TFile *file, bool skip_fragments);
365 
366 
367  // Sets up internal data after loading the file, based on the char
368  // properties. Called from load_from_file, but also needs to be run
369  // during set_unicharset_properties.
370  void post_load_setup();
371 
372  // Returns true if right_to_left scripts are significant in the unicharset,
373  // but without being so sensitive that "universal" unicharsets containing
374  // characters from many scripts, like orientation and script detection,
375  // look like they are right_to_left.
376  bool major_right_to_left() const;
377 
378  // Set a whitelist and/or blacklist of characters to recognize.
379  // An empty or NULL whitelist enables everything (minus any blacklist).
380  // An empty or NULL blacklist disables nothing.
381  // An empty or NULL unblacklist has no effect.
382  // The blacklist overrides the whitelist.
383  // The unblacklist overrides the blacklist.
384  // Each list is a string of utf8 character strings. Boundaries between
385  // unicharset units are worked out automatically, and characters not in
386  // the unicharset are silently ignored.
387  void set_black_and_whitelist(const char* blacklist, const char* whitelist,
388  const char* unblacklist);
389 
390  // Set the isalpha property of the given unichar to the given value.
391  void set_isalpha(UNICHAR_ID unichar_id, bool value) {
392  unichars[unichar_id].properties.isalpha = value;
393  }
394 
395  // Set the islower property of the given unichar to the given value.
396  void set_islower(UNICHAR_ID unichar_id, bool value) {
397  unichars[unichar_id].properties.islower = value;
398  }
399 
400  // Set the isupper property of the given unichar to the given value.
401  void set_isupper(UNICHAR_ID unichar_id, bool value) {
402  unichars[unichar_id].properties.isupper = value;
403  }
404 
405  // Set the isdigit property of the given unichar to the given value.
406  void set_isdigit(UNICHAR_ID unichar_id, bool value) {
407  unichars[unichar_id].properties.isdigit = value;
408  }
409 
410  // Set the ispunctuation property of the given unichar to the given value.
411  void set_ispunctuation(UNICHAR_ID unichar_id, bool value) {
412  unichars[unichar_id].properties.ispunctuation = value;
413  }
414 
415  // Set the isngram property of the given unichar to the given value.
416  void set_isngram(UNICHAR_ID unichar_id, bool value) {
417  unichars[unichar_id].properties.isngram = value;
418  }
419 
420  // Set the script name of the given unichar to the given value.
421  // Value is copied and thus can be a temporary;
422  void set_script(UNICHAR_ID unichar_id, const char* value) {
423  unichars[unichar_id].properties.script_id = add_script(value);
424  }
425 
426  // Set other_case unichar id in the properties for the given unichar id.
427  void set_other_case(UNICHAR_ID unichar_id, UNICHAR_ID other_case) {
428  unichars[unichar_id].properties.other_case = other_case;
429  }
430 
431  // Set the direction property of the given unichar to the given value.
433  unichars[unichar_id].properties.direction = value;
434  }
435 
436  // Set mirror unichar id in the properties for the given unichar id.
437  void set_mirror(UNICHAR_ID unichar_id, UNICHAR_ID mirror) {
438  unichars[unichar_id].properties.mirror = mirror;
439  }
440 
441  // Record normalized version of unichar with the given unichar_id.
442  void set_normed(UNICHAR_ID unichar_id, const char* normed) {
443  unichars[unichar_id].properties.normed = normed;
444  unichars[unichar_id].properties.normed_ids.truncate(0);
445  }
446  // Sets the normed_ids vector from the normed string. normed_ids is not
447  // stored in the file, and needs to be set when the UNICHARSET is loaded.
448  void set_normed_ids(UNICHAR_ID unichar_id);
449 
450  // Return the isalpha property of the given unichar.
451  bool get_isalpha(UNICHAR_ID unichar_id) const {
452  if (INVALID_UNICHAR_ID == unichar_id) return false;
453  ASSERT_HOST(contains_unichar_id(unichar_id));
454  return unichars[unichar_id].properties.isalpha;
455  }
456 
457  // Return the islower property of the given unichar.
458  bool get_islower(UNICHAR_ID unichar_id) const {
459  if (INVALID_UNICHAR_ID == unichar_id) return false;
460  ASSERT_HOST(contains_unichar_id(unichar_id));
461  return unichars[unichar_id].properties.islower;
462  }
463 
464  // Return the isupper property of the given unichar.
465  bool get_isupper(UNICHAR_ID unichar_id) const {
466  if (INVALID_UNICHAR_ID == unichar_id) return false;
467  ASSERT_HOST(contains_unichar_id(unichar_id));
468  return unichars[unichar_id].properties.isupper;
469  }
470 
471  // Return the isdigit property of the given unichar.
472  bool get_isdigit(UNICHAR_ID unichar_id) const {
473  if (INVALID_UNICHAR_ID == unichar_id) return false;
474  ASSERT_HOST(contains_unichar_id(unichar_id));
475  return unichars[unichar_id].properties.isdigit;
476  }
477 
478  // Return the ispunctuation property of the given unichar.
479  bool get_ispunctuation(UNICHAR_ID unichar_id) const {
480  if (INVALID_UNICHAR_ID == unichar_id) return false;
481  ASSERT_HOST(contains_unichar_id(unichar_id));
482  return unichars[unichar_id].properties.ispunctuation;
483  }
484 
485  // Return the isngram property of the given unichar.
486  bool get_isngram(UNICHAR_ID unichar_id) const {
487  if (INVALID_UNICHAR_ID == unichar_id) return false;
488  ASSERT_HOST(contains_unichar_id(unichar_id));
489  return unichars[unichar_id].properties.isngram;
490  }
491 
492  // Returns whether the unichar id represents a unicode value in the private
493  // use area.
494  bool get_isprivate(UNICHAR_ID unichar_id) const;
495 
496  // Returns true if the ids have useful min/max top/bottom values.
497  bool top_bottom_useful() const {
498  return top_bottom_set_;
499  }
500  // Sets all ranges to empty, so they can be expanded to set the values.
501  void set_ranges_empty();
502  // Sets all the properties for this unicharset given a src_unicharset with
503  // everything set. The unicharsets don't have to be the same, and graphemes
504  // are correctly accounted for.
506  PartialSetPropertiesFromOther(0, src);
507  }
508  // Sets properties from Other, starting only at the given index.
509  void PartialSetPropertiesFromOther(int start_index, const UNICHARSET& src);
510  // Expands the tops and bottoms and widths for this unicharset given a
511  // src_unicharset with ranges in it. The unicharsets don't have to be the
512  // same, and graphemes are correctly accounted for.
513  void ExpandRangesFromOther(const UNICHARSET& src);
514  // Makes this a copy of src. Clears this completely first, so the automattic
515  // ids will not be present in this if not in src.
516  void CopyFrom(const UNICHARSET& src);
517  // For each id in src, if it does not occur in this, add it, as in
518  // SetPropertiesFromOther, otherwise expand the ranges, as in
519  // ExpandRangesFromOther.
520  void AppendOtherUnicharset(const UNICHARSET& src);
521  // Returns true if the acceptable ranges of the tops of the characters do
522  // not overlap, making their x-height calculations distinct.
523  bool SizesDistinct(UNICHAR_ID id1, UNICHAR_ID id2) const;
524  // Returns the min and max bottom and top of the given unichar in
525  // baseline-normalized coordinates, ie, where the baseline is
526  // kBlnBaselineOffset and the meanline is kBlnBaselineOffset + kBlnXHeight
527  // (See normalis.h for the definitions).
528  void get_top_bottom(UNICHAR_ID unichar_id,
529  int* min_bottom, int* max_bottom,
530  int* min_top, int* max_top) const {
531  if (INVALID_UNICHAR_ID == unichar_id) {
532  *min_bottom = *min_top = 0;
533  *max_bottom = *max_top = 256; // kBlnCellHeight
534  return;
535  }
536  ASSERT_HOST(contains_unichar_id(unichar_id));
537  *min_bottom = unichars[unichar_id].properties.min_bottom;
538  *max_bottom = unichars[unichar_id].properties.max_bottom;
539  *min_top = unichars[unichar_id].properties.min_top;
540  *max_top = unichars[unichar_id].properties.max_top;
541  }
542  void set_top_bottom(UNICHAR_ID unichar_id,
543  int min_bottom, int max_bottom,
544  int min_top, int max_top) {
545  unichars[unichar_id].properties.min_bottom =
546  static_cast<uinT8>(ClipToRange(min_bottom, 0, MAX_UINT8));
547  unichars[unichar_id].properties.max_bottom =
548  static_cast<uinT8>(ClipToRange(max_bottom, 0, MAX_UINT8));
549  unichars[unichar_id].properties.min_top =
550  static_cast<uinT8>(ClipToRange(min_top, 0, MAX_UINT8));
551  unichars[unichar_id].properties.max_top =
552  static_cast<uinT8>(ClipToRange(max_top, 0, MAX_UINT8));
553  }
554  // Returns the width stats (as mean, sd) of the given unichar relative to the
555  // median advance of all characters in the character set.
556  void get_width_stats(UNICHAR_ID unichar_id,
557  float* width, float* width_sd) const {
558  if (INVALID_UNICHAR_ID == unichar_id) {
559  *width = 0.0f;
560  *width_sd = 0.0f;;
561  return;
562  }
563  ASSERT_HOST(contains_unichar_id(unichar_id));
564  *width = unichars[unichar_id].properties.width;
565  *width_sd = unichars[unichar_id].properties.width_sd;
566  }
567  void set_width_stats(UNICHAR_ID unichar_id, float width, float width_sd) {
568  unichars[unichar_id].properties.width = width;
569  unichars[unichar_id].properties.width_sd = width_sd;
570  }
571  // Returns the stats of the x-bearing (as mean, sd) of the given unichar
572  // relative to the median advance of all characters in the character set.
573  void get_bearing_stats(UNICHAR_ID unichar_id,
574  float* bearing, float* bearing_sd) const {
575  if (INVALID_UNICHAR_ID == unichar_id) {
576  *bearing = *bearing_sd = 0.0f;
577  return;
578  }
579  ASSERT_HOST(contains_unichar_id(unichar_id));
580  *bearing = unichars[unichar_id].properties.bearing;
581  *bearing_sd = unichars[unichar_id].properties.bearing_sd;
582  }
583  void set_bearing_stats(UNICHAR_ID unichar_id,
584  float bearing, float bearing_sd) {
585  unichars[unichar_id].properties.bearing = bearing;
586  unichars[unichar_id].properties.bearing_sd = bearing_sd;
587  }
588  // Returns the stats of the x-advance of the given unichar (as mean, sd)
589  // relative to the median advance of all characters in the character set.
590  void get_advance_stats(UNICHAR_ID unichar_id,
591  float* advance, float* advance_sd) const {
592  if (INVALID_UNICHAR_ID == unichar_id) {
593  *advance = *advance_sd = 0;
594  return;
595  }
596  ASSERT_HOST(contains_unichar_id(unichar_id));
597  *advance = unichars[unichar_id].properties.advance;
598  *advance_sd = unichars[unichar_id].properties.advance_sd;
599  }
600  void set_advance_stats(UNICHAR_ID unichar_id,
601  float advance, float advance_sd) {
602  unichars[unichar_id].properties.advance = advance;
603  unichars[unichar_id].properties.advance_sd = advance_sd;
604  }
605  // Returns true if the font metrics properties are empty.
606  bool PropertiesIncomplete(UNICHAR_ID unichar_id) const {
607  return unichars[unichar_id].properties.AnyRangeEmpty();
608  }
609 
610  // Returns true if the script of the given id is space delimited.
611  // Returns false for Han and Thai scripts.
612  bool IsSpaceDelimited(UNICHAR_ID unichar_id) const {
613  if (INVALID_UNICHAR_ID == unichar_id) return true;
614  int script_id = get_script(unichar_id);
615  return script_id != han_sid_ && script_id != thai_sid_ &&
616  script_id != hangul_sid_ && script_id != hiragana_sid_ &&
617  script_id != katakana_sid_;
618  }
619 
620  // Return the script name of the given unichar.
621  // The returned pointer will always be the same for the same script, it's
622  // managed by unicharset and thus MUST NOT be deleted
623  int get_script(UNICHAR_ID unichar_id) const {
624  if (INVALID_UNICHAR_ID == unichar_id) return null_sid_;
625  ASSERT_HOST(contains_unichar_id(unichar_id));
626  return unichars[unichar_id].properties.script_id;
627  }
628 
629  // Return the character properties, eg. alpha/upper/lower/digit/punct,
630  // as a bit field of unsigned int.
631  unsigned int get_properties(UNICHAR_ID unichar_id) const;
632 
633  // Return the character property as a single char. If a character has
634  // multiple attributes, the main property is defined by the following order:
635  // upper_case : 'A'
636  // lower_case : 'a'
637  // alpha : 'x'
638  // digit : '0'
639  // punctuation: 'p'
640  char get_chartype(UNICHAR_ID unichar_id) const;
641 
642  // Get other_case unichar id in the properties for the given unichar id.
644  if (INVALID_UNICHAR_ID == unichar_id) return INVALID_UNICHAR_ID;
645  ASSERT_HOST(contains_unichar_id(unichar_id));
646  return unichars[unichar_id].properties.other_case;
647  }
648 
649  // Returns the direction property of the given unichar.
650  Direction get_direction(UNICHAR_ID unichar_id) const {
651  if (INVALID_UNICHAR_ID == unichar_id) return UNICHARSET::U_OTHER_NEUTRAL;
652  ASSERT_HOST(contains_unichar_id(unichar_id));
653  return unichars[unichar_id].properties.direction;
654  }
655 
656  // Get mirror unichar id in the properties for the given unichar id.
657  UNICHAR_ID get_mirror(UNICHAR_ID unichar_id) const {
658  if (INVALID_UNICHAR_ID == unichar_id) return INVALID_UNICHAR_ID;
659  ASSERT_HOST(contains_unichar_id(unichar_id));
660  return unichars[unichar_id].properties.mirror;
661  }
662 
663  // Returns UNICHAR_ID of the corresponding lower-case unichar.
664  UNICHAR_ID to_lower(UNICHAR_ID unichar_id) const {
665  if (INVALID_UNICHAR_ID == unichar_id) return INVALID_UNICHAR_ID;
666  ASSERT_HOST(contains_unichar_id(unichar_id));
667  if (unichars[unichar_id].properties.islower) return unichar_id;
668  return unichars[unichar_id].properties.other_case;
669  }
670 
671  // Returns UNICHAR_ID of the corresponding upper-case unichar.
672  UNICHAR_ID to_upper(UNICHAR_ID unichar_id) const {
673  if (INVALID_UNICHAR_ID == unichar_id) return INVALID_UNICHAR_ID;
674  ASSERT_HOST(contains_unichar_id(unichar_id));
675  if (unichars[unichar_id].properties.isupper) return unichar_id;
676  return unichars[unichar_id].properties.other_case;
677  }
678 
679  // Returns true if this UNICHARSET has the special codes in
680  // SpecialUnicharCodes available. If false then there are normal unichars
681  // at these codes and they should not be used.
682  bool has_special_codes() const {
683  return get_fragment(UNICHAR_BROKEN) != NULL &&
684  strcmp(id_to_unichar(UNICHAR_BROKEN),
685  kSpecialUnicharCodes[UNICHAR_BROKEN]) == 0;
686  }
687 
688  // Returns true if there are any repeated unicodes in the normalized
689  // text of any unichar-id in the unicharset.
690  bool AnyRepeatedUnicodes() const;
691 
692  // Return a pointer to the CHAR_FRAGMENT class if the given
693  // unichar id represents a character fragment.
694  const CHAR_FRAGMENT *get_fragment(UNICHAR_ID unichar_id) const {
695  if (INVALID_UNICHAR_ID == unichar_id) return NULL;
696  ASSERT_HOST(contains_unichar_id(unichar_id));
697  return unichars[unichar_id].properties.fragment;
698  }
699 
700  // Return the isalpha property of the given unichar representation.
701  bool get_isalpha(const char* const unichar_repr) const {
702  return get_isalpha(unichar_to_id(unichar_repr));
703  }
704 
705  // Return the islower property of the given unichar representation.
706  bool get_islower(const char* const unichar_repr) const {
707  return get_islower(unichar_to_id(unichar_repr));
708  }
709 
710  // Return the isupper property of the given unichar representation.
711  bool get_isupper(const char* const unichar_repr) const {
712  return get_isupper(unichar_to_id(unichar_repr));
713  }
714 
715  // Return the isdigit property of the given unichar representation.
716  bool get_isdigit(const char* const unichar_repr) const {
717  return get_isdigit(unichar_to_id(unichar_repr));
718  }
719 
720  // Return the ispunctuation property of the given unichar representation.
721  bool get_ispunctuation(const char* const unichar_repr) const {
722  return get_ispunctuation(unichar_to_id(unichar_repr));
723  }
724 
725  // Return the character properties, eg. alpha/upper/lower/digit/punct,
726  // of the given unichar representation
727  unsigned int get_properties(const char* const unichar_repr) const {
728  return get_properties(unichar_to_id(unichar_repr));
729  }
730 
731  char get_chartype(const char* const unichar_repr) const {
732  return get_chartype(unichar_to_id(unichar_repr));
733  }
734 
735  // Return the script name of the given unichar representation.
736  // The returned pointer will always be the same for the same script, it's
737  // managed by unicharset and thus MUST NOT be deleted
738  int get_script(const char* const unichar_repr) const {
739  return get_script(unichar_to_id(unichar_repr));
740  }
741 
742  // Return a pointer to the CHAR_FRAGMENT class struct if the given
743  // unichar representation represents a character fragment.
744  const CHAR_FRAGMENT *get_fragment(const char* const unichar_repr) const {
745  if (unichar_repr == NULL || unichar_repr[0] == '\0' ||
746  !ids.contains(unichar_repr)) {
747  return NULL;
748  }
749  return get_fragment(unichar_to_id(unichar_repr));
750  }
751 
752  // Return the isalpha property of the given unichar representation.
753  // Only the first length characters from unichar_repr are used.
754  bool get_isalpha(const char* const unichar_repr,
755  int length) const {
756  return get_isalpha(unichar_to_id(unichar_repr, length));
757  }
758 
759  // Return the islower property of the given unichar representation.
760  // Only the first length characters from unichar_repr are used.
761  bool get_islower(const char* const unichar_repr,
762  int length) const {
763  return get_islower(unichar_to_id(unichar_repr, length));
764  }
765 
766  // Return the isupper property of the given unichar representation.
767  // Only the first length characters from unichar_repr are used.
768  bool get_isupper(const char* const unichar_repr,
769  int length) const {
770  return get_isupper(unichar_to_id(unichar_repr, length));
771  }
772 
773  // Return the isdigit property of the given unichar representation.
774  // Only the first length characters from unichar_repr are used.
775  bool get_isdigit(const char* const unichar_repr,
776  int length) const {
777  return get_isdigit(unichar_to_id(unichar_repr, length));
778  }
779 
780  // Return the ispunctuation property of the given unichar representation.
781  // Only the first length characters from unichar_repr are used.
782  bool get_ispunctuation(const char* const unichar_repr,
783  int length) const {
784  return get_ispunctuation(unichar_to_id(unichar_repr, length));
785  }
786 
787  // Returns normalized version of unichar with the given unichar_id.
788  const char *get_normed_unichar(UNICHAR_ID unichar_id) const {
789  if (unichar_id == UNICHAR_SPACE) return " ";
790  return unichars[unichar_id].properties.normed.string();
791  }
792  // Returns a vector of UNICHAR_IDs that represent the ids of the normalized
793  // version of the given id. There may be more than one UNICHAR_ID in the
794  // vector if unichar_id represents a ligature.
796  return unichars[unichar_id].properties.normed_ids;
797  }
798 
799  // Return the script name of the given unichar representation.
800  // Only the first length characters from unichar_repr are used.
801  // The returned pointer will always be the same for the same script, it's
802  // managed by unicharset and thus MUST NOT be deleted
803  int get_script(const char* const unichar_repr,
804  int length) const {
805  return get_script(unichar_to_id(unichar_repr, length));
806  }
807 
808  // Return the (current) number of scripts in the script table
809  int get_script_table_size() const {
810  return script_table_size_used;
811  }
812 
813  // Return the script string from its id
814  const char* get_script_from_script_id(int id) const {
815  if (id >= script_table_size_used || id < 0)
816  return null_script;
817  return script_table[id];
818  }
819 
820  // Returns the id from the name of the script, or 0 if script is not found.
821  // Note that this is an expensive operation since it involves iteratively
822  // comparing strings in the script table. To avoid dependency on STL, we
823  // won't use a hash. Instead, the calling function can use this to lookup
824  // and save the ID for relevant scripts for fast comparisons later.
825  int get_script_id_from_name(const char* script_name) const;
826 
827  // Return true if the given script is the null script
828  bool is_null_script(const char* script) const {
829  return script == null_script;
830  }
831 
832  // Uniquify the given script. For two scripts a and b, if strcmp(a, b) == 0,
833  // then the returned pointer will be the same.
834  // The script parameter is copied and thus can be a temporary.
835  int add_script(const char* script);
836 
837  // Return the enabled property of the given unichar.
838  bool get_enabled(UNICHAR_ID unichar_id) const {
839  return unichars[unichar_id].properties.enabled;
840  }
841 
842 
843  int null_sid() const { return null_sid_; }
844  int common_sid() const { return common_sid_; }
845  int latin_sid() const { return latin_sid_; }
846  int cyrillic_sid() const { return cyrillic_sid_; }
847  int greek_sid() const { return greek_sid_; }
848  int han_sid() const { return han_sid_; }
849  int hiragana_sid() const { return hiragana_sid_; }
850  int katakana_sid() const { return katakana_sid_; }
851  int thai_sid() const { return thai_sid_; }
852  int hangul_sid() const { return hangul_sid_; }
853  int default_sid() const { return default_sid_; }
854 
855  // Returns true if the unicharset has the concept of upper/lower case.
856  bool script_has_upper_lower() const {
857  return script_has_upper_lower_;
858  }
859  // Returns true if the unicharset has the concept of x-height.
860  // script_has_xheight can be true even if script_has_upper_lower is not,
861  // when the script has a sufficiently predominant top line with ascenders,
862  // such as Devanagari and Thai.
863  bool script_has_xheight() const {
864  return script_has_xheight_;
865  }
866 
867  private:
868 
869  struct UNICHAR_PROPERTIES {
870  UNICHAR_PROPERTIES();
871  // Initializes all properties to sensible default values.
872  void Init();
873  // Sets all ranges wide open. Initialization default in case there are
874  // no useful values available.
875  void SetRangesOpen();
876  // Sets all ranges to empty. Used before expanding with font-based data.
877  void SetRangesEmpty();
878  // Returns true if any of the top/bottom/width/bearing/advance ranges/stats
879  // is emtpy.
880  bool AnyRangeEmpty() const;
881  // Expands the ranges with the ranges from the src properties.
882  void ExpandRangesFrom(const UNICHAR_PROPERTIES& src);
883  // Copies the properties from src into this.
884  void CopyFrom(const UNICHAR_PROPERTIES& src);
885 
886  bool isalpha;
887  bool islower;
888  bool isupper;
889  bool isdigit;
890  bool ispunctuation;
891  bool isngram;
892  bool enabled;
893  // Possible limits of the top and bottom of the bounding box in
894  // baseline-normalized coordinates, ie, where the baseline is
895  // kBlnBaselineOffset and the meanline is kBlnBaselineOffset + kBlnXHeight
896  // (See normalis.h for the definitions).
897  uinT8 min_bottom;
898  uinT8 max_bottom;
899  uinT8 min_top;
900  uinT8 max_top;
901  // Statstics of the widths of bounding box, relative to the median advance.
902  float width;
903  float width_sd;
904  // Stats of the x-bearing and advance, also relative to the median advance.
905  float bearing;
906  float bearing_sd;
907  float advance;
908  float advance_sd;
909  int script_id;
910  UNICHAR_ID other_case; // id of the corresponding upper/lower case unichar
911  Direction direction; // direction of this unichar
912  // Mirror property is useful for reverse DAWG lookup for words in
913  // right-to-left languages (e.g. "(word)" would be in
914  // '[open paren]' 'w' 'o' 'r' 'd' '[close paren]' in a UTF8 string.
915  // However, what we want in our DAWG is
916  // '[open paren]', 'd', 'r', 'o', 'w', '[close paren]' not
917  // '[close paren]', 'd', 'r', 'o', 'w', '[open paren]'.
918  UNICHAR_ID mirror;
919  // A string of unichar_ids that represent the corresponding normed string.
920  // For awkward characters like em-dash, this gives hyphen.
921  // For ligatures, this gives the string of normal unichars.
922  GenericVector<UNICHAR_ID> normed_ids;
923  STRING normed; // normalized version of this unichar
924  // Contains meta information about the fragment if a unichar represents
925  // a fragment of a character, otherwise should be set to NULL.
926  // It is assumed that character fragments are added to the unicharset
927  // after the corresponding 'base' characters.
928  CHAR_FRAGMENT *fragment;
929  };
930 
931  struct UNICHAR_SLOT {
932  char representation[UNICHAR_LEN + 1];
933  UNICHAR_PROPERTIES properties;
934  };
935 
936  // Internal recursive version of encode_string above.
937  // str is the start of the whole string.
938  // str_index is the current position in str.
939  // str_length is the length of str.
940  // encoding is a working encoding of str.
941  // lengths is a working set of lengths of each element of encoding.
942  // best_total_length is the longest length of str that has been successfully
943  // encoded so far.
944  // On return:
945  // best_encoding contains the encoding that used the longest part of str.
946  // best_lengths (may be null) contains the lengths of best_encoding.
947  void encode_string(const char* str, int str_index, int str_length,
948  GenericVector<UNICHAR_ID>* encoding,
949  GenericVector<char>* lengths,
950  int* best_total_length,
951  GenericVector<UNICHAR_ID>* best_encoding,
952  GenericVector<char>* best_lengths) const;
953 
954  // Gets the properties for a grapheme string, combining properties for
955  // multiple characters in a meaningful way where possible.
956  // Returns false if no valid match was found in the unicharset.
957  // NOTE that script_id, mirror, and other_case refer to this unicharset on
958  // return and will need redirecting if the target unicharset is different.
959  bool GetStrProperties(const char* utf8_str,
960  UNICHAR_PROPERTIES* props) const;
961 
962  // Load ourselves from a "file" where our only interface to the file is
963  // an implementation of fgets(). This is the parsing primitive accessed by
964  // the public routines load_from_file() and load_from_inmemory_file().
965  bool load_via_fgets(TessResultCallback2<char *, char *, int> *fgets_cb,
966  bool skip_fragments);
967 
968  UNICHAR_SLOT* unichars;
969  UNICHARMAP ids;
970  int size_used;
971  int size_reserved;
972  char** script_table;
973  int script_table_size_used;
974  int script_table_size_reserved;
975  const char* null_script;
976  // True if the unichars have their tops/bottoms set.
977  bool top_bottom_set_;
978  // True if the unicharset has significant upper/lower case chars.
979  bool script_has_upper_lower_;
980  // True if the unicharset has a significant mean-line with significant
981  // ascenders above that.
982  bool script_has_xheight_;
983 
984  // A few convenient script name-to-id mapping without using hash.
985  // These are initialized when unicharset file is loaded. Anything
986  // missing from this list can be looked up using get_script_id_from_name.
987  int null_sid_;
988  int common_sid_;
989  int latin_sid_;
990  int cyrillic_sid_;
991  int greek_sid_;
992  int han_sid_;
993  int hiragana_sid_;
994  int katakana_sid_;
995  int thai_sid_;
996  int hangul_sid_;
997  // The most frequently occurring script in the charset.
998  int default_sid_;
999 };
1000 
1001 #endif // TESSERACT_CCUTIL_UNICHARSET_H_
int common_sid() const
Definition: unicharset.h:844
STRING to_string() const
Definition: unicharset.h:73
void set_isupper(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:401
int get_script(const char *const unichar_repr) const
Definition: unicharset.h:738
void set_all(const char *unichar, int pos, int total, bool natural)
Definition: unicharset.h:52
bool equals(const CHAR_FRAGMENT *other) const
Definition: unicharset.h:84
int latin_sid() const
Definition: unicharset.h:845
bool is_ending() const
Definition: unicharset.h:102
int hiragana_sid() const
Definition: unicharset.h:849
const GenericVector< UNICHAR_ID > & normed_ids(UNICHAR_ID unichar_id) const
Definition: unicharset.h:795
void set_total(int t)
Definition: unicharset.h:63
void set_width_stats(UNICHAR_ID unichar_id, float width, float width_sd)
Definition: unicharset.h:567
int UNICHAR_ID
Definition: unichar.h:33
bool is_beginning() const
Definition: unicharset.h:99
bool get_ispunctuation(UNICHAR_ID unichar_id) const
Definition: unicharset.h:479
const CHAR_FRAGMENT * get_fragment(UNICHAR_ID unichar_id) const
Definition: unicharset.h:694
bool is_null_script(const char *script) const
Definition: unicharset.h:828
void delete_pointers_in_unichars()
Definition: unicharset.h:255
#define TESS_API
Definition: platform.h:81
bool load_from_inmemory_file(const char *const memory, int mem_size)
Definition: unicharset.h:341
static const int kMaxChunks
Definition: unicharset.h:49
int get_pos() const
Definition: unicharset.h:65
const char * get_unichar() const
Definition: unicharset.h:64
#define MAX_UINT8
Definition: host.h:63
bool get_ispunctuation(const char *const unichar_repr) const
Definition: unicharset.h:721
bool load_from_file(const char *const filename)
Definition: unicharset.h:356
bool equals(const char *other_unichar, int other_pos, int other_total) const
Definition: unicharset.h:79
const char * get_script_from_script_id(int id) const
Definition: unicharset.h:814
void SetPropertiesFromOther(const UNICHARSET &src)
Definition: unicharset.h:505
bool contains_unichar_id(UNICHAR_ID unichar_id) const
Definition: unicharset.h:241
int direction(EDGEPT *point)
Definition: vecfuncs.cpp:43
STRING debug_str(const char *unichar_repr) const
Definition: unicharset.h:232
inT32 length() const
Definition: strngs.cpp:193
unsigned int get_properties(const char *const unichar_repr) const
Definition: unicharset.h:727
void set_normed(UNICHAR_ID unichar_id, const char *normed)
Definition: unicharset.h:442
void set_natural(bool value)
Definition: unicharset.h:108
int get_script(const char *const unichar_repr, int length) const
Definition: unicharset.h:803
int16_t inT16
Definition: host.h:36
int han_sid() const
Definition: unicharset.h:848
#define UNICHAR_LEN
Definition: unichar.h:30
Direction get_direction(UNICHAR_ID unichar_id) const
Definition: unicharset.h:650
#define ASSERT_HOST(x)
Definition: errcode.h:84
void clear()
Definition: unicharset.h:265
int thai_sid() const
Definition: unicharset.h:851
int get_script(UNICHAR_ID unichar_id) const
Definition: unicharset.h:623
bool get_enabled(UNICHAR_ID unichar_id) const
Definition: unicharset.h:838
void get_width_stats(UNICHAR_ID unichar_id, float *width, float *width_sd) const
Definition: unicharset.h:556
void set_advance_stats(UNICHAR_ID unichar_id, float advance, float advance_sd)
Definition: unicharset.h:600
bool get_islower(const char *const unichar_repr) const
Definition: unicharset.h:706
bool get_isupper(const char *const unichar_repr) const
Definition: unicharset.h:711
void set_ispunctuation(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:411
bool get_isalpha(UNICHAR_ID unichar_id) const
Definition: unicharset.h:451
void set_direction(UNICHAR_ID unichar_id, UNICHARSET::Direction value)
Definition: unicharset.h:432
bool get_isalpha(const char *const unichar_repr) const
Definition: unicharset.h:701
bool load_from_file(FILE *file)
Definition: unicharset.h:363
void get_advance_stats(UNICHAR_ID unichar_id, float *advance, float *advance_sd) const
Definition: unicharset.h:590
bool get_isdigit(UNICHAR_ID unichar_id) const
Definition: unicharset.h:472
UNICHAR_ID get_other_case(UNICHAR_ID unichar_id) const
Definition: unicharset.h:643
int get_script_table_size() const
Definition: unicharset.h:809
void get_top_bottom(UNICHAR_ID unichar_id, int *min_bottom, int *max_bottom, int *min_top, int *max_top) const
Definition: unicharset.h:528
UNICHAR_ID to_lower(UNICHAR_ID unichar_id) const
Definition: unicharset.h:664
bool PropertiesIncomplete(UNICHAR_ID unichar_id) const
Definition: unicharset.h:606
Definition: strngs.h:45
bool has_special_codes() const
Definition: unicharset.h:682
UNICHAR_ID get_mirror(UNICHAR_ID unichar_id) const
Definition: unicharset.h:657
bool IsSpaceDelimited(UNICHAR_ID unichar_id) const
Definition: unicharset.h:612
T ClipToRange(const T &x, const T &lower_bound, const T &upper_bound)
Definition: helpers.h:122
int get_total() const
Definition: unicharset.h:66
bool get_islower(const char *const unichar_repr, int length) const
Definition: unicharset.h:761
bool save_to_file(tesseract::TFile *file) const
Definition: unicharset.h:324
bool get_isdigit(const char *const unichar_repr, int length) const
Definition: unicharset.h:775
static CHAR_FRAGMENT * parse_from_string(const char *str)
int cyrillic_sid() const
Definition: unicharset.h:846
bool is_continuation_of(const CHAR_FRAGMENT *fragment) const
Definition: unicharset.h:92
int FWrite(const void *buffer, int size, int count)
Definition: serialis.cpp:148
void set_isngram(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:416
int hangul_sid() const
Definition: unicharset.h:852
static const int kMaxLen
Definition: unicharset.h:47
void get_bearing_stats(UNICHAR_ID unichar_id, float *bearing, float *bearing_sd) const
Definition: unicharset.h:573
bool load_from_file(const char *const filename, bool skip_fragments)
Definition: unicharset.h:348
int katakana_sid() const
Definition: unicharset.h:850
void set_script(UNICHAR_ID unichar_id, const char *value)
Definition: unicharset.h:422
bool save_to_file(FILE *file) const
Definition: unicharset.h:318
bool get_isupper(const char *const unichar_repr, int length) const
Definition: unicharset.h:768
int greek_sid() const
Definition: unicharset.h:847
bool save_to_file(const char *const filename) const
Definition: unicharset.h:308
const char * filename
Definition: ioapi.h:38
bool get_isdigit(const char *const unichar_repr) const
Definition: unicharset.h:716
static const int kMinLen
Definition: unicharset.h:45
UNICHAR_ID to_upper(UNICHAR_ID unichar_id) const
Definition: unicharset.h:672
uint8_t uinT8
Definition: host.h:35
void set_other_case(UNICHAR_ID unichar_id, UNICHAR_ID other_case)
Definition: unicharset.h:427
int size() const
Definition: unicharset.h:299
char get_chartype(const char *const unichar_repr) const
Definition: unicharset.h:731
int null_sid() const
Definition: unicharset.h:843
void set_pos(int p)
Definition: unicharset.h:62
void set_isalpha(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:391
bool get_isupper(UNICHAR_ID unichar_id) const
Definition: unicharset.h:465
bool get_islower(UNICHAR_ID unichar_id) const
Definition: unicharset.h:458
void set_unichar(const char *uch)
Definition: unicharset.h:58
int default_sid() const
Definition: unicharset.h:853
void set_mirror(UNICHAR_ID unichar_id, UNICHAR_ID mirror)
Definition: unicharset.h:437
bool top_bottom_useful() const
Definition: unicharset.h:497
bool script_has_upper_lower() const
Definition: unicharset.h:856
bool get_isalpha(const char *const unichar_repr, int length) const
Definition: unicharset.h:754
void set_top_bottom(UNICHAR_ID unichar_id, int min_bottom, int max_bottom, int min_top, int max_top)
Definition: unicharset.h:542
SpecialUnicharCodes
Definition: unicharset.h:34
const char * get_normed_unichar(UNICHAR_ID unichar_id) const
Definition: unicharset.h:788
bool script_has_xheight() const
Definition: unicharset.h:863
bool is_natural() const
Definition: unicharset.h:107
bool get_isngram(UNICHAR_ID unichar_id) const
Definition: unicharset.h:486
void set_isdigit(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:406
const CHAR_FRAGMENT * get_fragment(const char *const unichar_repr) const
Definition: unicharset.h:744
bool get_ispunctuation(const char *const unichar_repr, int length) const
Definition: unicharset.h:782
void set_bearing_stats(UNICHAR_ID unichar_id, float bearing, float bearing_sd)
Definition: unicharset.h:583
void set_islower(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:396