/******************************************************************
* File: fixspace.cpp (Formerly fixspace.c)
* Description: Implements a pass over the page res, exploring the alternative
* spacing possibilities, trying to use context to improve the
word spacing
* Author: Phil Cheatle
* Created: Thu Oct 21 11:38:43 BST 1993
*
* (C) Copyright 1993, Hewlett-Packard Ltd.
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
** http://www.apache.org/licenses/LICENSE-2.0
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
*
**********************************************************************/
#include "mfcpch.h"
#include <ctype.h>
#include "reject.h"
#include "statistc.h"
#include "genblob.h"
#include "control.h"
#include "fixspace.h"
#include "tessvars.h"
#include "tessbox.h"
#include "secname.h"
#include "globals.h"
#include "tesseractclass.h"
#define EXTERN
EXTERN BOOL_VAR (fixsp_check_for_fp_noise_space, TRUE,
"Try turning noise to space in fixed pitch");
EXTERN BOOL_VAR (fixsp_fp_eval, TRUE, "Use alternate evaluation for fp");
EXTERN BOOL_VAR (fixsp_noise_score_fixing, TRUE, "More sophisticated?");
EXTERN INT_VAR (fixsp_non_noise_limit, 1,
"How many non-noise blbs either side?");
EXTERN double_VAR (fixsp_small_outlines_size, 0.28, "Small if lt xht x this");
EXTERN BOOL_VAR (fixsp_ignore_punct, TRUE, "In uniform spacing calc");
EXTERN BOOL_VAR (fixsp_numeric_fix, TRUE, "Try to deal with numeric punct");
EXTERN BOOL_VAR (fixsp_prefer_joined_1s, TRUE, "Arbitrary boost");
EXTERN BOOL_VAR (tessedit_test_uniform_wd_spacing, FALSE,
"Limit context word spacing");
EXTERN BOOL_VAR (tessedit_prefer_joined_punct, FALSE,
"Reward punctation joins");
EXTERN INT_VAR (fixsp_done_mode, 1, "What constitues done for spacing");
EXTERN INT_VAR (debug_fix_space_level, 0, "Contextual fixspace debug");
EXTERN STRING_VAR (numeric_punctuation, ".,",
"Punct. chs expected WITHIN numbers");
#define PERFECT_WERDS 999
#define MAXSPACING 128 /*max expected spacing in pix */
/*************************************************************************
* fix_fuzzy_spaces()
* Walk over the page finding sequences of words joined by fuzzy spaces. Extract
* them as a sublist, process the sublist to find the optimal arrangement of
* spaces then replace the sublist in the ROW_RES.
*************************************************************************/
namespace tesseract {
void Tesseract::fix_fuzzy_spaces( //find fuzzy words
//progress monitor
volatile ETEXT_DESC *monitor,
//count of words in doc
inT32 word_count,
PAGE_RES *page_res) {
BLOCK_RES_IT block_res_it; //iterators
ROW_RES_IT row_res_it;
WERD_RES_IT word_res_it_from;
WERD_RES_IT word_res_it_to;
WERD_RES *word_res;
WERD_RES_LIST fuzzy_space_words;
inT16 new_length;
BOOL8 prevent_null_wd_fixsp; //DONT process blobless wds
inT32 word_index; //current word
block_res_it.set_to_list (&page_res->block_res_list);
word_index = 0;
for (block_res_it.mark_cycle_pt ();
!block_res_it.cycled_list (); block_res_it.forward ()) {
row_res_it.set_to_list (&block_res_it.data ()->row_res_list);
for (row_res_it.mark_cycle_pt ();
!row_res_it.cycled_list (); row_res_it.forward ()) {
word_res_it_from.set_to_list (&row_res_it.data ()->word_res_list);
while (!word_res_it_from.at_last ()) {
word_res = word_res_it_from.data ();
while (!word_res_it_from.at_last () &&
!(word_res->combination ||
word_res_it_from.data_relative (1)->
word->flag (W_FUZZY_NON) ||
word_res_it_from.data_relative (1)->
word->flag (W_FUZZY_SP))) {
fix_sp_fp_word(word_res_it_from, row_res_it.data()->row,
block_res_it.data()->block);
word_res = word_res_it_from.forward ();
word_index++;
if (monitor != NULL) {
monitor->ocr_alive = TRUE;
monitor->progress = 90 + 5 * word_index / word_count;
}
}
if (!word_res_it_from.at_last ()) {
word_res_it_to = word_res_it_from;
prevent_null_wd_fixsp =
word_res->word->gblob_list ()->empty ();
if (check_debug_pt (word_res, 60))
debug_fix_space_level.set_value (10);
word_res_it_to.forward ();
word_index++;
if (monitor != NULL) {
monitor->ocr_alive = TRUE;
monitor->progress = 90 + 5 * word_index / word_count;
}
while (!word_res_it_to.at_last () &&
(word_res_it_to.data_relative (1)->
word->flag (W_FUZZY_NON) ||
word_res_it_to.data_relative (1)->
word->flag (W_FUZZY_SP))) {
if (check_debug_pt (word_res, 60))
debug_fix_space_level.set_value (10);
if (word_res->word->gblob_list ()->empty ())
prevent_null_wd_fixsp = TRUE;
word_res = word_res_it_to.forward ();
}
if (check_debug_pt (word_res, 60))
debug_fix_space_level.set_value (10);
if (word_res->word->gblob_list ()->empty ())
prevent_null_wd_fixsp = TRUE;
if (prevent_null_wd_fixsp) {
word_res_it_from = word_res_it_to;
} else {
fuzzy_space_words.assign_to_sublist (&word_res_it_from,
&word_res_it_to);
fix_fuzzy_space_list (fuzzy_space_words,
row_res_it.data()->row,
block_res_it.data()->block);
new_length = fuzzy_space_words.length ();
word_res_it_from.add_list_before (&fuzzy_space_words);
for (;
(!word_res_it_from.at_last () &&
(new_length > 0)); new_length--) {
word_res_it_from.forward ();
}
}
if (test_pt)
debug_fix_space_level.set_value (0);
}
fix_sp_fp_word(word_res_it_from, row_res_it.data ()->row,
block_res_it.data()->block);
//Last word in row
}
}
}
}
void Tesseract::fix_fuzzy_space_list( //space explorer
WERD_RES_LIST &best_perm,
ROW *row,
BLOCK* block) {
inT16 best_score;
WERD_RES_LIST current_perm;
inT16 current_score;
BOOL8 improved = FALSE;
best_score = eval_word_spacing(best_perm); // default score
dump_words (best_perm, best_score, 1, improved);
if (best_score != PERFECT_WERDS)
initialise_search(best_perm, current_perm);
while ((best_score != PERFECT_WERDS) && !current_perm.empty ()) {
match_current_words(current_perm, row, block);
current_score = eval_word_spacing (current_perm);
dump_words (current_perm, current_score, 2, improved);
if (current_score > best_score) {
best_perm.clear ();
best_perm.deep_copy(¤t_perm, &WERD_RES::deep_copy);
best_score = current_score;
improved = TRUE;
}
if (current_score < PERFECT_WERDS)
transform_to_next_perm(current_perm);
}
dump_words (best_perm, best_score, 3, improved);
}
} // namespace tesseract
void initialise_search(WERD_RES_LIST &src_list, WERD_RES_LIST &new_list) {
WERD_RES_IT src_it(&src_list);
WERD_RES_IT new_it(&new_list);
WERD_RES *src_wd;
WERD_RES *new_wd;
for (src_it.mark_cycle_pt (); !src_it.cycled_list (); src_it.forward ()) {
src_wd = src_it.data ();
if (!src_wd->combination) {
new_wd = new WERD_RES (*src_wd);
new_wd->combination = FALSE;
new_wd->part_of_combo = FALSE;
new_it.add_after_then_move (new_wd);
}
}
}
namespace tesseract {
void Tesseract::match_current_words(WERD_RES_LIST &words, ROW *row,
BLOCK* block) {
WERD_RES_IT word_it(&words);
WERD_RES *word;
for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
word = word_it.data ();
if ((!word->part_of_combo) && (word->outword == NULL))
classify_word_pass2(word, block, row);
}
}
/*************************************************************************
* eval_word_spacing()
* The basic measure is the number of characters in contextually confirmed
* words. (I.e the word is done)
* If all words are contextually confirmed the evaluation is deemed perfect.
*
* Some fiddles are done to handle "1"s as these are VERY frequent causes of
* fuzzy spaces. The problem with the basic measure is that "561 63" would score
* the same as "56163", though given our knowledge that the space is fuzzy, and
* that there is a "1" next to the fuzzy space, we need to ensure that "56163"
* is prefered.
*
* The solution is to NOT COUNT the score of any word which has a digit at one
* end and a "1Il" as the character the other side of the space.
*
* Conversly, any character next to a "1" within a word is counted as a positive
* score. Thus "561 63" would score 4 (3 chars in a numeric word plus 1 side of
* the "1" joined). "56163" would score 7 - all chars in a numeric word + 2
* sides of a "1" joined.
*
* The joined 1 rule is applied to any word REGARDLESS of contextual
* confirmation. Thus "PS7a71 3/7a" scores 1 (neither word is contexutally
* confirmed. The only score is from the joined 1. "PS7a713/7a" scores 2.
*
*************************************************************************/
inT16 Tesseract::eval_word_spacing(WERD_RES_LIST &word_res_list) {
WERD_RES_IT word_res_it(&word_res_list);
inT16 total_score = 0;
inT16 word_count = 0;
inT16 done_word_count = 0;
inT16 word_len;
inT16 i;
inT16 offset;
WERD_RES *word; //current word
inT16 prev_word_score = 0;
BOOL8 prev_word_done = FALSE;
BOOL8 prev_char_1 = FALSE; //prev ch a "1/I/l"?
BOOL8 prev_char_digit = FALSE; //prev ch 2..9 or 0
BOOL8 current_char_1 = FALSE;
BOOL8 current_word_ok_so_far;
STRING punct_chars = "!\"`',.:;";
BOOL8 prev_char_punct = FALSE;
BOOL8 current_char_punct = FALSE;
BOOL8 word_done = FALSE;
do {
word = word_res_it.data ();
word_done = fixspace_thinks_word_done (word);
word_count++;
if (word->tess_failed) {
total_score += prev_word_score;
if (prev_word_done)
done_word_count++;
prev_word_score = 0;
prev_char_1 = FALSE;
prev_char_digit = FALSE;
prev_word_done = FALSE;
}
else {
/*
Can we add the prev word score and potentially count this word?
Yes IF it didnt end in a 1 when the first char of this word is a digit
AND it didnt end in a digit when the first char of this word is a 1
*/
word_len = word->reject_map.length ();
current_word_ok_so_far = FALSE;
if (!((prev_char_1 &&
digit_or_numeric_punct (word, 0)) ||
(prev_char_digit &&
((word_done &&
(word->best_choice->unichar_lengths().string()[0] == 1 &&
word->best_choice->unichar_string()[0] == '1')) ||
(!word_done &&
STRING(conflict_set_I_l_1).contains(
word->best_choice->unichar_string ()[0])))))) {
total_score += prev_word_score;
if (prev_word_done)
done_word_count++;
current_word_ok_so_far = word_done;
}
if ((current_word_ok_so_far) &&
(!tessedit_test_uniform_wd_spacing ||
((word->best_choice->permuter () == NUMBER_PERM) ||
uniformly_spaced (word)))) {
prev_word_done = TRUE;
prev_word_score = word_len;
}
else {
prev_word_done = FALSE;
prev_word_score = 0;
}
if (fixsp_prefer_joined_1s) {
/* Add 1 to total score for every joined 1 regardless of context and
rejtn */
for (i = 0, prev_char_1 = FALSE; i < word_len; i++) {
current_char_1 = word->best_choice->unichar_string()[i] == '1';
if (prev_char_1 || (current_char_1 && (i > 0)))
total_score++;
prev_char_1 = current_char_1;
}
}
/* Add 1 to total score for every joined punctuation regardless of context
and rejtn */
if (tessedit_prefer_joined_punct) {
for (i = 0, offset = 0, prev_char_punct = FALSE; i < word_len;
offset += word->best_choice->unichar_lengths()[i++]) {
current_char_punct =
punct_chars.contains (word->best_choice->unichar_string()[offset]);
if (prev_char_punct || (current_char_punct && (i > 0)))
total_score++;
prev_char_punct = current_char_punct;
}
}
prev_char_digit = digit_or_numeric_punct (word, word_len - 1);
for (i = 0, offset = 0; i < word_len - 1;
offset += word->best_choice->unichar_lengths()[i++]);
prev_char_1 =
((word_done
&& (word->best_choice->unichar_string()[offset] == '1'))
|| (!word_done
&& STRING(conflict_set_I_l_1).contains(
word->best_choice->unichar_string()[offset])));
}
/* Find next word */
do
word_res_it.forward ();
while (word_res_it.data ()->part_of_combo);
}
while (!word_res_it.at_first ());
total_score += prev_word_score;
if (prev_word_done)
done_word_count++;
if (done_word_count == word_count)
return PERFECT_WERDS;
else
return total_score;
}
BOOL8 Tesseract::digit_or_numeric_punct(WERD_RES *word, int char_position) {
int i;
int offset;
for (i = 0, offset = 0; i < char_position;
offset += word->best_choice->unichar_lengths()[i++]);
return (unicharset.get_isdigit(word->best_choice->unichar_string().string() + offset,
word->best_choice->unichar_lengths()[i]) ||
(fixsp_numeric_fix &&
(word->best_choice->permuter () == NUMBER_PERM) &&
STRING (numeric_punctuation).contains
(word->best_choice->unichar_string().string()[offset])));
}
} // namespace tesseract
/*************************************************************************
* transform_to_next_perm()
* Examines the current word list to find the smallest word gap size. Then walks
* the word list closing any gaps of this size by either inserted new
* combination words, or extending existing ones.
*
* The routine COULD be limited to stop it building words longer than N blobs.
*
* If there are no more gaps then it DELETES the entire list and returns the
* empty list to cause termination.
*************************************************************************/
void transform_to_next_perm(WERD_RES_LIST &words) {
WERD_RES_IT word_it(&words);
WERD_RES_IT prev_word_it(&words);
WERD_RES *word;
WERD_RES *prev_word;
WERD_RES *combo;
WERD *copy_word;
inT16 prev_right = -1;
TBOX box;
inT16 gap;
inT16 min_gap = MAX_INT16;
for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
word = word_it.data ();
if (!word->part_of_combo) {
box = word->word->bounding_box ();
if (prev_right >= 0) {
gap = box.left () - prev_right;
if (gap < min_gap)
min_gap = gap;
}
prev_right = box.right ();
}
}
if (min_gap < MAX_INT16) {
prev_right = -1; //back to start
word_it.set_to_list (&words);
for (; //cant use cycle pt due to inserted combos at start of list
(prev_right < 0) || !word_it.at_first (); word_it.forward ()) {
word = word_it.data ();
if (!word->part_of_combo) {
box = word->word->bounding_box ();
if (prev_right >= 0) {
gap = box.left () - prev_right;
if (gap <= min_gap) {
prev_word = prev_word_it.data ();
if (prev_word->combination)
combo = prev_word;
else {
/* Make a new combination and insert before the first word being joined */
copy_word = new WERD;
*copy_word = *(prev_word->word);
//deep copy
combo = new WERD_RES (copy_word);
combo->combination = TRUE;
combo->x_height = prev_word->x_height;
prev_word->part_of_combo = TRUE;
prev_word_it.add_before_then_move (combo);
}
combo->word->set_flag (W_EOL, word->word->flag (W_EOL));
if (word->combination) {
combo->word->join_on (word->word);
//Move blbs to combo
//old combo no longer needed
delete word_it.extract ();
}
else {
//Cpy current wd to combo
combo->copy_on (word);
word->part_of_combo = TRUE;
}
combo->done = FALSE;
if (combo->outword != NULL) {
delete combo->outword;
delete combo->best_choice;
delete combo->raw_choice;
combo->outword = NULL;
combo->best_choice = NULL;
combo->raw_choice = NULL;
}
}
else
//catch up
prev_word_it = word_it;
}
prev_right = box.right ();
}
}
}
else
words.clear (); //signal termination
}
void dump_words(WERD_RES_LIST &perm, inT16 score, inT16 mode, BOOL8 improved) {
WERD_RES_IT word_res_it(&perm);
static STRING initial_str;
if (debug_fix_space_level > 0) {
if (mode == 1) {
initial_str = "";
for (word_res_it.mark_cycle_pt ();
!word_res_it.cycled_list (); word_res_it.forward ()) {
if (!word_res_it.data ()->part_of_combo) {
initial_str += word_res_it.data()->best_choice->unichar_string();
initial_str += ' ';
}
}
}
#ifndef SECURE_NAMES
if (debug_fix_space_level > 1) {
switch (mode) {
case 1:
tprintf ("EXTRACTED (%d): \"", score);
break;
case 2:
tprintf ("TESTED (%d): \"", score);
break;
case 3:
tprintf ("RETURNED (%d): \"", score);
break;
}
for (word_res_it.mark_cycle_pt ();
!word_res_it.cycled_list (); word_res_it.forward ()) {
if (!word_res_it.data ()->part_of_combo)
tprintf("%s/%1d ",
word_res_it.data()->best_choice->unichar_string().string(),
(int)word_res_it.data()->best_choice->permuter());
}
tprintf ("\"\n");
}
else if (improved) {
tprintf ("FIX SPACING \"%s\" => \"", initial_str.string ());
for (word_res_it.mark_cycle_pt ();
!word_res_it.cycled_list (); word_res_it.forward ()) {
if (!word_res_it.data ()->part_of_combo)
tprintf ("%s/%1d ",
word_res_it.data()->best_choice->unichar_string().string(),
(int)word_res_it.data()->best_choice->permuter());
}
tprintf ("\"\n");
}
#endif
}
}
/*************************************************************************
* uniformly_spaced()
* Return true if one of the following are true:
* - All inter-char gaps are the same width
* - The largest gap is no larger than twice the mean/median of the others
* - The largest gap is < 64/5 = 13 and all others are <= 0
* **** REMEMBER - WE'RE NOW WORKING WITH A BLN WERD !!!
*************************************************************************/
BOOL8 uniformly_spaced( //sensible word
WERD_RES *word) {
PBLOB_IT blob_it;
TBOX box;
inT16 prev_right = -MAX_INT16;
inT16 gap;
inT16 max_gap = -MAX_INT16;
inT16 max_gap_count = 0;
STATS gap_stats (0, MAXSPACING);
BOOL8 result;
const ROW *row = word->denorm.row ();
float max_non_space;
float normalised_max_nonspace;
inT16 i = 0;
inT16 offset = 0;
STRING punct_chars = "\"`',.:;";
blob_it.set_to_list (word->outword->blob_list ());
for (blob_it.mark_cycle_pt (); !blob_it.cycled_list (); blob_it.forward ()) {
box = blob_it.data ()->bounding_box ();
if ((prev_right > -MAX_INT16) &&
(!fixsp_ignore_punct ||
(!punct_chars.contains (word->best_choice->unichar_string()
[offset - word->best_choice->unichar_lengths()[i - 1]]) &&
!punct_chars.contains (word->best_choice->unichar_string()[offset])))) {
gap = box.left () - prev_right;
if (gap < max_gap)
gap_stats.add (gap, 1);
else if (gap == max_gap)
max_gap_count++;
else {
if (max_gap_count > 0)
gap_stats.add (max_gap, max_gap_count);
max_gap = gap;
max_gap_count = 1;
}
}
prev_right = box.right ();
offset += word->best_choice->unichar_lengths()[i++];
}
max_non_space = (row->space () + 3 * row->kern ()) / 4;
normalised_max_nonspace = max_non_space * bln_x_height / row->x_height ();
result = ((gap_stats.get_total () == 0) ||
(max_gap <= normalised_max_nonspace) ||
((gap_stats.get_total () > 2) &&
(max_gap <= 2 * gap_stats.median ())) ||
((gap_stats.get_total () <= 2) &&
(max_gap <= 2 * gap_stats.mean ())));
#ifndef SECURE_NAMES
if ((debug_fix_space_level > 1)) {
if (result)
tprintf
("ACCEPT SPACING FOR: \"%s\" norm_maxnon = %f max=%d maxcount=%d total=%d mean=%f median=%f\n",
word->best_choice->unichar_string().string (), normalised_max_nonspace,
max_gap, max_gap_count, gap_stats.get_total (), gap_stats.mean (),
gap_stats.median ());
else
tprintf
("REJECT SPACING FOR: \"%s\" norm_maxnon = %f max=%d maxcount=%d total=%d mean=%f median=%f\n",
word->best_choice->unichar_string().string (), normalised_max_nonspace,
max_gap, max_gap_count, gap_stats.get_total (), gap_stats.mean (),
gap_stats.median ());
}
#endif
return result;
}
BOOL8 fixspace_thinks_word_done(WERD_RES *word) {
if (word->done)
return TRUE;
/*
Use all the standard pass 2 conditions for mode 5 in set_done() in
reject.c BUT DONT REJECT IF THE WERD IS AMBIGUOUS - FOR SPACING WE DONT
CARE WHETHER WE HAVE of/at on/an etc.
*/
if ((fixsp_done_mode > 0) &&
(word->tess_accepted ||
((fixsp_done_mode == 2) &&
(word->reject_map.reject_count () == 0)) ||
(fixsp_done_mode == 3)) &&
(strchr (word->best_choice->unichar_string().string (), ' ') == NULL) &&
((word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
(word->best_choice->permuter () == FREQ_DAWG_PERM) ||
(word->best_choice->permuter () == USER_DAWG_PERM) ||
(word->best_choice->permuter () == NUMBER_PERM)))
return TRUE;
else
return FALSE;
}
/*************************************************************************
* fix_sp_fp_word()
* Test the current word to see if it can be split by deleting noise blobs. If
* so, do the buisiness.
* Return with the iterator pointing to the same place if the word is unchanged,
* or the last of the replacement words.
*************************************************************************/
namespace tesseract {
void Tesseract::fix_sp_fp_word(WERD_RES_IT &word_res_it, ROW *row,
BLOCK* block) {
WERD_RES *word_res;
WERD_RES_LIST sub_word_list;
WERD_RES_IT sub_word_list_it(&sub_word_list);
inT16 blob_index;
inT16 new_length;
float junk;
word_res = word_res_it.data ();
if (!fixsp_check_for_fp_noise_space ||
word_res->word->flag (W_REP_CHAR) ||
word_res->combination ||
word_res->part_of_combo || !word_res->word->flag (W_DONT_CHOP))
return;
blob_index = worst_noise_blob (word_res, &junk);
if (blob_index < 0)
return;
#ifndef SECURE_NAMES
if (debug_fix_space_level > 1) {
tprintf ("FP fixspace working on \"%s\"\n",
word_res->best_choice->unichar_string().string());
}
#endif
gblob_sort_list ((PBLOB_LIST *) word_res->word->rej_cblob_list (), FALSE);
sub_word_list_it.add_after_stay_put (word_res_it.extract ());
fix_noisy_space_list(sub_word_list, row, block);
new_length = sub_word_list.length ();
word_res_it.add_list_before (&sub_word_list);
for (; (!word_res_it.at_last () && (new_length > 1)); new_length--) {
word_res_it.forward ();
}
}
void Tesseract::fix_noisy_space_list(WERD_RES_LIST &best_perm, ROW *row,
BLOCK* block) {
inT16 best_score;
WERD_RES_IT best_perm_it(&best_perm);
WERD_RES_LIST current_perm;
WERD_RES_IT current_perm_it(¤t_perm);
WERD_RES *old_word_res;
WERD_RES *new_word_res;
inT16 current_score;
BOOL8 improved = FALSE;
//default score
best_score = fp_eval_word_spacing (best_perm);
dump_words (best_perm, best_score, 1, improved);
new_word_res = new WERD_RES;
old_word_res = best_perm_it.data ();
//Kludge to force deep copy
old_word_res->combination = TRUE;
*new_word_res = *old_word_res; //deep copy
//Undo kludge
old_word_res->combination = FALSE;
//Undo kludge
new_word_res->combination = FALSE;
current_perm_it.add_to_end (new_word_res);
break_noisiest_blob_word(current_perm);
while ((best_score != PERFECT_WERDS) && !current_perm.empty ()) {
match_current_words(current_perm, row, block);
current_score = fp_eval_word_spacing (current_perm);
dump_words (current_perm, current_score, 2, improved);
if (current_score > best_score) {
best_perm.clear ();
best_perm.deep_copy(¤t_perm, &WERD_RES::deep_copy);
best_score = current_score;
improved = TRUE;
}
if (current_score < PERFECT_WERDS)
break_noisiest_blob_word(current_perm);
}
dump_words (best_perm, best_score, 3, improved);
}
} // namespace tesseract
/*************************************************************************
* break_noisiest_blob_word()
* Find the word with the blob which looks like the worst noise.
* Break the word into two, deleting the noise blob.
*************************************************************************/
void break_noisiest_blob_word(WERD_RES_LIST &words) {
WERD_RES_IT word_it(&words);
WERD_RES_IT worst_word_it;
float worst_noise_score = 9999;
int worst_blob_index = -1; //noisiest blb of noisiest wd
int blob_index; //of wds noisiest blb
float noise_score; //of wds noisiest blb
WERD_RES *word_res;
C_BLOB_IT blob_it;
C_BLOB_IT rej_cblob_it;
C_BLOB_LIST new_blob_list;
C_BLOB_IT new_blob_it;
C_BLOB_IT new_rej_cblob_it;
WERD *new_word;
inT16 start_of_noise_blob;
inT16 i;
for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
blob_index = worst_noise_blob (word_it.data (), &noise_score);
if ((blob_index > -1) && (worst_noise_score > noise_score)) {
worst_noise_score = noise_score;
worst_blob_index = blob_index;
worst_word_it = word_it;
}
}
if (worst_blob_index < 0) {
words.clear (); //signal termination
return;
}
/* Now split the worst_word_it */
word_res = worst_word_it.data ();
/* Move blobs before noise blob to a new bloblist */
new_blob_it.set_to_list (&new_blob_list);
blob_it.set_to_list (word_res->word->cblob_list ());
for (i = 0; i < worst_blob_index; i++, blob_it.forward ()) {
new_blob_it.add_after_then_move (blob_it.extract ());
}
start_of_noise_blob = blob_it.data ()->bounding_box ().left ();
delete blob_it.extract (); //throw out noise blb
new_word = new WERD (&new_blob_list, word_res->word);
new_word->set_flag (W_EOL, FALSE);
word_res->word->set_flag (W_BOL, FALSE);
word_res->word->set_blanks (1);//After break
new_rej_cblob_it.set_to_list (new_word->rej_cblob_list ());
rej_cblob_it.set_to_list (word_res->word->rej_cblob_list ());
for (;
(!rej_cblob_it.empty () &&
(rej_cblob_it.data ()->bounding_box ().left () <
start_of_noise_blob)); rej_cblob_it.forward ()) {
new_rej_cblob_it.add_after_then_move (rej_cblob_it.extract ());
}
worst_word_it.add_before_then_move (new WERD_RES (new_word));
word_res->done = FALSE;
if (word_res->outword != NULL) {
delete word_res->outword;
delete word_res->best_choice;
delete word_res->raw_choice;
word_res->outword = NULL;
word_res->best_choice = NULL;
word_res->raw_choice = NULL;
}
}
inT16 worst_noise_blob(WERD_RES *word_res, float *worst_noise_score) {
PBLOB_IT blob_it;
inT16 blob_count;
float noise_score[512];
int i;
int min_noise_blob; //1st contender
int max_noise_blob; //last contender
int non_noise_count;
int worst_noise_blob; //Worst blob
float small_limit = bln_x_height * fixsp_small_outlines_size;
float non_noise_limit = bln_x_height * 0.8;
blob_it.set_to_list (word_res->outword->blob_list ());
//normalised
blob_count = blob_it.length ();
ASSERT_HOST (blob_count <= 512);
if (blob_count < 5)
return -1; //too short to split
/* Get the noise scores for all blobs */
#ifndef SECURE_NAMES
if (debug_fix_space_level > 5)
tprintf ("FP fixspace Noise metrics for \"%s\": ",
word_res->best_choice->unichar_string().string());
#endif
for (i = 0; i < blob_count; i++, blob_it.forward ()) {
if (word_res->reject_map[i].accepted ())
noise_score[i] = non_noise_limit;
else
noise_score[i] = blob_noise_score (blob_it.data ());
if (debug_fix_space_level > 5)
tprintf ("%1.1f ", noise_score[i]);
}
if (debug_fix_space_level > 5)
tprintf ("\n");
/* Now find the worst one which is far enough away from the end of the word */
non_noise_count = 0;
for (i = 0;
(i < blob_count) && (non_noise_count < fixsp_non_noise_limit); i++) {
if (noise_score[i] >= non_noise_limit)
non_noise_count++;
}
if (non_noise_count < fixsp_non_noise_limit)
return -1;
min_noise_blob = i;
non_noise_count = 0;
for (i = blob_count - 1;
(i >= 0) && (non_noise_count < fixsp_non_noise_limit); i--) {
if (noise_score[i] >= non_noise_limit)
non_noise_count++;
}
if (non_noise_count < fixsp_non_noise_limit)
return -1;
max_noise_blob = i;
if (min_noise_blob > max_noise_blob)
return -1;
*worst_noise_score = small_limit;
worst_noise_blob = -1;
for (i = min_noise_blob; i <= max_noise_blob; i++) {
if (noise_score[i] < *worst_noise_score) {
worst_noise_blob = i;
*worst_noise_score = noise_score[i];
}
}
return worst_noise_blob;
}
float blob_noise_score(PBLOB *blob) {
OUTLINE_IT outline_it;
TBOX box; //BB of outline
inT16 outline_count = 0;
inT16 max_dimension;
inT16 largest_outline_dimension = 0;
outline_it.set_to_list (blob->out_list ());
for (outline_it.mark_cycle_pt ();
!outline_it.cycled_list (); outline_it.forward ()) {
outline_count++;
box = outline_it.data ()->bounding_box ();
if (box.height () > box.width ())
max_dimension = box.height ();
else
max_dimension = box.width ();
if (largest_outline_dimension < max_dimension)
largest_outline_dimension = max_dimension;
}
if (fixsp_noise_score_fixing) {
if (outline_count > 5)
//penalise LOTS of blobs
largest_outline_dimension *= 2;
box = blob->bounding_box ();
if ((box.bottom () > bln_baseline_offset * 4) ||
(box.top () < bln_baseline_offset / 2))
//Lax blob is if high or low
largest_outline_dimension /= 2;
}
return largest_outline_dimension;
}
void fixspace_dbg(WERD_RES *word) {
TBOX box = word->word->bounding_box ();
BOOL8 show_map_detail = FALSE;
inT16 i;
box.print ();
#ifndef SECURE_NAMES
tprintf (" \"%s\" ", word->best_choice->unichar_string().string ());
tprintf ("Blob count: %d (word); %d/%d (outword)\n",
word->word->gblob_list ()->length (),
word->outword->gblob_list ()->length (),
word->outword->rej_blob_list ()->length ());
word->reject_map.print (debug_fp);
tprintf ("\n");
if (show_map_detail) {
tprintf ("\"%s\"\n", word->best_choice->unichar_string().string ());
for (i = 0; word->best_choice->unichar_string()[i] != '\0'; i++) {
tprintf ("**** \"%c\" ****\n", word->best_choice->unichar_string()[i]);
word->reject_map[i].full_print (debug_fp);
}
}
tprintf ("Tess Accepted: %s\n", word->tess_accepted ? "TRUE" : "FALSE");
tprintf ("Done flag: %s\n\n", word->done ? "TRUE" : "FALSE");
#endif
}
/*************************************************************************
* fp_eval_word_spacing()
* Evaluation function for fixed pitch word lists.
*
* Basically, count the number of "nice" characters - those which are in tess
* acceptable words or in dict words and are not rejected.
* Penalise any potential noise chars
*************************************************************************/
namespace tesseract {
inT16 Tesseract::fp_eval_word_spacing(WERD_RES_LIST &word_res_list) {
WERD_RES_IT word_it(&word_res_list);
WERD_RES *word;
PBLOB_IT blob_it;
inT16 word_length;
inT16 score = 0;
inT16 i;
float small_limit = bln_x_height * fixsp_small_outlines_size;
if (!fixsp_fp_eval)
return (eval_word_spacing (word_res_list));
for (word_it.mark_cycle_pt (); !word_it.cycled_list (); word_it.forward ()) {
word = word_it.data ();
word_length = word->reject_map.length ();
if ((word->done ||
word->tess_accepted) ||
(word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
(word->best_choice->permuter () == FREQ_DAWG_PERM) ||
(word->best_choice->permuter () == USER_DAWG_PERM) ||
(safe_dict_word(*(word->best_choice)) > 0)) {
blob_it.set_to_list (word->outword->blob_list ());
UNICHAR_ID space = getDict().getUnicharset().unichar_to_id(" ");
for (i = 0; i < word->best_choice->length(); ++i, blob_it.forward()) {
if (word->best_choice->unichar_id(i) == space ||
(blob_noise_score(blob_it.data()) < small_limit)) {
score -= 1; //penalise possibly erroneous non-space
} else if (word->reject_map[i].accepted()) {
score++;
}
}
}
}
if (score < 0)
score = 0;
return score;
}
} // namespace tesseract