From 7c786c655a2e8bf7dfd38f6965eee1274065bf7a Mon Sep 17 00:00:00 2001 From: dhdfu Date: Tue, 4 Dec 2007 13:59:30 +0000 Subject: [PATCH] Fill in more of the interface (including an intentional misspeling to annoy future generations), and add a plan. I'm not convinced that the "virtual function" implementation here is actually going to be necessary, but it will make things easier for the time being. git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase@7462 94700074-3cef-4d97-a70e-9c8c206c02f5 --- include/ngram_model.h | 36 +-- src/libsphinxbase/lm/ngram_model.c | 39 ++- src/libsphinxbase/lm/ngram_model_arpa.c | 253 ++++++++++++++++++++ src/libsphinxbase/lm/ngram_model_arpa.h | 8 +- src/libsphinxbase/lm/ngram_model_dmp.c | 8 + src/libsphinxbase/lm/ngram_model_dmp.h | 8 +- src/libsphinxbase/lm/ngram_model_internal.h | 4 + src/libsphinxbase/lm/plan.txt | 9 + 8 files changed, 334 insertions(+), 31 deletions(-) create mode 100644 src/libsphinxbase/lm/plan.txt diff --git a/include/ngram_model.h b/include/ngram_model.h index 8dcf2e6..60dbb91 100644 --- a/include/ngram_model.h +++ b/include/ngram_model.h @@ -106,7 +106,7 @@ void ngram_model_free(ngram_model_t *model); * your system does not have iconv, this function may fail. Also, * because all file formats consist of 8-bit character streams, * attempting to convert to or from UTF-16 (or any other encoding - * which contains null bytes) is a recipe for total disaster. + * which contains null bytes) is a recipe for total desaster. * * We have no interest in supporting UTF-16, so don't ask. * @@ -118,12 +118,16 @@ void ngram_model_free(ngram_model_t *model); int ngram_model_recode(ngram_model_t *model, const char *from, const char *to); /** - * Apply a language weight, insertion penalty, and unigram weight internally. + * Irreversibly apply a language weight, insertion penalty, and + * unigram weight internally. * * This will change the values output by ngram_score() and friends. * This is done for efficiency since in decoding, these are the only * values we actually need. Call ngram_prob() if you want the "raw" * N-Gram probability estimate. + * + * Note that the unigram probability will still be interpolated in the + * output of ngram_prob(), which may be a bug. */ int ngram_apply_weights(ngram_model_t *model, float32 lw, float32 wip, float32 uw); @@ -140,18 +144,13 @@ int ngram_apply_weights(ngram_model_t *model, * score = ngram_score(model, "joy", "whole", "a", NULL); * * This is not the function to use in decoding, because it has some - * overhead for looking up words. Use ngram_tg_score() or - * ngram_bg_score() instead. In the future there will probably be a - * version that takes a general language model state object, to - * support suffix-array LM and things like that. + * overhead for looking up words. Use ngram_ng_score(), + * ngram_tg_score(), or ngram_bg_score() instead. In the future there + * will probably be a version that takes a general language model + * state object, to support suffix-array LM and things like that. */ int32 ngram_score(ngram_model_t *model, const char *word, ...); -/** - * Explicit va_list version of ngram_score(). - */ -int32 ngram_score_v(ngram_model_t *model, const char *word, va_list history); - /** * Quick trigram score lookup. */ @@ -162,17 +161,26 @@ int32 ngram_tg_score(ngram_model_t *model, int32 w3, int32 w2, int32 w1); */ int32 ngram_bg_score(ngram_model_t *model, int32 w2, int32 w1); +/** + * Quick general N-Gram score lookup. + */ +int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist); + /** * Get the "raw" log-probability for a general N-Gram. * - * See documentation for ngram_score() for an explanation of this. + * See documentation for ngram_score() and ngram_apply_weights() for + * an explanation of this. */ int32 ngram_prob(ngram_model_t *model, const char *word, ...); /** - * Explicit va_list version of ngram_prob(). + * Quick "raw" probability lookup for a general N-Gram. + * + * See documentation for ngram_ng_score() and ngram_apply_weights() + * for an explanation of this. */ -int32 ngram_prob_v(ngram_model_t *model, const char *word, va_list history); +int32 ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist); /** * Look up numerical word ID. diff --git a/src/libsphinxbase/lm/ngram_model.c b/src/libsphinxbase/lm/ngram_model.c index dcb5b93..e1224ed 100644 --- a/src/libsphinxbase/lm/ngram_model.c +++ b/src/libsphinxbase/lm/ngram_model.c @@ -287,43 +287,58 @@ int ngram_apply_weights(ngram_model_t *model, float32 lw, float32 wip, float32 uw) { - return -1; + return (*model->funcs->apply_weights)(model, lw, wip, uw); } int32 ngram_score(ngram_model_t *model, const char *word, ...) { - return NGRAM_SCORE_ERROR; -} + va_list history; + int32 prob; -int32 -ngram_score_v(ngram_model_t *model, const char *word, va_list history) -{ - return NGRAM_SCORE_ERROR; + va_start(history, word); + + va_end(history); + + return prob; } int32 ngram_tg_score(ngram_model_t *model, int32 w3, int32 w2, int32 w1) { - return NGRAM_SCORE_ERROR; + int32 hist[2] = { w1, w2 }; + return (*model->funcs->score)(model, w3, hist, 2); } int32 ngram_bg_score(ngram_model_t *model, int32 w2, int32 w1) { - return NGRAM_SCORE_ERROR; + return (*model->funcs->score)(model, w2, &w1, 1); +} + +int32 +ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist) +{ + return (*model->funcs->score)(model, wid, history, n_hist); } int32 ngram_prob(ngram_model_t *model, const char *word, ...) { - return NGRAM_SCORE_ERROR; + va_list history; + int32 prob; + + va_start(history, word); + + va_end(history); + + return prob; } int32 -ngram_prob_v(ngram_model_t *model, const char *word, va_list history) +ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist) { - return NGRAM_SCORE_ERROR; + return (*model->funcs->raw_score)(model, wid, history, n_hist); } int32 diff --git a/src/libsphinxbase/lm/ngram_model_arpa.c b/src/libsphinxbase/lm/ngram_model_arpa.c index 8fe25f4..b41864c 100644 --- a/src/libsphinxbase/lm/ngram_model_arpa.c +++ b/src/libsphinxbase/lm/ngram_model_arpa.c @@ -44,6 +44,7 @@ #include "ngram_model_arpa.h" #include "err.h" #include "pio.h" +#include "linklist.h" #include @@ -467,6 +468,9 @@ ngram_model_arpa_read(cmd_ln_t *config, base->n_1g_alloc = base->n_counts[0] = n_unigram; base->n_counts[1] = n_bigram; base->n_counts[2] = n_trigram; + base->lw = 1.0; + base->wip = 1.0; + base->uw = 1.0; /* Allocate space for word strings. */ base->word_str = ckd_calloc(n_unigram, sizeof(char *)); /* NOTE: They are no longer case-insensitive since we are allowing @@ -523,6 +527,10 @@ ngram_model_arpa_read(cmd_ln_t *config, E_INFO("%8d = #prob3 entries\n", model->n_prob3); free_sorted_list(&model->sorted_prob3); + + /* Initialize tginfo */ + model->tginfo = + ckd_calloc(base->n_1g_alloc, sizeof(tginfo_t *)); } fclose_comp(fp, is_pipe); @@ -540,9 +548,235 @@ static int ngram_model_arpa_apply_weights(ngram_model_t *base, float32 lw, float32 wip, float32 uw) { + ngram_model_arpa_t *model = (ngram_model_arpa_t *)base; + int32 log_wip, log_uw, log_uniform; + int i; + + /* Precalculate some log values we will like. */ + log_wip = logmath_log(base->lmath, wip); + log_uw = logmath_log(base->lmath, uw); + log_uniform = logmath_log(base->lmath, 1.0 / (base->n_counts[0] - 1)) + + logmath_log(base->lmath, 1.0 - uw); + + for (i = 0; i < base->n_counts[0]; ++i) { + model->unigrams[i].bo_wt1.l *= lw; + + if (strcmp(base->word_str[i], "") == 0) { /* FIXME: configurable start_sym */ + /* Apply language weight and WIP */ + model->unigrams[i].prob1.l *= lw; + model->unigrams[i].prob1.l += log_wip; + } + else { + /* Interpolate unigram probability with uniform. */ + model->unigrams[i].prob1.l += log_uw; + model->unigrams[i].prob1.l = + logmath_add(base->lmath, + model->unigrams[i].prob1.l, + log_uniform); + /* Apply language weight and WIP */ + model->unigrams[i].prob1.l *= lw; + model->unigrams[i].prob1.l += log_wip; + } + } + + for (i = 0; i < model->n_prob2; ++i) { + model->prob2[i].l *= lw; + model->prob2[i].l += log_wip; + } + + if (base->n > 2) { + for (i = 0; i < model->n_bo_wt2; ++i) { + model->bo_wt2[i].l *= lw; + } + for (i = 0; i < model->n_prob3; i++) { + model->prob3[i].l *= lw; + model->prob3[i].l += log_wip; + } + } return 0; } +static int32 +lm3g_ug_score(ngram_model_arpa_t *model, int32 lwid) +{ + return model->unigrams[lwid].prob1.l; +} + +/* Locate a specific bigram within a bigram list */ +#define BINARY_SEARCH_THRESH 16 +static int32 +find_bg(bigram_t * bg, int32 n, int32 w) +{ + int32 i, b, e; + + /* Binary search until segment size < threshold */ + b = 0; + e = n; + while (e - b > BINARY_SEARCH_THRESH) { + i = (b + e) >> 1; + if (bg[i].wid < w) + b = i + 1; + else if (bg[i].wid > w) + e = i; + else + return i; + } + + /* Linear search within narrowed segment */ + for (i = b; (i < e) && (bg[i].wid != w); i++); + return ((i < e) ? i : -1); +} + +static int32 +lm3g_bg_score(ngram_model_arpa_t *model, int32 lw1, int32 lw2) +{ + int32 i, n, b, score; + bigram_t *bg; + + b = FIRST_BG(model, lw1); + n = FIRST_BG(model, lw1 + 1) - b; + bg = model->bigrams + b; + + if ((i = find_bg(bg, n, lw2)) >= 0) { + score = model->prob2[bg[i].prob2].l; + } + else { + score = model->unigrams[lw1].bo_wt1.l + model->unigrams[lw2].prob1.l; + } + + return (score); +} + +static void +load_tginfo(ngram_model_arpa_t *model, int32 lw1, int32 lw2) +{ + int32 i, n, b, t; + bigram_t *bg; + tginfo_t *tginfo; + + /* First allocate space for tg information for bg lw1,lw2 */ + tginfo = (tginfo_t *) listelem_alloc(sizeof(tginfo_t)); + tginfo->w1 = lw1; + tginfo->tg = NULL; + tginfo->next = model->tginfo[lw2]; + model->tginfo[lw2] = tginfo; + + /* Locate bigram lw1,lw2 */ + + b = model->unigrams[lw1].bigrams; + n = model->unigrams[lw1 + 1].bigrams - b; + bg = model->bigrams + b; + + if ((n > 0) && ((i = find_bg(bg, n, lw2)) >= 0)) { + tginfo->bowt = model->bo_wt2[bg[i].bo_wt2].l; + + /* Find t = Absolute first trigram index for bigram lw1,lw2 */ + b += i; /* b = Absolute index of bigram lw1,lw2 on disk */ + t = FIRST_TG(model, b); + + tginfo->tg = model->trigrams + t; + + /* Find #tg for bigram w1,w2 */ + tginfo->n_tg = FIRST_TG(model, b + 1) - t; + } + else { /* No bigram w1,w2 */ + tginfo->bowt = 0; + tginfo->n_tg = 0; + } +} + +/* Similar to find_bg */ +static int32 +find_tg(trigram_t * tg, int32 n, int32 w) +{ + int32 i, b, e; + + b = 0; + e = n; + while (e - b > BINARY_SEARCH_THRESH) { + i = (b + e) >> 1; + if (tg[i].wid < w) + b = i + 1; + else if (tg[i].wid > w) + e = i; + else + return i; + } + + for (i = b; (i < e) && (tg[i].wid != w); i++); + return ((i < e) ? i : -1); +} + +static int32 +lm3g_tg_score(ngram_model_arpa_t *model, int32 lw1, int32 lw2, int32 lw3) +{ + ngram_model_t *base = &model->base; + int32 i, n, score; + trigram_t *tg; + tginfo_t *tginfo, *prev_tginfo; + + if ((base->n < 3) || (lw1 < 0)) + return (lm3g_bg_score(model, lw2, lw3)); + + prev_tginfo = NULL; + for (tginfo = model->tginfo[lw2]; tginfo; tginfo = tginfo->next) { + if (tginfo->w1 == lw1) + break; + prev_tginfo = tginfo; + } + + if (!tginfo) { + load_tginfo(model, lw1, lw2); + tginfo = model->tginfo[lw2]; + } + else if (prev_tginfo) { + prev_tginfo->next = tginfo->next; + tginfo->next = model->tginfo[lw2]; + model->tginfo[lw2] = tginfo; + } + + tginfo->used = 1; + + /* Trigrams for w1,w2 now pointed to by tginfo */ + n = tginfo->n_tg; + tg = tginfo->tg; + if ((i = find_tg(tg, n, lw3)) >= 0) { + score = model->prob3[tg[i].prob3].l; + } + else { + score = tginfo->bowt + lm3g_bg_score(model, lw2, lw3); + } + + return (score); +} + +static void +lm3g_cache_reset(ngram_model_arpa_t *model) +{ + ngram_model_t *base = &model->base; + int32 i; + tginfo_t *tginfo, *next_tginfo, *prev_tginfo; + + for (i = 0; i < base->n_counts[0]; i++) { + prev_tginfo = NULL; + for (tginfo = model->tginfo[i]; tginfo; tginfo = next_tginfo) { + next_tginfo = tginfo->next; + + if (!tginfo->used) { + listelem_free((void *) tginfo, sizeof(tginfo_t)); + if (prev_tginfo) + prev_tginfo->next = next_tginfo; + else + model->tginfo[i] = next_tginfo; + } + else { + tginfo->used = 0; + prev_tginfo = tginfo; + } + } + } +} + static int32 ngram_model_arpa_score(ngram_model_t *base, int32 wid, int32 *history, int32 n_hist) @@ -550,6 +784,13 @@ ngram_model_arpa_score(ngram_model_t *base, int32 wid, return NGRAM_SCORE_ERROR; } +static int32 +ngram_model_arpa_raw_score(ngram_model_t *base, int32 wid, + int32 *history, int32 n_hist) +{ + return NGRAM_SCORE_ERROR; +} + static void ngram_model_arpa_free(ngram_model_t *base) { @@ -561,11 +802,23 @@ ngram_model_arpa_free(ngram_model_t *base) ckd_free(model->prob2); ckd_free(model->bo_wt2); ckd_free(model->prob3); + if (model->tginfo) { + int32 u; + for (u = 0; u < base->n_1g_alloc; u++) { + tginfo_t *tginfo, *next_tginfo; + for (tginfo = model->tginfo[u]; tginfo; tginfo = next_tginfo) { + next_tginfo = tginfo->next; + listelem_free(tginfo, sizeof(*tginfo)); + } + } + ckd_free(model->tginfo); + } ckd_free(model->tseg_base); } static ngram_funcs_t ngram_model_arpa_funcs = { ngram_model_arpa_apply_weights, /* apply_weights */ ngram_model_arpa_score, /* score */ + ngram_model_arpa_raw_score, /* raw_score */ ngram_model_arpa_free /* free */ }; diff --git a/src/libsphinxbase/lm/ngram_model_arpa.h b/src/libsphinxbase/lm/ngram_model_arpa.h index 1876acb..30d1b61 100644 --- a/src/libsphinxbase/lm/ngram_model_arpa.h +++ b/src/libsphinxbase/lm/ngram_model_arpa.h @@ -77,6 +77,7 @@ typedef struct unigram_s { */ #define BG_SEG_SZ 512 /* chosen so that #trigram/segment <= 2**16 */ #define LOG_BG_SEG_SZ 9 +#define TSEG_BASE(m,b) ((m)->tseg_base[(b)>>LOG_BG_SEG_SZ]) /** * Bigram structure. @@ -108,7 +109,6 @@ typedef struct trigram_s { * tree to locate trigrams for a given bigram (lw1,lw2). The organization is optimized * for locality of access (to the same lw1), given lw2. */ -#define TSEG_BASE(m,b) ((m)->tseg_base[(b)>>LOG_BG_SEG_SZ]) typedef struct tginfo_s { int32 w1; /**< lw1 component of bigram lw1,lw2. All bigrams with same lw2 linked together (see lm_t.tginfo). */ @@ -165,7 +165,11 @@ typedef struct ngram_model_arpa_s { int32 *tseg_base; /* tseg_base[i>>LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i>>LOG_BG_SEG_SZ) */ - /* Arrays of unique bigram probs and bo-wts, and trigram probs */ + tginfo_t **tginfo; /* tginfo[lw2] is head of linked list of trigram information for + some cached subset of bigrams (*,lw2). */ + + /* Arrays of unique bigram probs and bo-wts, and trigram probs + * (these are temporary, actually) */ sorted_list_t sorted_prob2; sorted_list_t sorted_bo_wt2; sorted_list_t sorted_prob3; diff --git a/src/libsphinxbase/lm/ngram_model_dmp.c b/src/libsphinxbase/lm/ngram_model_dmp.c index 2d591d5..82e0d59 100644 --- a/src/libsphinxbase/lm/ngram_model_dmp.c +++ b/src/libsphinxbase/lm/ngram_model_dmp.c @@ -437,6 +437,13 @@ ngram_model_dmp_score(ngram_model_t *model, int32 wid, return NGRAM_SCORE_ERROR; } +static int32 +ngram_model_dmp_raw_score(ngram_model_t *model, int32 wid, + int32 *history, int32 n_hist) +{ + return NGRAM_SCORE_ERROR; +} + static void ngram_model_dmp_free(ngram_model_t *base) { @@ -472,5 +479,6 @@ ngram_model_dmp_free(ngram_model_t *base) static ngram_funcs_t ngram_model_dmp_funcs = { ngram_model_dmp_apply_weights, /* apply_weights */ ngram_model_dmp_score, /* score */ + ngram_model_dmp_raw_score, /* raw_score */ ngram_model_dmp_free /* free */ }; diff --git a/src/libsphinxbase/lm/ngram_model_dmp.h b/src/libsphinxbase/lm/ngram_model_dmp.h index 66a8f65..2e1d645 100644 --- a/src/libsphinxbase/lm/ngram_model_dmp.h +++ b/src/libsphinxbase/lm/ngram_model_dmp.h @@ -46,17 +46,19 @@ #include "ngram_model_internal.h" #include "mmio.h" -/** On-disk representation of language model probabilities. */ +/** + * Language model probabilities. + */ typedef union { float32 f; int32 l; } lmprob_t; /** - * On-disk representation of unigrams. + * Unigram objects. */ typedef struct unigram_s { - int32 mapid; /**< Holds dictionary ID or whatever. */ + int32 mapid; /**< UNUSED. */ lmprob_t prob1; /**< Unigram probability. */ lmprob_t bo_wt1; /**< Unigram backoff weight. */ int32 bigrams; /**< Index of 1st entry in lm_t.bigrams[] */ diff --git a/src/libsphinxbase/lm/ngram_model_internal.h b/src/libsphinxbase/lm/ngram_model_internal.h index 564328c..f820cd5 100644 --- a/src/libsphinxbase/lm/ngram_model_internal.h +++ b/src/libsphinxbase/lm/ngram_model_internal.h @@ -60,12 +60,16 @@ struct ngram_model_s { char **word_str; /**< Unigram names */ hash_table_t *wid; /**< Mapping of unigram names to word IDs. */ logmath_t *lmath; /**< Log-math object */ + float32 lw; /**< Language model scaling factor */ + float32 wip; /**< Word insertion penalty */ + float32 uw; /**< Unigram weight */ struct ngram_funcs_s *funcs; /**< Implementation-specific methods. */ }; typedef struct ngram_funcs_s { int (*apply_weights)(ngram_model_t *model, float32 lw, float32 wip, float32 uw); int32 (*score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist); + int32 (*raw_score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist); void (*free)(ngram_model_t *model); } ngram_funcs_t; diff --git a/src/libsphinxbase/lm/plan.txt b/src/libsphinxbase/lm/plan.txt new file mode 100644 index 0000000..7e41634 --- /dev/null +++ b/src/libsphinxbase/lm/plan.txt @@ -0,0 +1,9 @@ +Implementation plan: + + * Get ARPABO and DMP stuff up and going, with a battery of appropriate + tests and regression test against PocketSphinx/Sphinx3 + + * Generalize the ARPABO code to handle arbitrary N-Grams + + * Establish the internal API needed to do LM conversion +