Fill in more of the interface (including an intentional misspeling to

annoy future generations), and add a plan.

I'm not convinced that the "virtual function" implementation here is
actually going to be necessary, but it will make things easier for the
time being.


git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase@7462 94700074-3cef-4d97-a70e-9c8c206c02f5

include/ngram_model.h

@@ -106,7 +106,7 @@ void ngram_model_free(ngram_model_t *model);
  * your system does not have iconv, this function may fail.  Also,
  * because all file formats consist of 8-bit character streams,
  * attempting to convert to or from UTF-16 (or any other encoding
- * which contains null bytes) is a recipe for total disaster.
+ * which contains null bytes) is a recipe for total desaster.
  *
  * We have no interest in supporting UTF-16, so don't ask.
  *
@@ -118,12 +118,16 @@ void ngram_model_free(ngram_model_t *model);
 int ngram_model_recode(ngram_model_t *model, const char *from, const char *to);
 
 /**
- * Apply a language weight, insertion penalty, and unigram weight internally.
+ * Irreversibly apply a language weight, insertion penalty, and
+ * unigram weight internally.
  *
  * This will change the values output by ngram_score() and friends.
  * This is done for efficiency since in decoding, these are the only
  * values we actually need.  Call ngram_prob() if you want the "raw"
  * N-Gram probability estimate.
+ *
+ * Note that the unigram probability will still be interpolated in the
+ * output of ngram_prob(), which may be a bug.
  */
 int ngram_apply_weights(ngram_model_t *model,
 			float32 lw, float32 wip, float32 uw);
@@ -140,18 +144,13 @@ int ngram_apply_weights(ngram_model_t *model,
  *  score = ngram_score(model, "joy", "whole", "a", NULL);
  *
  * This is not the function to use in decoding, because it has some
- * overhead for looking up words.  Use ngram_tg_score() or
- * ngram_bg_score() instead.  In the future there will probably be a
- * version that takes a general language model state object, to
- * support suffix-array LM and things like that.
+ * overhead for looking up words.  Use ngram_ng_score(),
+ * ngram_tg_score(), or ngram_bg_score() instead.  In the future there
+ * will probably be a version that takes a general language model
+ * state object, to support suffix-array LM and things like that.
  */
 int32 ngram_score(ngram_model_t *model, const char *word, ...);
 
-/**
- * Explicit va_list version of ngram_score().
- */
-int32 ngram_score_v(ngram_model_t *model, const char *word, va_list history);
-
 /**
  * Quick trigram score lookup.
  */
@@ -162,17 +161,26 @@ int32 ngram_tg_score(ngram_model_t *model, int32 w3, int32 w2, int32 w1);
  */
 int32 ngram_bg_score(ngram_model_t *model, int32 w2, int32 w1);
 
+/**
+ * Quick general N-Gram score lookup.
+ */
+int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
+
 /**
  * Get the "raw" log-probability for a general N-Gram.
  *
- * See documentation for ngram_score() for an explanation of this.
+ * See documentation for ngram_score() and ngram_apply_weights() for
+ * an explanation of this.
  */
 int32 ngram_prob(ngram_model_t *model, const char *word, ...);
 
 /**
- * Explicit va_list version of ngram_prob().
+ * Quick "raw" probability lookup for a general N-Gram.
+ *
+ * See documentation for ngram_ng_score() and ngram_apply_weights()
+ * for an explanation of this.
  */
-int32 ngram_prob_v(ngram_model_t *model, const char *word, va_list history);
+int32 ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
 
 /**
  * Look up numerical word ID.

src/libsphinxbase/lm/ngram_model.c

@@ -287,43 +287,58 @@ int
 ngram_apply_weights(ngram_model_t *model,
 		    float32 lw, float32 wip, float32 uw)
 {
-    return -1;
+    return (*model->funcs->apply_weights)(model, lw, wip, uw);
 }
 
 int32
 ngram_score(ngram_model_t *model, const char *word, ...)
 {
-    return NGRAM_SCORE_ERROR;
-}
+    va_list history;
+    int32 prob;
 
-int32
-ngram_score_v(ngram_model_t *model, const char *word, va_list history)
-{
-    return NGRAM_SCORE_ERROR;
+    va_start(history, word);
+
+    va_end(history);
+
+    return prob;
 }
 
 int32
 ngram_tg_score(ngram_model_t *model, int32 w3, int32 w2, int32 w1)
 {
-    return NGRAM_SCORE_ERROR;
+    int32 hist[2] = { w1, w2 };
+    return (*model->funcs->score)(model, w3, hist, 2);
 }
 
 int32
 ngram_bg_score(ngram_model_t *model, int32 w2, int32 w1)
 {
-    return NGRAM_SCORE_ERROR;
+    return (*model->funcs->score)(model, w2, &w1, 1);
+}
+
+int32
+ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist)
+{
+    return (*model->funcs->score)(model, wid, history, n_hist);
 }
 
 int32
 ngram_prob(ngram_model_t *model, const char *word, ...)
 {
-    return NGRAM_SCORE_ERROR;
+    va_list history;
+    int32 prob;
+
+    va_start(history, word);
+
+    va_end(history);
+
+    return prob;
 }
 
 int32
-ngram_prob_v(ngram_model_t *model, const char *word, va_list history)
+ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist)
 {
-    return NGRAM_SCORE_ERROR;
+    return (*model->funcs->raw_score)(model, wid, history, n_hist);
 }
 
 int32

src/libsphinxbase/lm/ngram_model_arpa.c

@@ -44,6 +44,7 @@
 #include "ngram_model_arpa.h"
 #include "err.h"
 #include "pio.h"
+#include "linklist.h"
 
 #include <string.h>
 
@@ -467,6 +468,9 @@ ngram_model_arpa_read(cmd_ln_t *config,
     base->n_1g_alloc = base->n_counts[0] = n_unigram;
     base->n_counts[1] = n_bigram;
     base->n_counts[2] = n_trigram;
+    base->lw = 1.0;
+    base->wip = 1.0;
+    base->uw = 1.0;
     /* Allocate space for word strings. */
     base->word_str = ckd_calloc(n_unigram, sizeof(char *));
     /* NOTE: They are no longer case-insensitive since we are allowing
@@ -523,6 +527,10 @@ ngram_model_arpa_read(cmd_ln_t *config,
         E_INFO("%8d = #prob3 entries\n", model->n_prob3);
 
         free_sorted_list(&model->sorted_prob3);
+
+        /* Initialize tginfo */
+        model->tginfo =
+            ckd_calloc(base->n_1g_alloc, sizeof(tginfo_t *));
     }
 
     fclose_comp(fp, is_pipe);
@@ -540,9 +548,235 @@ static int
 ngram_model_arpa_apply_weights(ngram_model_t *base, float32 lw,
                               float32 wip, float32 uw)
 {
+    ngram_model_arpa_t *model = (ngram_model_arpa_t *)base;
+    int32 log_wip, log_uw, log_uniform;
+    int i;
+
+    /* Precalculate some log values we will like. */
+    log_wip = logmath_log(base->lmath, wip);
+    log_uw = logmath_log(base->lmath, uw);
+    log_uniform = logmath_log(base->lmath, 1.0 / (base->n_counts[0] - 1))
+        + logmath_log(base->lmath, 1.0 - uw);
+
+    for (i = 0; i < base->n_counts[0]; ++i) {
+        model->unigrams[i].bo_wt1.l *= lw;
+
+        if (strcmp(base->word_str[i], "<s>") == 0) { /* FIXME: configurable start_sym */
+            /* Apply language weight and WIP */
+            model->unigrams[i].prob1.l *= lw;
+            model->unigrams[i].prob1.l += log_wip;
+        }
+        else {
+            /* Interpolate unigram probability with uniform. */
+            model->unigrams[i].prob1.l += log_uw;
+            model->unigrams[i].prob1.l =
+                logmath_add(base->lmath,
+                            model->unigrams[i].prob1.l,
+                            log_uniform);
+            /* Apply language weight and WIP */
+            model->unigrams[i].prob1.l *= lw;
+            model->unigrams[i].prob1.l += log_wip;
+        }
+    }
+
+    for (i = 0; i < model->n_prob2; ++i) {
+        model->prob2[i].l *= lw;
+        model->prob2[i].l += log_wip;
+    }
+
+    if (base->n > 2) {
+        for (i = 0; i < model->n_bo_wt2; ++i) {
+            model->bo_wt2[i].l *= lw;
+        }
+        for (i = 0; i < model->n_prob3; i++) {
+            model->prob3[i].l *= lw;
+            model->prob3[i].l += log_wip;
+        }
+    }
     return 0;
 }
 
+static int32
+lm3g_ug_score(ngram_model_arpa_t *model, int32 lwid)
+{
+    return model->unigrams[lwid].prob1.l;
+}
+
+/* Locate a specific bigram within a bigram list */
+#define BINARY_SEARCH_THRESH	16
+static int32
+find_bg(bigram_t * bg, int32 n, int32 w)
+{
+    int32 i, b, e;
+
+    /* Binary search until segment size < threshold */
+    b = 0;
+    e = n;
+    while (e - b > BINARY_SEARCH_THRESH) {
+        i = (b + e) >> 1;
+        if (bg[i].wid < w)
+            b = i + 1;
+        else if (bg[i].wid > w)
+            e = i;
+        else
+            return i;
+    }
+
+    /* Linear search within narrowed segment */
+    for (i = b; (i < e) && (bg[i].wid != w); i++);
+    return ((i < e) ? i : -1);
+}
+
+static int32
+lm3g_bg_score(ngram_model_arpa_t *model, int32 lw1, int32 lw2)
+{
+    int32 i, n, b, score;
+    bigram_t *bg;
+
+    b = FIRST_BG(model, lw1);
+    n = FIRST_BG(model, lw1 + 1) - b;
+    bg = model->bigrams + b;
+
+    if ((i = find_bg(bg, n, lw2)) >= 0) {
+        score = model->prob2[bg[i].prob2].l;
+    }
+    else {
+        score = model->unigrams[lw1].bo_wt1.l + model->unigrams[lw2].prob1.l;
+    }
+
+    return (score);
+}
+
+static void
+load_tginfo(ngram_model_arpa_t *model, int32 lw1, int32 lw2)
+{
+    int32 i, n, b, t;
+    bigram_t *bg;
+    tginfo_t *tginfo;
+
+    /* First allocate space for tg information for bg lw1,lw2 */
+    tginfo = (tginfo_t *) listelem_alloc(sizeof(tginfo_t));
+    tginfo->w1 = lw1;
+    tginfo->tg = NULL;
+    tginfo->next = model->tginfo[lw2];
+    model->tginfo[lw2] = tginfo;
+
+    /* Locate bigram lw1,lw2 */
+
+    b = model->unigrams[lw1].bigrams;
+    n = model->unigrams[lw1 + 1].bigrams - b;
+    bg = model->bigrams + b;
+
+    if ((n > 0) && ((i = find_bg(bg, n, lw2)) >= 0)) {
+        tginfo->bowt = model->bo_wt2[bg[i].bo_wt2].l;
+
+        /* Find t = Absolute first trigram index for bigram lw1,lw2 */
+        b += i;                 /* b = Absolute index of bigram lw1,lw2 on disk */
+        t = FIRST_TG(model, b);
+
+        tginfo->tg = model->trigrams + t;
+
+        /* Find #tg for bigram w1,w2 */
+        tginfo->n_tg = FIRST_TG(model, b + 1) - t;
+    }
+    else {                      /* No bigram w1,w2 */
+        tginfo->bowt = 0;
+        tginfo->n_tg = 0;
+    }
+}
+
+/* Similar to find_bg */
+static int32
+find_tg(trigram_t * tg, int32 n, int32 w)
+{
+    int32 i, b, e;
+
+    b = 0;
+    e = n;
+    while (e - b > BINARY_SEARCH_THRESH) {
+        i = (b + e) >> 1;
+        if (tg[i].wid < w)
+            b = i + 1;
+        else if (tg[i].wid > w)
+            e = i;
+        else
+            return i;
+    }
+
+    for (i = b; (i < e) && (tg[i].wid != w); i++);
+    return ((i < e) ? i : -1);
+}
+
+static int32
+lm3g_tg_score(ngram_model_arpa_t *model, int32 lw1, int32 lw2, int32 lw3)
+{
+    ngram_model_t *base = &model->base;
+    int32 i, n, score;
+    trigram_t *tg;
+    tginfo_t *tginfo, *prev_tginfo;
+
+    if ((base->n < 3) || (lw1 < 0))
+        return (lm3g_bg_score(model, lw2, lw3));
+
+    prev_tginfo = NULL;
+    for (tginfo = model->tginfo[lw2]; tginfo; tginfo = tginfo->next) {
+        if (tginfo->w1 == lw1)
+            break;
+        prev_tginfo = tginfo;
+    }
+
+    if (!tginfo) {
+        load_tginfo(model, lw1, lw2);
+        tginfo = model->tginfo[lw2];
+    }
+    else if (prev_tginfo) {
+        prev_tginfo->next = tginfo->next;
+        tginfo->next = model->tginfo[lw2];
+        model->tginfo[lw2] = tginfo;
+    }
+
+    tginfo->used = 1;
+
+    /* Trigrams for w1,w2 now pointed to by tginfo */
+    n = tginfo->n_tg;
+    tg = tginfo->tg;
+    if ((i = find_tg(tg, n, lw3)) >= 0) {
+        score = model->prob3[tg[i].prob3].l;
+    }
+    else {
+        score = tginfo->bowt + lm3g_bg_score(model, lw2, lw3);
+    }
+
+    return (score);
+}
+
+static void
+lm3g_cache_reset(ngram_model_arpa_t *model)
+{
+    ngram_model_t *base = &model->base;
+    int32 i;
+    tginfo_t *tginfo, *next_tginfo, *prev_tginfo;
+
+    for (i = 0; i < base->n_counts[0]; i++) {
+        prev_tginfo = NULL;
+        for (tginfo = model->tginfo[i]; tginfo; tginfo = next_tginfo) {
+            next_tginfo = tginfo->next;
+
+            if (!tginfo->used) {
+                listelem_free((void *) tginfo, sizeof(tginfo_t));
+                if (prev_tginfo)
+                    prev_tginfo->next = next_tginfo;
+                else
+                    model->tginfo[i] = next_tginfo;
+            }
+            else {
+                tginfo->used = 0;
+                prev_tginfo = tginfo;
+            }
+        }
+    }
+}
+
 static int32
 ngram_model_arpa_score(ngram_model_t *base, int32 wid,
                       int32 *history, int32 n_hist)
@@ -550,6 +784,13 @@ ngram_model_arpa_score(ngram_model_t *base, int32 wid,
     return NGRAM_SCORE_ERROR;
 }
 
+static int32
+ngram_model_arpa_raw_score(ngram_model_t *base, int32 wid,
+                           int32 *history, int32 n_hist)
+{
+    return NGRAM_SCORE_ERROR;
+}
+
 static void
 ngram_model_arpa_free(ngram_model_t *base)
 {
@@ -561,11 +802,23 @@ ngram_model_arpa_free(ngram_model_t *base)
     ckd_free(model->prob2);
     ckd_free(model->bo_wt2);
     ckd_free(model->prob3);
+    if (model->tginfo) {
+        int32 u;
+        for (u = 0; u < base->n_1g_alloc; u++) {
+            tginfo_t *tginfo, *next_tginfo;
+            for (tginfo = model->tginfo[u]; tginfo; tginfo = next_tginfo) {
+                next_tginfo = tginfo->next;
+                listelem_free(tginfo, sizeof(*tginfo));
+            }
+        }
+        ckd_free(model->tginfo);
+    }
     ckd_free(model->tseg_base);
 }
 
 static ngram_funcs_t ngram_model_arpa_funcs = {
     ngram_model_arpa_apply_weights, /* apply_weights */
     ngram_model_arpa_score,         /* score */
+    ngram_model_arpa_raw_score,     /* raw_score */
     ngram_model_arpa_free           /* free */
 };

src/libsphinxbase/lm/ngram_model_arpa.h

@@ -77,6 +77,7 @@ typedef struct unigram_s {
  */
 #define BG_SEG_SZ	512	/* chosen so that #trigram/segment <= 2**16 */
 #define LOG_BG_SEG_SZ	9
+#define TSEG_BASE(m,b)		((m)->tseg_base[(b)>>LOG_BG_SEG_SZ])
 
 /**
  * Bigram structure.
@@ -108,7 +109,6 @@ typedef struct trigram_s {
  * tree to locate trigrams for a given bigram (lw1,lw2).  The organization is optimized
  * for locality of access (to the same lw1), given lw2.
  */
-#define TSEG_BASE(m,b)		((m)->tseg_base[(b)>>LOG_BG_SEG_SZ])
 typedef struct tginfo_s {
     int32 w1;			/**< lw1 component of bigram lw1,lw2.  All bigrams with
 				   same lw2 linked together (see lm_t.tginfo). */
@@ -165,7 +165,11 @@ typedef struct ngram_model_arpa_s {
     int32 *tseg_base;	/* tseg_base[i>>LOG_BG_SEG_SZ] = index of 1st
 			   trigram for bigram segment (i>>LOG_BG_SEG_SZ) */
 
-    /* Arrays of unique bigram probs and bo-wts, and trigram probs */
+    tginfo_t **tginfo;	/* tginfo[lw2] is head of linked list of trigram information for
+			   some cached subset of bigrams (*,lw2). */
+
+    /* Arrays of unique bigram probs and bo-wts, and trigram probs
+     * (these are temporary, actually) */
     sorted_list_t sorted_prob2;
     sorted_list_t sorted_bo_wt2;
     sorted_list_t sorted_prob3;

src/libsphinxbase/lm/ngram_model_dmp.c

@@ -437,6 +437,13 @@ ngram_model_dmp_score(ngram_model_t *model, int32 wid,
     return NGRAM_SCORE_ERROR;
 }
 
+static int32
+ngram_model_dmp_raw_score(ngram_model_t *model, int32 wid,
+                          int32 *history, int32 n_hist)
+{
+    return NGRAM_SCORE_ERROR;
+}
+
 static void
 ngram_model_dmp_free(ngram_model_t *base)
 {
@@ -472,5 +479,6 @@ ngram_model_dmp_free(ngram_model_t *base)
 static ngram_funcs_t ngram_model_dmp_funcs = {
     ngram_model_dmp_apply_weights, /* apply_weights */
     ngram_model_dmp_score,         /* score */
+    ngram_model_dmp_raw_score,     /* raw_score */
     ngram_model_dmp_free           /* free */
 };

src/libsphinxbase/lm/ngram_model_dmp.h

@@ -46,17 +46,19 @@
 #include "ngram_model_internal.h"
 #include "mmio.h"
 
-/** On-disk representation of language model probabilities. */
+/**
+ * Language model probabilities.
+ */
 typedef union {
     float32 f;
     int32 l;
 } lmprob_t;
 
 /**
- * On-disk representation of unigrams.
+ * Unigram objects.
  */
 typedef struct unigram_s {
-    int32 mapid;        /**< Holds dictionary ID or whatever. */
+    int32 mapid;        /**< UNUSED. */
     lmprob_t prob1;     /**< Unigram probability. */
     lmprob_t bo_wt1;    /**< Unigram backoff weight. */
     int32 bigrams;	/**< Index of 1st entry in lm_t.bigrams[] */

src/libsphinxbase/lm/ngram_model_internal.h

@@ -60,12 +60,16 @@ struct ngram_model_s {
     char **word_str;    /**< Unigram names */
     hash_table_t *wid;  /**< Mapping of unigram names to word IDs. */
     logmath_t *lmath;   /**< Log-math object */
+    float32 lw;         /**< Language model scaling factor */
+    float32 wip;        /**< Word insertion penalty */
+    float32 uw;         /**< Unigram weight */
     struct ngram_funcs_s *funcs; /**< Implementation-specific methods. */
 };
 
 typedef struct ngram_funcs_s {
     int (*apply_weights)(ngram_model_t *model, float32 lw, float32 wip, float32 uw);
     int32 (*score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
+    int32 (*raw_score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
     void (*free)(ngram_model_t *model);
 } ngram_funcs_t;
 

src/libsphinxbase/lm/plan.txt

@@ -0,0 +1,9 @@
+Implementation plan:
+
+ * Get ARPABO and DMP stuff up and going, with a battery of appropriate
+   tests and regression test against PocketSphinx/Sphinx3
+
+ * Generalize the ARPABO code to handle arbitrary N-Grams
+
+ * Establish the internal API needed to do LM conversion
+