Update and pull things from my bzr repository. Lattice rescoring,

with and without FSTs, which may be useful, plus a little script to
generate composed class language model FSTs.


git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/SphinxTrain@10060 94700074-3cef-4d97-a70e-9c8c206c02f5

python/cmusphinx/classlm2fst.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010  Carnegie Mellon University
+#
+# You may copy and modify this freely under the same terms as
+# Sphinx-III
+
+"""
+FST utility functions
+"""
+
+__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
+__version__ = "$Revision $"
+
+import sphinxbase
+import fstutils
+import openfst
+import sys
+
+if __name__ == '__main__':
+    lmfile, probdef = sys.argv[1:]
+    lm = sphinxbase.NGramModel(lmfile)
+    lmfst = fstutils.build_class_lmfst(lm, probdef, True)
+    openfst.StdVectorFst(lmfst).Write(lmfile + ".fst")

python/cmusphinx/fstutils.py

@@ -315,9 +315,7 @@ def build_class_lmfst(lm, probdef, use_phi=False):
     a VectorFst from it and project it to its input.
     """
     lmfst = build_lmfst(lm, use_phi)
-    lmfst.Write("lm.fst")
     classfst = build_classfst(probdef, lmfst.InputSymbols())
-    classfst.Write("class.fst")
     openfst.ArcSortInput(lmfst)
     openfst.ArcSortInput(classfst)
     return openfst.StdComposeFst(classfst, lmfst)

python/cmusphinx/lat_rescore.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010  Carnegie Mellon University
+#
+# You may copy and modify this freely under the same terms as
+# Sphinx-III
+
+"""
+Rescore a lattice using a language model directly
+"""
+
+__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
+__version__ = "$Revision $"
+
+
+import sphinxbase
+import lattice
+import math
+import sys
+import os
+
+def lat_rescore(latfile, lmfst):
+    """
+    Rescore a lattice using a language model.
+    """
+    dag = lattice.Dag(latfile)
+    end = dag.bestpath(lm)
+    words = []
+    return [lattice.baseword(x.sym) for x in dag.backtrace(end)], end.score
+
+if __name__ == '__main__':
+    ctlfile, latdir, lmfile = sys.argv[1:]
+    lm = sphinxbase.NGramModel(lmfile, wip=1.0, lw=9.5)
+    for spam in file(ctlfile):
+        latfile = os.path.join(latdir, spam.strip() + ".lat.gz")
+        words, score = lat_rescore(latfile, lm)
+        print " ".join(words), "(%s %f)" % (spam.strip(), score)

python/cmusphinx/lat_rescore_fst.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010  Carnegie Mellon University
+#
+# You may copy and modify this freely under the same terms as
+# Sphinx-III
+
+"""
+Rescore a lattice using a language model FST (or a set of them).
+"""
+
+__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
+__version__ = "$Revision $"
+
+
+import openfst
+import lattice
+import lat2fsg
+import math
+import sys
+import os
+
+def lat_rescore(dag, lmfst, lw=9.5):
+    """
+    Rescore a lattice using a language model FST.
+    """
+    fst = lat2fsg.build_lattice_fsg(dag, lmfst.InputSymbols(), 1./lw)
+    phi = lmfst.InputSymbols().Find("&phi;")
+    if phi != -1:
+        opts = openfst.StdPhiComposeOptions()
+        opts.matcher1 = openfst.StdPhiMatcher(fst, openfst.MATCH_NONE)
+        opts.matcher2 = openfst.StdPhiMatcher(lmfst, openfst.MATCH_INPUT, phi)
+        c = openfst.StdComposeFst(fst, lmfst, opts)
+    else:
+        c = openfst.StdComposeFst(fst, lmfst)
+    o = openfst.StdVectorFst()
+    openfst.ShortestPath(c, o, 1)
+    words = ['<s>']
+    st = o.Start()
+    score = 0
+    while st != -1 and o.NumArcs(st):
+        a = o.GetArc(st, 0)
+        if a.olabel != 0:
+            words.append(lmfst.InputSymbols().Find(a.ilabel))
+        score -= a.weight.Value()
+        st = a.nextstate
+    return words, score
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser(usage="%prog CTL LATDIR [LMFST]")
+    parser.add_option("--lmnamectl")
+    parser.add_option("--lmdir", default=".")
+    parser.add_option("--lw", type="float", default=7)
+    opts, args = parser.parse_args(sys.argv[1:])
+    ctlfile, latdir = args[0:2]
+    if len(args) > 2:
+        lmfst = openfst.StdVectorFst.Read(args[2])
+        lmnamectl = None
+    elif opts.lmnamectl:
+        lmnamectl = file(opts.lmnamectl)
+        lmfsts = {}
+    else:
+        parser.error("either --lmnamectl or LMFST must be given")
+    for spam in file(ctlfile):
+        if lmnamectl:
+            lmname = lmnamectl.readline().strip()
+            if lmname not in lmfsts:
+                lmfsts[lmname] = openfst.StdVectorFst.Read(os.path.join(opts.lmdir,
+                                                                        lmname + ".arpa.fst"))
+            lmfst = lmfsts[lmname]
+        try:
+            dag = lattice.Dag(os.path.join(latdir, spam.strip() + ".lat.gz"))
+        except IOError:
+            try:
+                dag = lattice.Dag(os.path.join(latdir, spam.strip() + ".lat"))
+            except IOError:
+                dag = lattice.Dag(htk_file=os.path.join(latdir, spam.strip() + ".slf"))
+        words, score = lat_rescore(dag, lmfst, opts.lw)
+        print " ".join(words), "(%s %f)" % (spam.strip(), score)

python/cmusphinx/lattice_error_fst.py

@@ -94,6 +94,7 @@ class CompoundWordModel(openfst.StdVectorFst):
 if __name__ == '__main__':
     from optparse import OptionParser
     parser = OptionParser(usage="%prog CTL REF LATDIR")
+    parser.add_option("--prune", type="float")
     opts, args = parser.parse_args(sys.argv[1:])
     
     ctl, ref, latdir = args
@@ -119,6 +120,8 @@ if __name__ == '__main__':
                 l = lattice.Dag(os.path.join(latdir, c + ".lat.gz"))
             except IOError:
                 l = lattice.Dag(htk_file=os.path.join(latdir, c + ".slf"))
+        if opts.prune != None:
+            l.posterior_prune(-opts.prune)
         # Convert it to an FSM
         lfst = lat2fsg.build_lattice_fsg(l, rfst.OutputSymbols(),
                                          addsyms=True, determinize=False,