From f337ea69682e82d7ab33ce5a4e036e0fb6a2ab54 Mon Sep 17 00:00:00 2001 From: dhdfu Date: Mon, 3 May 2010 04:48:55 +0000 Subject: [PATCH] Update and pull things from my bzr repository. Lattice rescoring, with and without FSTs, which may be useful, plus a little script to generate composed class language model FSTs. git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/SphinxTrain@10060 94700074-3cef-4d97-a70e-9c8c206c02f5 --- python/cmusphinx/classlm2fst.py | 24 ++++++++ python/cmusphinx/fstutils.py | 2 - python/cmusphinx/lat_rescore.py | 37 +++++++++++++ python/cmusphinx/lat_rescore_fst.py | 80 +++++++++++++++++++++++++++ python/cmusphinx/lattice_error_fst.py | 3 + 5 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 python/cmusphinx/classlm2fst.py create mode 100644 python/cmusphinx/lat_rescore.py create mode 100644 python/cmusphinx/lat_rescore_fst.py diff --git a/python/cmusphinx/classlm2fst.py b/python/cmusphinx/classlm2fst.py new file mode 100644 index 00000000..7dead191 --- /dev/null +++ b/python/cmusphinx/classlm2fst.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Carnegie Mellon University +# +# You may copy and modify this freely under the same terms as +# Sphinx-III + +""" +FST utility functions +""" + +__author__ = "David Huggins-Daines " +__version__ = "$Revision $" + +import sphinxbase +import fstutils +import openfst +import sys + +if __name__ == '__main__': + lmfile, probdef = sys.argv[1:] + lm = sphinxbase.NGramModel(lmfile) + lmfst = fstutils.build_class_lmfst(lm, probdef, True) + openfst.StdVectorFst(lmfst).Write(lmfile + ".fst") diff --git a/python/cmusphinx/fstutils.py b/python/cmusphinx/fstutils.py index 89e1c796..249945a0 100644 --- a/python/cmusphinx/fstutils.py +++ b/python/cmusphinx/fstutils.py @@ -315,9 +315,7 @@ def build_class_lmfst(lm, probdef, use_phi=False): a VectorFst from it and project it to its input. """ lmfst = build_lmfst(lm, use_phi) - lmfst.Write("lm.fst") classfst = build_classfst(probdef, lmfst.InputSymbols()) - classfst.Write("class.fst") openfst.ArcSortInput(lmfst) openfst.ArcSortInput(classfst) return openfst.StdComposeFst(classfst, lmfst) diff --git a/python/cmusphinx/lat_rescore.py b/python/cmusphinx/lat_rescore.py new file mode 100644 index 00000000..f851d9b6 --- /dev/null +++ b/python/cmusphinx/lat_rescore.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Carnegie Mellon University +# +# You may copy and modify this freely under the same terms as +# Sphinx-III + +""" +Rescore a lattice using a language model directly +""" + +__author__ = "David Huggins-Daines " +__version__ = "$Revision $" + + +import sphinxbase +import lattice +import math +import sys +import os + +def lat_rescore(latfile, lmfst): + """ + Rescore a lattice using a language model. + """ + dag = lattice.Dag(latfile) + end = dag.bestpath(lm) + words = [] + return [lattice.baseword(x.sym) for x in dag.backtrace(end)], end.score + +if __name__ == '__main__': + ctlfile, latdir, lmfile = sys.argv[1:] + lm = sphinxbase.NGramModel(lmfile, wip=1.0, lw=9.5) + for spam in file(ctlfile): + latfile = os.path.join(latdir, spam.strip() + ".lat.gz") + words, score = lat_rescore(latfile, lm) + print " ".join(words), "(%s %f)" % (spam.strip(), score) diff --git a/python/cmusphinx/lat_rescore_fst.py b/python/cmusphinx/lat_rescore_fst.py new file mode 100644 index 00000000..3226bb8a --- /dev/null +++ b/python/cmusphinx/lat_rescore_fst.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Carnegie Mellon University +# +# You may copy and modify this freely under the same terms as +# Sphinx-III + +""" +Rescore a lattice using a language model FST (or a set of them). +""" + +__author__ = "David Huggins-Daines " +__version__ = "$Revision $" + + +import openfst +import lattice +import lat2fsg +import math +import sys +import os + +def lat_rescore(dag, lmfst, lw=9.5): + """ + Rescore a lattice using a language model FST. + """ + fst = lat2fsg.build_lattice_fsg(dag, lmfst.InputSymbols(), 1./lw) + phi = lmfst.InputSymbols().Find("φ") + if phi != -1: + opts = openfst.StdPhiComposeOptions() + opts.matcher1 = openfst.StdPhiMatcher(fst, openfst.MATCH_NONE) + opts.matcher2 = openfst.StdPhiMatcher(lmfst, openfst.MATCH_INPUT, phi) + c = openfst.StdComposeFst(fst, lmfst, opts) + else: + c = openfst.StdComposeFst(fst, lmfst) + o = openfst.StdVectorFst() + openfst.ShortestPath(c, o, 1) + words = [''] + st = o.Start() + score = 0 + while st != -1 and o.NumArcs(st): + a = o.GetArc(st, 0) + if a.olabel != 0: + words.append(lmfst.InputSymbols().Find(a.ilabel)) + score -= a.weight.Value() + st = a.nextstate + return words, score + +if __name__ == '__main__': + from optparse import OptionParser + parser = OptionParser(usage="%prog CTL LATDIR [LMFST]") + parser.add_option("--lmnamectl") + parser.add_option("--lmdir", default=".") + parser.add_option("--lw", type="float", default=7) + opts, args = parser.parse_args(sys.argv[1:]) + ctlfile, latdir = args[0:2] + if len(args) > 2: + lmfst = openfst.StdVectorFst.Read(args[2]) + lmnamectl = None + elif opts.lmnamectl: + lmnamectl = file(opts.lmnamectl) + lmfsts = {} + else: + parser.error("either --lmnamectl or LMFST must be given") + for spam in file(ctlfile): + if lmnamectl: + lmname = lmnamectl.readline().strip() + if lmname not in lmfsts: + lmfsts[lmname] = openfst.StdVectorFst.Read(os.path.join(opts.lmdir, + lmname + ".arpa.fst")) + lmfst = lmfsts[lmname] + try: + dag = lattice.Dag(os.path.join(latdir, spam.strip() + ".lat.gz")) + except IOError: + try: + dag = lattice.Dag(os.path.join(latdir, spam.strip() + ".lat")) + except IOError: + dag = lattice.Dag(htk_file=os.path.join(latdir, spam.strip() + ".slf")) + words, score = lat_rescore(dag, lmfst, opts.lw) + print " ".join(words), "(%s %f)" % (spam.strip(), score) diff --git a/python/cmusphinx/lattice_error_fst.py b/python/cmusphinx/lattice_error_fst.py index ca85ad41..2a90ac42 100644 --- a/python/cmusphinx/lattice_error_fst.py +++ b/python/cmusphinx/lattice_error_fst.py @@ -94,6 +94,7 @@ class CompoundWordModel(openfst.StdVectorFst): if __name__ == '__main__': from optparse import OptionParser parser = OptionParser(usage="%prog CTL REF LATDIR") + parser.add_option("--prune", type="float") opts, args = parser.parse_args(sys.argv[1:]) ctl, ref, latdir = args @@ -119,6 +120,8 @@ if __name__ == '__main__': l = lattice.Dag(os.path.join(latdir, c + ".lat.gz")) except IOError: l = lattice.Dag(htk_file=os.path.join(latdir, c + ".slf")) + if opts.prune != None: + l.posterior_prune(-opts.prune) # Convert it to an FSM lfst = lat2fsg.build_lattice_fsg(l, rfst.OutputSymbols(), addsyms=True, determinize=False,