From eb14c19ef0125a0cb34e09c7bc9320d09208fc23 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Mon, 9 Nov 2020 16:36:16 -0500 Subject: [PATCH] port to python3 and test --- python/cmusphinx/s3file.py | 8 ++--- python/cmusphinx/s3gau.py | 44 +++++++++++++++------------ python/cmusphinx/s3mdef.py | 52 +++++++++++++++++--------------- python/cmusphinx/s3mixw.py | 22 ++++++++------ python/cmusphinx/s3tmat.py | 24 +++++++++------ python/cmusphinx/test_s3model.py | 19 ++++++------ 6 files changed, 93 insertions(+), 76 deletions(-) diff --git a/python/cmusphinx/s3file.py b/python/cmusphinx/s3file.py index fbbc5ef6..e3cdefb9 100644 --- a/python/cmusphinx/s3file.py +++ b/python/cmusphinx/s3file.py @@ -17,7 +17,7 @@ from struct import unpack, pack from numpy import reshape, shape, frombuffer -class S3File(object): +class S3File: "Read Sphinx-III binary files" def __init__(self, filename=None, mode="rb"): self.fh = None @@ -112,7 +112,7 @@ class S3File(object): self.d1 * self.d2 * self.d3, self.d1, self.d2, self.d3)) spam = self.fh.read(self._nfloats * 4) - params = frombuffer(spam, 'f') + params = frombuffer(spam, 'f').copy() if self.otherend: params = params.byteswap() return reshape(params, (self.d1, self.d2, self.d3)).astype('d') @@ -129,7 +129,7 @@ class S3File(object): self.d1 * self.d2, self.d1, self.d2)) spam = self.fh.read(self._nfloats * 4) - params = frombuffer(spam, 'f') + params = frombuffer(spam, 'f').copy() if self.otherend: params = params.byteswap() return reshape(params, (self.d1, self.d2)).astype('d') @@ -143,7 +143,7 @@ class S3File(object): % (self._nfloats, self.d1)) spam = self.fh.read(self._nfloats * 4) - params = frombuffer(spam, 'f') + params = frombuffer(spam, 'f').copy() if self.otherend: params = params.byteswap() return params.astype('d') diff --git a/python/cmusphinx/s3gau.py b/python/cmusphinx/s3gau.py index af0f5566..d1944e7a 100644 --- a/python/cmusphinx/s3gau.py +++ b/python/cmusphinx/s3gau.py @@ -9,33 +9,36 @@ This module reads and writes the mean and variance parameter files used by SphinxTrain, Sphinx-III, and PocketSphinx. """ -__author__ = "David Huggins-Daines " +__author__ = "David Huggins-Daines " __version__ = "$Revision$" from struct import unpack, pack -from numpy import array, reshape, shape, fromstring -from s3file import S3File, S3File_write +from numpy import reshape, shape, frombuffer +from .s3file import S3File, S3File_write -def open(filename, mode="rb", attr={"version":1.0}): - if mode in ("r", "rb"): - return S3GauFile(filename, mode) - elif mode in ("w", "wb"): - return S3GauFile_write(filename, mode, attr) - else: - raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'" -def open_full(filename, mode="rb", attr={"version":1.0}): +def open(filename, mode="rb", attr={"version": 1.0}): if mode in ("r", "rb"): - return S3FullGauFile(filename, mode) + return S3GauFile(filename) elif mode in ("w", "wb"): - return S3FullGauFile_write(filename, mode, attr) + return S3GauFile_write(filename, attr=attr) else: - raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'" + raise Exception("mode must be 'r', 'rb', 'w', or 'wb'") + + +def open_full(filename, mode="rb", attr={"version": 1.0}): + if mode in ("r", "rb"): + return S3FullGauFile(filename) + elif mode in ("w", "wb"): + return S3FullGauFile_write(filename, attr=attr) + else: + raise Exception("mode must be 'r', 'rb', 'w', or 'wb'") + class S3GauFile(S3File): "Read Sphinx-III format Gaussian parameter files" - def __init__(self, filename, mode): - S3File.__init__(self, filename, mode) + def __init__(self, filename, mode="rb"): + super().__init__(filename=filename, mode=mode) self._load() def readgauheader(self): @@ -62,7 +65,7 @@ class S3GauFile(S3File): self.n_mgau, self.density, self.blk)) # First load everything into a really big Numeric array. spam = self.fh.read(self._nfloats * 4) - data = fromstring(spam, 'f') + data = frombuffer(spam, 'f').copy() if self.otherend: data = data.byteswap() # The on-disk layout is bogus so we have to slice and dice it. @@ -80,6 +83,7 @@ class S3GauFile(S3File): r = rnext self._params = params + class S3FullGauFile(S3GauFile): "Read Sphinx-III format Gaussian full covariance matrix files" def _load(self): @@ -94,7 +98,7 @@ class S3FullGauFile(S3GauFile): # First load everything into a really big Numeric array. # This is inefficient, but in the absence of fromfile()... spam = self.fh.read(self._nfloats * 4) - data = fromstring(spam, 'f') + data = frombuffer(spam, 'f').copy() if self.otherend: data = data.byteswap() # The on-disk layout is bogus so we have to slice and dice it. @@ -114,6 +118,7 @@ class S3FullGauFile(S3GauFile): r = rnext self._params = params + class S3GauFile_write(S3File_write): "Write Sphinx-III format Gaussian parameter files" def writeall(self, stuff): @@ -141,13 +146,14 @@ class S3GauFile_write(S3File_write): for f in m: f.ravel().astype('f').tofile(self.fh) + class S3FullGauFile_write(S3GauFile_write): "Write Sphinx-III format Gaussian full covariance matrix files" def writeall(self, stuff): # This will break for multi-stream files n_mgau, n_feat, density, veclen, veclen2 = shape(stuff) if n_feat != 1: - raise Exception, "Multi-stream files not supported" + raise Exception("Multi-stream files not supported") # Write the header self.fh.seek(self.data_start, 0) self.fh.write(pack("=IIIII", diff --git a/python/cmusphinx/s3mdef.py b/python/cmusphinx/s3mdef.py index 4b8f15ef..4e206817 100644 --- a/python/cmusphinx/s3mdef.py +++ b/python/cmusphinx/s3mdef.py @@ -10,34 +10,37 @@ to senone mapping) files used by SphinxTrain, Sphinx-III, and PocketSphinx. """ -__author__ = "David Huggins-Daines " +__author__ = "David Huggins-Daines " __version__ = "$Revision$" from numpy import ones, empty +import io + def open(file): return S3Mdef(file) + class S3Mdef: "Read Sphinx-III format model definition files" def __init__(self, filename=None): self.info = {} self.phoneset = {} - if filename != None: + if filename is not None: self.read(filename) def read(self, filename): - self.fh = file(filename) + fh = io.open(filename, "r") while True: - version = self.fh.readline().rstrip() + version = fh.readline().rstrip() if not version.startswith("#"): break if version != "0.3": raise Exception("Model definition version %s is not 0.3" % version) info = {} while True: - spam = self.fh.readline().rstrip() + spam = fh.readline().rstrip() if spam.startswith("#"): break val, key = spam.split() @@ -50,9 +53,9 @@ class S3Mdef: self.n_tmat = info['n_tied_tmat'] # Skip field description lines - spam = self.fh.readline().rstrip() - spam = self.fh.readline().rstrip() - + spam = fh.readline() + spam = fh.readline() + ssidmap = {} self.phonemap = {} self.trimap = [] @@ -63,7 +66,7 @@ class S3Mdef: phoneid = 0 self.max_emit_state = 0 while True: - spam = self.fh.readline().rstrip() + spam = fh.readline().rstrip() if spam == "": break fields = spam.split() @@ -84,7 +87,7 @@ class S3Mdef: self.trimap.append((base, lc, rc, wpos)) self.fillermap[phoneid] = (attrib == 'filler') self.tmatmap[phoneid] = int(tmat) - + # Build senone sequence mapping sseq = ",".join(sids) if sseq not in ssidmap: @@ -101,16 +104,17 @@ class S3Mdef: # Fill an array with -1 (which is the ID for non-emitting # states) self.sseq = -1 * ones((len(ssidmap), self.max_emit_state+1), 'i') - + sseqid = 0 self.pidmap = [] - for sseq, phones in ssidmap.iteritems(): - sids = map(int, sseq.split(',')) - self.sseq[sseqid,0:len(sids)] = sids + for sseq, phones in ssidmap.items(): + sids = list(map(int, sseq.split(','))) + self.sseq[sseqid, 0:len(sids)] = sids self.pidmap.append(phones) for p in phones: self.sseqmap[p] = sseqid sseqid = sseqid + 1 + fh.close() def is_ciphone(self, sid): return sid >= 0 and sid < self.n_ci @@ -119,15 +123,15 @@ class S3Mdef: return sid >= 0 and sid < self.n_ci_sen def phone_id(self, ci, lc='-', rc='-', wpos=None): - if wpos == None: + if wpos is None: if lc != '-': # Try all word positions to find one that matches - for new_wpos, pmap in self.phonemap.iteritems(): + for new_wpos, pmap in self.phonemap.items(): if ci in pmap and lc in pmap[ci] and rc in pmap[ci][lc]: wpos = new_wpos break else: - wpos = '-' # context-independent phones have no wpos + wpos = '-' # context-independent phones have no wpos if wpos == '-': # It's context-indepedent so ignore lc, rc return self.phonemap[wpos][ci]['-']['-'] @@ -135,16 +139,16 @@ class S3Mdef: return self.phonemap[wpos][ci][lc][rc] def phone_id_nearest(self, ci, lc='-', rc='-', wpos=None): - if wpos == None or wpos == '-': + if wpos is None or wpos == '-': return self.phone_id(ci, lc, rc, wpos) else: # First try to back off to a different word position - for new_wpos, pmap in self.phonemap.iteritems(): + for new_wpos, pmap in self.phonemap.items(): if ci in pmap and lc in pmap[ci] and rc in pmap[ci][lc]: return self.phonemap[new_wpos][ci][lc][rc] # If not, try using silence in the left/right context if wpos == 'e' and 'SIL' in self.phonemap[wpos][ci][lc]: - return self.phonemap[wpos][ci][lc]['SIL'] + return self.phonemap[wpos][ci][lc]['SIL'] if wpos == 'b' \ and 'SIL' in self.phonemap[wpos][ci] \ and rc in self.phonemap[wpos][ci]['SIL']: @@ -156,15 +160,15 @@ class S3Mdef: return self.trimap[pid] # FIXME: This may be bogus, see def. of sidmap above - def phone_id_from_senone_id(self,sid): + def phone_id_from_senone_id(self, sid): return self.sidmap[sid] - + # FIXME: This may be bogus, see def. of sidmap above def phone_from_senone_id(self, sid): return self.trimap[int(self.sidmap[sid])] # FIXME: This may be bogus, see def. of sidmap above - def ciphone_id_from_senone_id(self,sid): + def ciphone_id_from_senone_id(self, sid): return self.cisidmap[sid] # FIXME: This may be bogus, see def. of sidmap above @@ -172,7 +176,7 @@ class S3Mdef: return self.trimap[int(self.cisidmap[sid])][0] def triphones(self, ci, lc, wpos=None): - if wpos == None: + if wpos is None: out = [] for wpos in self.phonemap: out.extend(self.triphones(ci, lc, wpos)) diff --git a/python/cmusphinx/s3mixw.py b/python/cmusphinx/s3mixw.py index 0ed9ee62..68c8fe15 100644 --- a/python/cmusphinx/s3mixw.py +++ b/python/cmusphinx/s3mixw.py @@ -9,24 +9,26 @@ This module reads and writes the Gaussian mixture weight files used by SphinxTrain, Sphinx-III, and PocketSphinx. """ -__author__ = "David Huggins-Daines " +__author__ = "David Huggins-Daines " __version__ = "$Revision$" -from s3file import S3File, S3File_write +from .s3file import S3File, S3File_write import os + def open(filename, mode="rb"): if mode in ("r", "rb"): - return S3MixwFile(filename, mode) + return S3MixwFile(filename) elif mode in ("w", "wb"): - return S3MixwFile_write(filename, mode) + return S3MixwFile_write(filename) else: - raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'" + raise Exception("mode must be 'r', 'rb', 'w', or 'wb'") + class S3MixwFile(S3File): "Read Sphinx-III format mixture weight files" - def __init__(self, file, mode): - S3File.__init__(self, file, mode) + def __init__(self, filename, mode="rb"): + super().__init__(filename=filename, mode=mode) self._params = self._load() def readgauheader(self): @@ -39,22 +41,24 @@ class S3MixwFile(S3File): self.fh.seek(self.data_start, 0) return self.read3d() + class S3MixwFile_write(S3File_write): "Write Sphinx-III format mixture weight files" def writeall(self, stuff): self.write3d(stuff) + def accumdirs(accumdirs): "Read and accumulate counts from several directories" mixw = None for d in accumdirs: try: submixw = S3MixwFile(os.path.join(d, "mixw_counts"), "rb") - except: + except OSError: submixw = None continue - if mixw == None: + if mixw is None: mixw = submixw else: mixw._params += submixw._params diff --git a/python/cmusphinx/s3tmat.py b/python/cmusphinx/s3tmat.py index feea0591..a4c2c1e9 100644 --- a/python/cmusphinx/s3tmat.py +++ b/python/cmusphinx/s3tmat.py @@ -9,35 +9,38 @@ This module reads and writes the HMM transition matrix files used by SphinxTrain, Sphinx-III, and PocketSphinx. """ -__author__ = "David Huggins-Daines " +__author__ = "David Huggins-Daines " __version__ = "$Revision$" -from s3file import S3File, S3File_write +from .s3file import S3File, S3File_write from numpy import shape -from struct import unpack,pack + def open(filename, mode="rb"): if mode in ("r", "rb"): - return S3TmatFile(filename, mode) + return S3TmatFile(filename) elif mode in ("w", "wb"): - return S3TmatFile_write(filename, mode) + return S3TmatFile_write(filename) else: - raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'" + raise Exception("mode must be 'r', 'rb', 'w', or 'wb'") + class S3TmatFile(S3File): "Read Sphinx-III format transition matrix files" - def __init__(self, file, mode): - S3File.__init__(self, file, mode) + def __init__(self, filename, mode="rb"): + super().__init__(filename=filename, mode=mode) self._params = self._load() def readgauheader(self): if self.fileattr["version"] != "1.0": raise Exception("Version mismatch: must be 1.0 but is " + self.fileattr["version"]) + def _load(self): self.readgauheader() self.fh.seek(self.data_start, 0) - return self.read3d(); + return self.read3d() + class S3TmatFile_write(S3File_write): "Write Sphinx-III format transition matrix files" @@ -45,6 +48,7 @@ class S3TmatFile_write(S3File_write): def writeall(self, stuff): n_tmat, n_state, spam = shape(stuff) if n_state + 1 != spam: - raise Exception("n_state rows %d != n_state columns %d - 1" % n_state, spam) + raise Exception("n_state rows %d != n_state columns %d - 1" + % (n_state, spam)) self.fh.seek(self.data_start, 0) self.write3d(stuff) diff --git a/python/cmusphinx/test_s3model.py b/python/cmusphinx/test_s3model.py index 025054fa..7985a1e3 100644 --- a/python/cmusphinx/test_s3model.py +++ b/python/cmusphinx/test_s3model.py @@ -1,12 +1,11 @@ #!/usr/bin/env python -import hmm import unittest -import s3model -import s2mfc -from feat import _1s_c_d_dd +from cmusphinx import s3model +from cmusphinx import s2mfc +from cmusphinx.feat import _1s_c_d_dd import os -import numpy + class TestS3Model(unittest.TestCase): def setUp(self): @@ -15,10 +14,10 @@ class TestS3Model(unittest.TestCase): self.acmod = s3model.S3Model(self.testdir) def test_created(self): - self.assert_(abs(self.acmod.tmat[0][0,1] - 0.3326) < 0.01) - self.assert_(abs(sum(self.acmod.mixw[0,0]) - 1.0) < 0.01) - self.assert_(abs(self.acmod.var[0][0][0][0] - 0.2583) < 0.01) - self.assert_(abs(self.acmod.var[35][0][0][38] - 16.9266) < 0.01) + self.assertTrue(abs(self.acmod.tmat[0][0,1] - 0.3326) < 0.01) + self.assertTrue(abs(sum(self.acmod.mixw[0,0]) - 1.0) < 0.01) + self.assertTrue(abs(self.acmod.var[0][0][0][0] - 0.2583) < 0.01) + self.assertTrue(abs(self.acmod.var[35][0][0][38] - 16.9266) < 0.01) def test_compute(self): mfcc = s2mfc.open(os.path.join(self.testdir, 'man.ah.111a.mfc')).getall() @@ -28,7 +27,7 @@ class TestS3Model(unittest.TestCase): expected = [3.03518949e-36, 1.00000000e+00, 4.47046728e-16, 1.07179724e-01] senscr = self.acmod.senone_compute(senones, feat[0]) for i,s in enumerate(senones): - self.assert_(abs(senscr[s] - expected[i]) < 0.01) + self.assertTrue(abs(senscr[s] - expected[i]) < 0.01) if __name__ == '__main__': unittest.main()