GitHub/sphinxtrain
1
0
Fork 0
mirror of https://github.com/cmusphinx/sphinxtrain.git synced 2026-06-16 13:14:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

port to python3 and test

Browse Source
This commit is contained in:
David Huggins-Daines
2020-11-09 16:36:16 -05:00
parent 6fcbf2b612
commit eb14c19ef0
6 changed files with 93 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/cmusphinx/s3file.py
+4 -4
View File
@@ -17,7 +17,7 @@ from struct import unpack, pack
from numpy import reshape, shape, frombuffer
class S3File(object):
class S3File:
"Read Sphinx-III binary files"
def __init__(self, filename=None, mode="rb"):
self.fh = None
@@ -112,7 +112,7 @@ class S3File(object):
self.d1 * self.d2 * self.d3,
self.d1, self.d2, self.d3))
spam = self.fh.read(self._nfloats * 4)
params = frombuffer(spam, 'f')
params = frombuffer(spam, 'f').copy()
if self.otherend:
params = params.byteswap()
return reshape(params, (self.d1, self.d2, self.d3)).astype('d')
@@ -129,7 +129,7 @@ class S3File(object):
self.d1 * self.d2,
self.d1, self.d2))
spam = self.fh.read(self._nfloats * 4)
params = frombuffer(spam, 'f')
params = frombuffer(spam, 'f').copy()
if self.otherend:
params = params.byteswap()
return reshape(params, (self.d1, self.d2)).astype('d')
@@ -143,7 +143,7 @@ class S3File(object):
%
(self._nfloats, self.d1))
spam = self.fh.read(self._nfloats * 4)
params = frombuffer(spam, 'f')
params = frombuffer(spam, 'f').copy()
if self.otherend:
params = params.byteswap()
return params.astype('d')
python/cmusphinx/s3gau.py
+25 -19
View File
@@ -9,33 +9,36 @@ This module reads and writes the mean and variance parameter files
used by SphinxTrain, Sphinx-III, and PocketSphinx.
"""
__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
__author__ = "David Huggins-Daines <dhdaines@gmail.com>"
__version__ = "$Revision$"
from struct import unpack, pack
from numpy import array, reshape, shape, fromstring
from s3file import S3File, S3File_write
from numpy import reshape, shape, frombuffer
from .s3file import S3File, S3File_write
def open(filename, mode="rb", attr={"version":1.0}):
if mode in ("r", "rb"):
return S3GauFile(filename, mode)
elif mode in ("w", "wb"):
return S3GauFile_write(filename, mode, attr)
else:
raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'"
def open_full(filename, mode="rb", attr={"version":1.0}):
def open(filename, mode="rb", attr={"version": 1.0}):
if mode in ("r", "rb"):
return S3FullGauFile(filename, mode)
return S3GauFile(filename)
elif mode in ("w", "wb"):
return S3FullGauFile_write(filename, mode, attr)
return S3GauFile_write(filename, attr=attr)
else:
raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'"
raise Exception("mode must be 'r', 'rb', 'w', or 'wb'")
def open_full(filename, mode="rb", attr={"version": 1.0}):
if mode in ("r", "rb"):
return S3FullGauFile(filename)
elif mode in ("w", "wb"):
return S3FullGauFile_write(filename, attr=attr)
else:
raise Exception("mode must be 'r', 'rb', 'w', or 'wb'")
class S3GauFile(S3File):
"Read Sphinx-III format Gaussian parameter files"
def __init__(self, filename, mode):
S3File.__init__(self, filename, mode)
def __init__(self, filename, mode="rb"):
super().__init__(filename=filename, mode=mode)
self._load()
def readgauheader(self):
@@ -62,7 +65,7 @@ class S3GauFile(S3File):
self.n_mgau, self.density, self.blk))
# First load everything into a really big Numeric array.
spam = self.fh.read(self._nfloats * 4)
data = fromstring(spam, 'f')
data = frombuffer(spam, 'f').copy()
if self.otherend:
data = data.byteswap()
# The on-disk layout is bogus so we have to slice and dice it.
@@ -80,6 +83,7 @@ class S3GauFile(S3File):
r = rnext
self._params = params
class S3FullGauFile(S3GauFile):
"Read Sphinx-III format Gaussian full covariance matrix files"
def _load(self):
@@ -94,7 +98,7 @@ class S3FullGauFile(S3GauFile):
# First load everything into a really big Numeric array.
# This is inefficient, but in the absence of fromfile()...
spam = self.fh.read(self._nfloats * 4)
data = fromstring(spam, 'f')
data = frombuffer(spam, 'f').copy()
if self.otherend:
data = data.byteswap()
# The on-disk layout is bogus so we have to slice and dice it.
@@ -114,6 +118,7 @@ class S3FullGauFile(S3GauFile):
r = rnext
self._params = params
class S3GauFile_write(S3File_write):
"Write Sphinx-III format Gaussian parameter files"
def writeall(self, stuff):
@@ -141,13 +146,14 @@ class S3GauFile_write(S3File_write):
for f in m:
f.ravel().astype('f').tofile(self.fh)
class S3FullGauFile_write(S3GauFile_write):
"Write Sphinx-III format Gaussian full covariance matrix files"
def writeall(self, stuff):
# This will break for multi-stream files
n_mgau, n_feat, density, veclen, veclen2 = shape(stuff)
if n_feat != 1:
raise Exception, "Multi-stream files not supported"
raise Exception("Multi-stream files not supported")
# Write the header
self.fh.seek(self.data_start, 0)
self.fh.write(pack("=IIIII",
python/cmusphinx/s3mdef.py
+28 -24
View File
@@ -10,34 +10,37 @@ to senone mapping) files used by SphinxTrain, Sphinx-III, and
PocketSphinx.
"""
__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
__author__ = "David Huggins-Daines <dhdaines@gmail.com>"
__version__ = "$Revision$"
from numpy import ones, empty
import io
def open(file):
return S3Mdef(file)
class S3Mdef:
"Read Sphinx-III format model definition files"
def __init__(self, filename=None):
self.info = {}
self.phoneset = {}
if filename != None:
if filename is not None:
self.read(filename)
def read(self, filename):
self.fh = file(filename)
fh = io.open(filename, "r")
while True:
version = self.fh.readline().rstrip()
version = fh.readline().rstrip()
if not version.startswith("#"):
break
if version != "0.3":
raise Exception("Model definition version %s is not 0.3" % version)
info = {}
while True:
spam = self.fh.readline().rstrip()
spam = fh.readline().rstrip()
if spam.startswith("#"):
break
val, key = spam.split()
@@ -50,9 +53,9 @@ class S3Mdef:
self.n_tmat = info['n_tied_tmat']
# Skip field description lines
spam = self.fh.readline().rstrip()
spam = self.fh.readline().rstrip()
spam = fh.readline()
spam = fh.readline()
ssidmap = {}
self.phonemap = {}
self.trimap = []
@@ -63,7 +66,7 @@ class S3Mdef:
phoneid = 0
self.max_emit_state = 0
while True:
spam = self.fh.readline().rstrip()
spam = fh.readline().rstrip()
if spam == "":
break
fields = spam.split()
@@ -84,7 +87,7 @@ class S3Mdef:
self.trimap.append((base, lc, rc, wpos))
self.fillermap[phoneid] = (attrib == 'filler')
self.tmatmap[phoneid] = int(tmat)
# Build senone sequence mapping
sseq = ",".join(sids)
if sseq not in ssidmap:
@@ -101,16 +104,17 @@ class S3Mdef:
# Fill an array with -1 (which is the ID for non-emitting
# states)
self.sseq = -1 * ones((len(ssidmap), self.max_emit_state+1), 'i')
sseqid = 0
self.pidmap = []
for sseq, phones in ssidmap.iteritems():
sids = map(int, sseq.split(','))
self.sseq[sseqid,0:len(sids)] = sids
for sseq, phones in ssidmap.items():
sids = list(map(int, sseq.split(',')))
self.sseq[sseqid, 0:len(sids)] = sids
self.pidmap.append(phones)
for p in phones:
self.sseqmap[p] = sseqid
sseqid = sseqid + 1
fh.close()
def is_ciphone(self, sid):
return sid >= 0 and sid < self.n_ci
@@ -119,15 +123,15 @@ class S3Mdef:
return sid >= 0 and sid < self.n_ci_sen
def phone_id(self, ci, lc='-', rc='-', wpos=None):
if wpos == None:
if wpos is None:
if lc != '-':
# Try all word positions to find one that matches
for new_wpos, pmap in self.phonemap.iteritems():
for new_wpos, pmap in self.phonemap.items():
if ci in pmap and lc in pmap[ci] and rc in pmap[ci][lc]:
wpos = new_wpos
break
else:
wpos = '-' # context-independent phones have no wpos
wpos = '-' # context-independent phones have no wpos
if wpos == '-':
# It's context-indepedent so ignore lc, rc
return self.phonemap[wpos][ci]['-']['-']
@@ -135,16 +139,16 @@ class S3Mdef:
return self.phonemap[wpos][ci][lc][rc]
def phone_id_nearest(self, ci, lc='-', rc='-', wpos=None):
if wpos == None or wpos == '-':
if wpos is None or wpos == '-':
return self.phone_id(ci, lc, rc, wpos)
else:
# First try to back off to a different word position
for new_wpos, pmap in self.phonemap.iteritems():
for new_wpos, pmap in self.phonemap.items():
if ci in pmap and lc in pmap[ci] and rc in pmap[ci][lc]:
return self.phonemap[new_wpos][ci][lc][rc]
# If not, try using silence in the left/right context
if wpos == 'e' and 'SIL' in self.phonemap[wpos][ci][lc]:
return self.phonemap[wpos][ci][lc]['SIL']
return self.phonemap[wpos][ci][lc]['SIL']
if wpos == 'b' \
and 'SIL' in self.phonemap[wpos][ci] \
and rc in self.phonemap[wpos][ci]['SIL']:
@@ -156,15 +160,15 @@ class S3Mdef:
return self.trimap[pid]
# FIXME: This may be bogus, see def. of sidmap above
def phone_id_from_senone_id(self,sid):
def phone_id_from_senone_id(self, sid):
return self.sidmap[sid]
# FIXME: This may be bogus, see def. of sidmap above
def phone_from_senone_id(self, sid):
return self.trimap[int(self.sidmap[sid])]
# FIXME: This may be bogus, see def. of sidmap above
def ciphone_id_from_senone_id(self,sid):
def ciphone_id_from_senone_id(self, sid):
return self.cisidmap[sid]
# FIXME: This may be bogus, see def. of sidmap above
@@ -172,7 +176,7 @@ class S3Mdef:
return self.trimap[int(self.cisidmap[sid])][0]
def triphones(self, ci, lc, wpos=None):
if wpos == None:
if wpos is None:
out = []
for wpos in self.phonemap:
out.extend(self.triphones(ci, lc, wpos))
python/cmusphinx/s3mixw.py
+13 -9
View File
@@ -9,24 +9,26 @@ This module reads and writes the Gaussian mixture weight files used by
SphinxTrain, Sphinx-III, and PocketSphinx.
"""
__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
__author__ = "David Huggins-Daines <dhdaines@gmail.com>"
__version__ = "$Revision$"
from s3file import S3File, S3File_write
from .s3file import S3File, S3File_write
import os
def open(filename, mode="rb"):
if mode in ("r", "rb"):
return S3MixwFile(filename, mode)
return S3MixwFile(filename)
elif mode in ("w", "wb"):
return S3MixwFile_write(filename, mode)
return S3MixwFile_write(filename)
else:
raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'"
raise Exception("mode must be 'r', 'rb', 'w', or 'wb'")
class S3MixwFile(S3File):
"Read Sphinx-III format mixture weight files"
def __init__(self, file, mode):
S3File.__init__(self, file, mode)
def __init__(self, filename, mode="rb"):
super().__init__(filename=filename, mode=mode)
self._params = self._load()
def readgauheader(self):
@@ -39,22 +41,24 @@ class S3MixwFile(S3File):
self.fh.seek(self.data_start, 0)
return self.read3d()
class S3MixwFile_write(S3File_write):
"Write Sphinx-III format mixture weight files"
def writeall(self, stuff):
self.write3d(stuff)
def accumdirs(accumdirs):
"Read and accumulate counts from several directories"
mixw = None
for d in accumdirs:
try:
submixw = S3MixwFile(os.path.join(d, "mixw_counts"), "rb")
except:
except OSError:
submixw = None
continue
if mixw == None:
if mixw is None:
mixw = submixw
else:
mixw._params += submixw._params
python/cmusphinx/s3tmat.py
+14 -10
View File
@@ -9,35 +9,38 @@ This module reads and writes the HMM transition matrix files used by
SphinxTrain, Sphinx-III, and PocketSphinx.
"""
__author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
__author__ = "David Huggins-Daines <dhdaines@gmail.com>"
__version__ = "$Revision$"
from s3file import S3File, S3File_write
from .s3file import S3File, S3File_write
from numpy import shape
from struct import unpack,pack
def open(filename, mode="rb"):
if mode in ("r", "rb"):
return S3TmatFile(filename, mode)
return S3TmatFile(filename)
elif mode in ("w", "wb"):
return S3TmatFile_write(filename, mode)
return S3TmatFile_write(filename)
else:
raise Exception, "mode must be 'r', 'rb', 'w', or 'wb'"
raise Exception("mode must be 'r', 'rb', 'w', or 'wb'")
class S3TmatFile(S3File):
"Read Sphinx-III format transition matrix files"
def __init__(self, file, mode):
S3File.__init__(self, file, mode)
def __init__(self, filename, mode="rb"):
super().__init__(filename=filename, mode=mode)
self._params = self._load()
def readgauheader(self):
if self.fileattr["version"] != "1.0":
raise Exception("Version mismatch: must be 1.0 but is "
+ self.fileattr["version"])
def _load(self):
self.readgauheader()
self.fh.seek(self.data_start, 0)
return self.read3d();
return self.read3d()
class S3TmatFile_write(S3File_write):
"Write Sphinx-III format transition matrix files"
@@ -45,6 +48,7 @@ class S3TmatFile_write(S3File_write):
def writeall(self, stuff):
n_tmat, n_state, spam = shape(stuff)
if n_state + 1 != spam:
raise Exception("n_state rows %d != n_state columns %d - 1" % n_state, spam)
raise Exception("n_state rows %d != n_state columns %d - 1"
% (n_state, spam))
self.fh.seek(self.data_start, 0)
self.write3d(stuff)
python/cmusphinx/test_s3model.py
+9 -10
View File
@@ -1,12 +1,11 @@
#!/usr/bin/env python
import hmm
import unittest
import s3model
import s2mfc
from feat import _1s_c_d_dd
from cmusphinx import s3model
from cmusphinx import s2mfc
from cmusphinx.feat import _1s_c_d_dd
import os
import numpy
class TestS3Model(unittest.TestCase):
def setUp(self):
@@ -15,10 +14,10 @@ class TestS3Model(unittest.TestCase):
self.acmod = s3model.S3Model(self.testdir)
def test_created(self):
self.assert_(abs(self.acmod.tmat[0][0,1] - 0.3326) < 0.01)
self.assert_(abs(sum(self.acmod.mixw[0,0]) - 1.0) < 0.01)
self.assert_(abs(self.acmod.var[0][0][0][0] - 0.2583) < 0.01)
self.assert_(abs(self.acmod.var[35][0][0][38] - 16.9266) < 0.01)
self.assertTrue(abs(self.acmod.tmat[0][0,1] - 0.3326) < 0.01)
self.assertTrue(abs(sum(self.acmod.mixw[0,0]) - 1.0) < 0.01)
self.assertTrue(abs(self.acmod.var[0][0][0][0] - 0.2583) < 0.01)
self.assertTrue(abs(self.acmod.var[35][0][0][38] - 16.9266) < 0.01)
def test_compute(self):
mfcc = s2mfc.open(os.path.join(self.testdir, 'man.ah.111a.mfc')).getall()
@@ -28,7 +27,7 @@ class TestS3Model(unittest.TestCase):
expected = [3.03518949e-36, 1.00000000e+00, 4.47046728e-16, 1.07179724e-01]
senscr = self.acmod.senone_compute(senones, feat[0])
for i,s in enumerate(senones):
self.assert_(abs(senscr[s] - expected[i]) < 0.01)
self.assertTrue(abs(senscr[s] - expected[i]) < 0.01)
if __name__ == '__main__':
unittest.main()