GitHub/sphinxbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for bzip2 to lm reading

Browse Source 
git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase@9453 94700074-3cef-4d97-a70e-9c8c206c02f5
This commit is contained in:
dhdfu
2009-11-19 01:53:00 +00:00
parent 86715aa57c
commit 5ee428fc26
12 changed files with 205 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configure.in
+1
View File
@@ -297,6 +297,7 @@ test/unit/test_logmath/Makefile
test/unit/test_ngram/Makefile
test/unit/test_fsg/Makefile
test/unit/test_thread/Makefile
test/unit/test_util/Makefile
test/regression/testfuncs.sh
test/regression/Makefile
])
include/win32/config.h
+3
View File
@@ -62,3 +62,6 @@
/* We do not have unistd.h. */
#define YY_NO_UNISTD_H 1
/* Extension for executables */
#define EXEEXT ".exe"
include/wince/sphinx_config.h
+3
View File
@@ -38,3 +38,6 @@
/* The size of `long', as computed by sizeof. */
#define SIZEOF_LONG 4
/* Extension for executables */
#define EXEEXT ".exe"
src/libsphinxbase/lm/ngram_model.c
+8
View File
@@ -74,6 +74,14 @@ ngram_file_name_to_type(const char *file_name)
return NGRAM_ARPA; /* Default file type */
}
}
else if (0 == strcmp_nocase(ext, ".bz2")) {
while (--ext >= file_name) {
if (*ext == '.') break;
}
if (ext < file_name) {
return NGRAM_ARPA; /* Default file type */
}
}
/* We use strncmp because there might be a .gz on the end. */
if (0 == strncmp_nocase(ext, ".ARPA", 5))
return NGRAM_ARPA;
src/libsphinxbase/util/pio.c
+118 -108
View File
@@ -47,8 +47,46 @@
#include "pio.h"
#include "err.h"
#include "strfuncs.h"
#include "ckd_alloc.h"
#ifndef EXEEXT
#define EXEEXT ""
#endif
enum {
COMP_NONE,
COMP_COMPRESS,
COMP_GZIP,
COMP_BZIP2
};
static void
guess_comptype(char const *file, int32 *ispipe, int32 *isgz)
{
int k;
k = strlen(file);
*ispipe = 0;
*isgz = COMP_NONE;
if ((k > 2)
&& ((strcmp(file + k - 2, ".Z") == 0)
|| (strcmp(file + k - 2, ".z") == 0))) {
*ispipe = 1;
*isgz = COMP_COMPRESS;
}
else if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
|| (strcmp(file + k - 3, ".GZ") == 0))) {
*ispipe = 1;
*isgz = COMP_GZIP;
}
else if ((k > 4) && ((strcmp(file + k - 4, ".bz2") == 0)
|| (strcmp(file + k - 4, ".BZ2") == 0))) {
*ispipe = 1;
*isgz = COMP_BZIP2;
}
}
FILE *
fopen_comp(const char *file, const char *mode, int32 * ispipe)
{
@@ -57,28 +95,8 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe)
#ifndef HAVE_POPEN
*ispipe = 0; /* No popen() on WinCE */
#else /* HAVE_POPEN */
int32 k, isgz;
k = strlen(file);
#if defined(WIN32)
*ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
|| (strcmp(file + k - 3, ".GZ") == 0));
isgz = *ispipe;
#else
*ispipe = 0;
isgz = 0;
if ((k > 2)
&& ((strcmp(file + k - 2, ".Z") == 0)
|| (strcmp(file + k - 2, ".z") == 0))) {
*ispipe = 1;
}
else {
if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
|| (strcmp(file + k - 3, ".GZ") == 0))) {
*ispipe = 1;
isgz = 1;
}
}
#endif /* NOT WIN32 */
int32 isgz;
guess_comptype(file, ispipe, &isgz);
#endif /* HAVE_POPEN */
if (*ispipe) {
@@ -86,55 +104,53 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe)
/* Shouldn't get here, anyway */
E_FATAL("No popen() on WinCE\n");
#else
char command[16384];
#if defined(WIN32)
char *command = NULL;
if (strcmp(mode, "r") == 0) {
sprintf(command, "gzip.exe -d -c %s", file);
if ((fp = _popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode);
switch (isgz) {
case COMP_GZIP:
command = string_join("gunzip" EXEEXT, " -c ", file, NULL);
break;
case COMP_COMPRESS:
command = string_join("zcat" EXEEXT, " ", file, NULL);
break;
case COMP_BZIP2:
command = string_join("bunzip2" EXEEXT, " -c ", file, NULL);
break;
default:
E_FATAL("Unknown compression type %d\n", isgz);
}
if ((fp = popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode);
ckd_free(command);
return NULL;
}
}
else if (strcmp(mode, "w") == 0) {
sprintf(command, "gzip.exe > %s", file);
if ((fp = _popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode);
switch (isgz) {
case COMP_GZIP:
command = string_join("gzip" EXEEXT, " > ", file, NULL);
break;
case COMP_COMPRESS:
command = string_join("compress" EXEEXT, " -c > ", file, NULL);
break;
case COMP_BZIP2:
command = string_join("bzip2" EXEEXT, " > ", file, NULL);
break;
default:
E_FATAL("Unknown compression type %d\n", isgz);
}
if ((fp = popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode);
ckd_free(command);
return NULL;
}
ckd_free(command);
}
else {
E_ERROR("fopen_comp not implemented for mode = %s\n", mode);
return NULL;
}
#else
if (strcmp(mode, "r") == 0) {
if (isgz)
sprintf(command, "gunzip -c %s", file);
else
sprintf(command, "zcat %s", file);
if ((fp = popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode);
return NULL;
}
}
else if (strcmp(mode, "w") == 0) {
if (isgz)
sprintf(command, "gzip > %s", file);
else
sprintf(command, "compress -c > %s", file);
if ((fp = popen(command, mode)) == NULL) {
E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode);
return NULL;
}
}
else {
E_ERROR("fopen_comp not implemented for mode = %s\n", mode);
return NULL;
}
#endif /* NOT WIN32 */
#endif /* HAVE_POPEN */
}
else {
@@ -170,64 +186,58 @@ fopen_compchk(const char *file, int32 * ispipe)
/* And therefore the rest of this function is useless. */
return (fopen_comp(file, "r", ispipe));
#else /* HAVE_POPEN */
char tmpfile[16384];
int32 k, isgz;
struct stat statbuf;
int32 isgz;
FILE *fh;
k = strlen(file);
#if defined(WIN32)
*ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
|| (strcmp(file + k - 3, ".GZ") == 0));
isgz = *ispipe;
#else
*ispipe = 0;
isgz = 0;
if ((k > 2)
&& ((strcmp(file + k - 2, ".Z") == 0)
|| (strcmp(file + k - 2, ".z") == 0))) {
*ispipe = 1;
}
/* First just try to fopen_comp() it */
if ((fh = fopen_comp(file, "r", ispipe)) != NULL)
return fh;
else {
if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0)
|| (strcmp(file + k - 3, ".GZ") == 0))) {
*ispipe = 1;
isgz = 1;
}
}
#endif
char *tmpfile;
int k;
strcpy(tmpfile, file);
if (stat(tmpfile, &statbuf) != 0) {
/* File doesn't exist; try other compressed/uncompressed form, as appropriate */
E_ERROR_SYSTEM("stat(%s) failed\n", tmpfile);
if (*ispipe) {
if (isgz)
tmpfile[k - 3] = '\0';
else
tmpfile[k - 2] = '\0';
if (stat(tmpfile, &statbuf) != 0)
return NULL;
}
else {
guess_comptype(file, ispipe, &isgz);
k = strlen(file);
tmpfile = ckd_calloc(k+5, 1);
strcpy(tmpfile, file);
switch (isgz) {
case COMP_GZIP:
tmpfile[k - 3] = '\0';
break;
case COMP_BZIP2:
tmpfile[k - 4] = '\0';
break;
case COMP_COMPRESS:
tmpfile[k - 2] = '\0';
break;
case COMP_NONE:
strcpy(tmpfile + k, ".gz");
if (stat(tmpfile, &statbuf) != 0) {
#if (! WIN32)
strcpy(tmpfile + k, ".Z");
if (stat(tmpfile, &statbuf) != 0)
return NULL;
#else
return NULL;
#endif
if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
E_WARN("Using %s instead of %s\n", tmpfile, file);
ckd_free(tmpfile);
return fh;
}
strcpy(tmpfile + k, ".bz2");
if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
E_WARN("Using %s instead of %s\n", tmpfile, file);
ckd_free(tmpfile);
return fh;
}
strcpy(tmpfile + k, ".Z");
if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) {
E_WARN("Using %s instead of %s\n", tmpfile, file);
ckd_free(tmpfile);
return fh;
}
ckd_free(tmpfile);
return NULL;
}
E_WARN("Using %s instead of %s\n", tmpfile, file);
fh = fopen_comp(tmpfile, "r", ispipe);
ckd_free(tmpfile);
return NULL;
}
return (fopen_comp(tmpfile, "r", ispipe));
#endif /* HAVE_POPEN */
}
test/unit/Makefile.am
+1
View File
@@ -19,5 +19,6 @@ SUBDIRS=test_ad \
test_logmath \
test_ngram \
test_fsg \
test_util \
$(maybe_threads)
test/unit/test_hash/test_hash_iter.c
+1 -1
View File
@@ -1,5 +1,5 @@
/**
* @file test_config.c Test hash tables
* @file test_hash_iter.c Test hash table iterators
* @author David Huggins-Daines <dhuggins@cs.cmu.edu>
*/
test/unit/test_ngram/100.arpa.bz2
BIN
View File
Binary file not shown.
test/unit/test_ngram/test_lm_read.c
+1 -1
View File
@@ -18,7 +18,7 @@ main(int argc, char *argv[])
/* Initialize a logmath object to pass to ngram_read */
lmath = logmath_init(1.0001, 0, 0);
/* Read a language model */
model = ngram_model_read(NULL, LMDIR "/100.arpa.gz", NGRAM_ARPA, lmath);
model = ngram_model_read(NULL, LMDIR "/100.arpa.bz2", NGRAM_ARPA, lmath);
TEST_ASSERT(model);
TEST_EQUAL(ngram_wid(model, "<UNK>"), 0);
TEST_EQUAL(strcmp(ngram_word(model, 0), "<UNK>"), 0);
test/unit/test_util/Makefile.am
+15
View File
@@ -0,0 +1,15 @@
check_PROGRAMS = \
test_fopen
TESTS = $(check_PROGRAMS)
INCLUDES = \
-I$(top_srcdir)/include \
-I$(top_builddir)/include \
-DLMDIR=\"${top_srcdir}/test/unit/test_ngram\"
LDADD = ${top_builddir}/src/libsphinxbase/libsphinxbase.la
noinst_HEADERS = test_macros.h
CLEANFILES = *.log
test/unit/test_util/test_fopen.c
+44
View File
@@ -0,0 +1,44 @@
/**
* @file test_fopen.c Test file opening
* @author David Huggins-Daines <dhuggins@cs.cmu.edu>
*/
#include "pio.h"
#include "test_macros.h"
#include <stdio.h>
#include <stdlib.h>
int
main(int argc, char *argv[])
{
FILE *fh;
char line[256], *c;
int ispipe;
fh = fopen_comp(LMDIR "/100.arpa.gz", "r", &ispipe);
TEST_ASSERT(fh != NULL);
c = fgets(line, sizeof(line), fh);
TEST_EQUAL('#', line[0]);
fclose_comp(fh, ispipe);
fh = fopen_compchk(LMDIR "/100.arpa.gz", &ispipe);
TEST_ASSERT(fh != NULL);
c = fgets(line, sizeof(line), fh);
TEST_EQUAL('#', line[0]);
fclose_comp(fh, ispipe);
fh = fopen_compchk(LMDIR "/100.arpa.bz2", &ispipe);
TEST_ASSERT(fh != NULL);
c = fgets(line, sizeof(line), fh);
TEST_EQUAL('#', line[0]);
fclose_comp(fh, ispipe);
fh = fopen_compchk(LMDIR "/100.arpa", &ispipe);
TEST_ASSERT(fh != NULL);
c = fgets(line, sizeof(line), fh);
TEST_EQUAL('#', line[0]);
fclose_comp(fh, ispipe);
return 0;
}
test/unit/test_util/test_macros.h
+10
View File
@@ -0,0 +1,10 @@
#include <stdio.h>
#include <math.h>
#include "logmath.h"
#define TEST_ASSERT(x) if (!(x)) { fprintf(stderr, "FAIL: %s\n", #x); exit(1); }
#define TEST_EQUAL(a,b) TEST_ASSERT((a) == (b))
#define TEST_EQUAL_FLOAT(a,b) TEST_ASSERT(fabs((a) - (b)) < EPSILON)
#define LOG_EPSILON 20
#define TEST_EQUAL_LOG(a,b) TEST_ASSERT(abs((a) - (b)) < LOG_EPSILON)