diff --git a/configure.in b/configure.in index 05ec3c8..7cc61ed 100644 --- a/configure.in +++ b/configure.in @@ -297,6 +297,7 @@ test/unit/test_logmath/Makefile test/unit/test_ngram/Makefile test/unit/test_fsg/Makefile test/unit/test_thread/Makefile +test/unit/test_util/Makefile test/regression/testfuncs.sh test/regression/Makefile ]) diff --git a/include/win32/config.h b/include/win32/config.h index d4b9b01..c7657a4 100644 --- a/include/win32/config.h +++ b/include/win32/config.h @@ -62,3 +62,6 @@ /* We do not have unistd.h. */ #define YY_NO_UNISTD_H 1 + +/* Extension for executables */ +#define EXEEXT ".exe" diff --git a/include/wince/sphinx_config.h b/include/wince/sphinx_config.h index 855eea0..851e421 100644 --- a/include/wince/sphinx_config.h +++ b/include/wince/sphinx_config.h @@ -38,3 +38,6 @@ /* The size of `long', as computed by sizeof. */ #define SIZEOF_LONG 4 + +/* Extension for executables */ +#define EXEEXT ".exe" diff --git a/src/libsphinxbase/lm/ngram_model.c b/src/libsphinxbase/lm/ngram_model.c index 3cedea3..c08020d 100644 --- a/src/libsphinxbase/lm/ngram_model.c +++ b/src/libsphinxbase/lm/ngram_model.c @@ -74,6 +74,14 @@ ngram_file_name_to_type(const char *file_name) return NGRAM_ARPA; /* Default file type */ } } + else if (0 == strcmp_nocase(ext, ".bz2")) { + while (--ext >= file_name) { + if (*ext == '.') break; + } + if (ext < file_name) { + return NGRAM_ARPA; /* Default file type */ + } + } /* We use strncmp because there might be a .gz on the end. */ if (0 == strncmp_nocase(ext, ".ARPA", 5)) return NGRAM_ARPA; diff --git a/src/libsphinxbase/util/pio.c b/src/libsphinxbase/util/pio.c index 9038974..41f72f6 100644 --- a/src/libsphinxbase/util/pio.c +++ b/src/libsphinxbase/util/pio.c @@ -47,8 +47,46 @@ #include "pio.h" #include "err.h" +#include "strfuncs.h" #include "ckd_alloc.h" +#ifndef EXEEXT +#define EXEEXT "" +#endif + +enum { + COMP_NONE, + COMP_COMPRESS, + COMP_GZIP, + COMP_BZIP2 +}; + +static void +guess_comptype(char const *file, int32 *ispipe, int32 *isgz) +{ + int k; + + k = strlen(file); + *ispipe = 0; + *isgz = COMP_NONE; + if ((k > 2) + && ((strcmp(file + k - 2, ".Z") == 0) + || (strcmp(file + k - 2, ".z") == 0))) { + *ispipe = 1; + *isgz = COMP_COMPRESS; + } + else if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) + || (strcmp(file + k - 3, ".GZ") == 0))) { + *ispipe = 1; + *isgz = COMP_GZIP; + } + else if ((k > 4) && ((strcmp(file + k - 4, ".bz2") == 0) + || (strcmp(file + k - 4, ".BZ2") == 0))) { + *ispipe = 1; + *isgz = COMP_BZIP2; + } +} + FILE * fopen_comp(const char *file, const char *mode, int32 * ispipe) { @@ -57,28 +95,8 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe) #ifndef HAVE_POPEN *ispipe = 0; /* No popen() on WinCE */ #else /* HAVE_POPEN */ - int32 k, isgz; - k = strlen(file); -#if defined(WIN32) - *ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0)); - isgz = *ispipe; -#else - *ispipe = 0; - isgz = 0; - if ((k > 2) - && ((strcmp(file + k - 2, ".Z") == 0) - || (strcmp(file + k - 2, ".z") == 0))) { - *ispipe = 1; - } - else { - if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0))) { - *ispipe = 1; - isgz = 1; - } - } -#endif /* NOT WIN32 */ + int32 isgz; + guess_comptype(file, ispipe, &isgz); #endif /* HAVE_POPEN */ if (*ispipe) { @@ -86,55 +104,53 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe) /* Shouldn't get here, anyway */ E_FATAL("No popen() on WinCE\n"); #else - char command[16384]; -#if defined(WIN32) + char *command = NULL; + if (strcmp(mode, "r") == 0) { - sprintf(command, "gzip.exe -d -c %s", file); - if ((fp = _popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode); + switch (isgz) { + case COMP_GZIP: + command = string_join("gunzip" EXEEXT, " -c ", file, NULL); + break; + case COMP_COMPRESS: + command = string_join("zcat" EXEEXT, " ", file, NULL); + break; + case COMP_BZIP2: + command = string_join("bunzip2" EXEEXT, " -c ", file, NULL); + break; + default: + E_FATAL("Unknown compression type %d\n", isgz); + } + if ((fp = popen(command, mode)) == NULL) { + E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); + ckd_free(command); return NULL; } } else if (strcmp(mode, "w") == 0) { - sprintf(command, "gzip.exe > %s", file); - - if ((fp = _popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode); + switch (isgz) { + case COMP_GZIP: + command = string_join("gzip" EXEEXT, " > ", file, NULL); + break; + case COMP_COMPRESS: + command = string_join("compress" EXEEXT, " -c > ", file, NULL); + break; + case COMP_BZIP2: + command = string_join("bzip2" EXEEXT, " > ", file, NULL); + break; + default: + E_FATAL("Unknown compression type %d\n", isgz); + } + if ((fp = popen(command, mode)) == NULL) { + E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); + ckd_free(command); return NULL; } + ckd_free(command); } else { E_ERROR("fopen_comp not implemented for mode = %s\n", mode); return NULL; } -#else - if (strcmp(mode, "r") == 0) { - if (isgz) - sprintf(command, "gunzip -c %s", file); - else - sprintf(command, "zcat %s", file); - - if ((fp = popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); - return NULL; - } - } - else if (strcmp(mode, "w") == 0) { - if (isgz) - sprintf(command, "gzip > %s", file); - else - sprintf(command, "compress -c > %s", file); - - if ((fp = popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); - return NULL; - } - } - else { - E_ERROR("fopen_comp not implemented for mode = %s\n", mode); - return NULL; - } -#endif /* NOT WIN32 */ #endif /* HAVE_POPEN */ } else { @@ -170,64 +186,58 @@ fopen_compchk(const char *file, int32 * ispipe) /* And therefore the rest of this function is useless. */ return (fopen_comp(file, "r", ispipe)); #else /* HAVE_POPEN */ - char tmpfile[16384]; - int32 k, isgz; - struct stat statbuf; + int32 isgz; + FILE *fh; - k = strlen(file); - -#if defined(WIN32) - *ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0)); - isgz = *ispipe; -#else - *ispipe = 0; - isgz = 0; - if ((k > 2) - && ((strcmp(file + k - 2, ".Z") == 0) - || (strcmp(file + k - 2, ".z") == 0))) { - *ispipe = 1; - } + /* First just try to fopen_comp() it */ + if ((fh = fopen_comp(file, "r", ispipe)) != NULL) + return fh; else { - if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0))) { - *ispipe = 1; - isgz = 1; - } - } -#endif + char *tmpfile; + int k; - strcpy(tmpfile, file); - if (stat(tmpfile, &statbuf) != 0) { /* File doesn't exist; try other compressed/uncompressed form, as appropriate */ - E_ERROR_SYSTEM("stat(%s) failed\n", tmpfile); - - if (*ispipe) { - if (isgz) - tmpfile[k - 3] = '\0'; - else - tmpfile[k - 2] = '\0'; - - if (stat(tmpfile, &statbuf) != 0) - return NULL; - } - else { + guess_comptype(file, ispipe, &isgz); + k = strlen(file); + tmpfile = ckd_calloc(k+5, 1); + strcpy(tmpfile, file); + switch (isgz) { + case COMP_GZIP: + tmpfile[k - 3] = '\0'; + break; + case COMP_BZIP2: + tmpfile[k - 4] = '\0'; + break; + case COMP_COMPRESS: + tmpfile[k - 2] = '\0'; + break; + case COMP_NONE: strcpy(tmpfile + k, ".gz"); - if (stat(tmpfile, &statbuf) != 0) { -#if (! WIN32) - strcpy(tmpfile + k, ".Z"); - if (stat(tmpfile, &statbuf) != 0) - return NULL; -#else - return NULL; -#endif + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; } + strcpy(tmpfile + k, ".bz2"); + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; + } + strcpy(tmpfile + k, ".Z"); + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; + } + ckd_free(tmpfile); + return NULL; } - E_WARN("Using %s instead of %s\n", tmpfile, file); + fh = fopen_comp(tmpfile, "r", ispipe); + ckd_free(tmpfile); + return NULL; } - - return (fopen_comp(tmpfile, "r", ispipe)); #endif /* HAVE_POPEN */ } diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 6c97ce9..af317ff 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -19,5 +19,6 @@ SUBDIRS=test_ad \ test_logmath \ test_ngram \ test_fsg \ + test_util \ $(maybe_threads) diff --git a/test/unit/test_hash/test_hash_iter.c b/test/unit/test_hash/test_hash_iter.c index e09c11b..5bc9f5b 100644 --- a/test/unit/test_hash/test_hash_iter.c +++ b/test/unit/test_hash/test_hash_iter.c @@ -1,5 +1,5 @@ /** - * @file test_config.c Test hash tables + * @file test_hash_iter.c Test hash table iterators * @author David Huggins-Daines */ diff --git a/test/unit/test_ngram/100.arpa.bz2 b/test/unit/test_ngram/100.arpa.bz2 new file mode 100644 index 0000000..4451d7e Binary files /dev/null and b/test/unit/test_ngram/100.arpa.bz2 differ diff --git a/test/unit/test_ngram/test_lm_read.c b/test/unit/test_ngram/test_lm_read.c index d4ec6f4..26eb07f 100644 --- a/test/unit/test_ngram/test_lm_read.c +++ b/test/unit/test_ngram/test_lm_read.c @@ -18,7 +18,7 @@ main(int argc, char *argv[]) /* Initialize a logmath object to pass to ngram_read */ lmath = logmath_init(1.0001, 0, 0); /* Read a language model */ - model = ngram_model_read(NULL, LMDIR "/100.arpa.gz", NGRAM_ARPA, lmath); + model = ngram_model_read(NULL, LMDIR "/100.arpa.bz2", NGRAM_ARPA, lmath); TEST_ASSERT(model); TEST_EQUAL(ngram_wid(model, ""), 0); TEST_EQUAL(strcmp(ngram_word(model, 0), ""), 0); diff --git a/test/unit/test_util/Makefile.am b/test/unit/test_util/Makefile.am new file mode 100644 index 0000000..9022892 --- /dev/null +++ b/test/unit/test_util/Makefile.am @@ -0,0 +1,15 @@ +check_PROGRAMS = \ + test_fopen + +TESTS = $(check_PROGRAMS) + +INCLUDES = \ + -I$(top_srcdir)/include \ + -I$(top_builddir)/include \ + -DLMDIR=\"${top_srcdir}/test/unit/test_ngram\" + +LDADD = ${top_builddir}/src/libsphinxbase/libsphinxbase.la + +noinst_HEADERS = test_macros.h + +CLEANFILES = *.log diff --git a/test/unit/test_util/test_fopen.c b/test/unit/test_util/test_fopen.c new file mode 100644 index 0000000..c3b4784 --- /dev/null +++ b/test/unit/test_util/test_fopen.c @@ -0,0 +1,44 @@ +/** + * @file test_fopen.c Test file opening + * @author David Huggins-Daines + */ + +#include "pio.h" +#include "test_macros.h" + +#include +#include + +int +main(int argc, char *argv[]) +{ + FILE *fh; + char line[256], *c; + int ispipe; + + fh = fopen_comp(LMDIR "/100.arpa.gz", "r", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa.gz", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa.bz2", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + return 0; +} diff --git a/test/unit/test_util/test_macros.h b/test/unit/test_util/test_macros.h new file mode 100644 index 0000000..675d2e1 --- /dev/null +++ b/test/unit/test_util/test_macros.h @@ -0,0 +1,10 @@ +#include +#include + +#include "logmath.h" + +#define TEST_ASSERT(x) if (!(x)) { fprintf(stderr, "FAIL: %s\n", #x); exit(1); } +#define TEST_EQUAL(a,b) TEST_ASSERT((a) == (b)) +#define TEST_EQUAL_FLOAT(a,b) TEST_ASSERT(fabs((a) - (b)) < EPSILON) +#define LOG_EPSILON 20 +#define TEST_EQUAL_LOG(a,b) TEST_ASSERT(abs((a) - (b)) < LOG_EPSILON)