From 5ee428fc26fcc5447ee41ecf4e8fbbe9d45742bb Mon Sep 17 00:00:00 2001 From: dhdfu Date: Thu, 19 Nov 2009 01:53:00 +0000 Subject: [PATCH] add support for bzip2 to lm reading git-svn-id: svn+ssh://svn.code.sf.net/p/cmusphinx/code/trunk/sphinxbase@9453 94700074-3cef-4d97-a70e-9c8c206c02f5 --- configure.in | 1 + include/win32/config.h | 3 + include/wince/sphinx_config.h | 3 + src/libsphinxbase/lm/ngram_model.c | 8 + src/libsphinxbase/util/pio.c | 226 ++++++++++++++------------- test/unit/Makefile.am | 1 + test/unit/test_hash/test_hash_iter.c | 2 +- test/unit/test_ngram/100.arpa.bz2 | Bin 0 -> 14653 bytes test/unit/test_ngram/test_lm_read.c | 2 +- test/unit/test_util/Makefile.am | 15 ++ test/unit/test_util/test_fopen.c | 44 ++++++ test/unit/test_util/test_macros.h | 10 ++ 12 files changed, 205 insertions(+), 110 deletions(-) create mode 100644 test/unit/test_ngram/100.arpa.bz2 create mode 100644 test/unit/test_util/Makefile.am create mode 100644 test/unit/test_util/test_fopen.c create mode 100644 test/unit/test_util/test_macros.h diff --git a/configure.in b/configure.in index 05ec3c8..7cc61ed 100644 --- a/configure.in +++ b/configure.in @@ -297,6 +297,7 @@ test/unit/test_logmath/Makefile test/unit/test_ngram/Makefile test/unit/test_fsg/Makefile test/unit/test_thread/Makefile +test/unit/test_util/Makefile test/regression/testfuncs.sh test/regression/Makefile ]) diff --git a/include/win32/config.h b/include/win32/config.h index d4b9b01..c7657a4 100644 --- a/include/win32/config.h +++ b/include/win32/config.h @@ -62,3 +62,6 @@ /* We do not have unistd.h. */ #define YY_NO_UNISTD_H 1 + +/* Extension for executables */ +#define EXEEXT ".exe" diff --git a/include/wince/sphinx_config.h b/include/wince/sphinx_config.h index 855eea0..851e421 100644 --- a/include/wince/sphinx_config.h +++ b/include/wince/sphinx_config.h @@ -38,3 +38,6 @@ /* The size of `long', as computed by sizeof. */ #define SIZEOF_LONG 4 + +/* Extension for executables */ +#define EXEEXT ".exe" diff --git a/src/libsphinxbase/lm/ngram_model.c b/src/libsphinxbase/lm/ngram_model.c index 3cedea3..c08020d 100644 --- a/src/libsphinxbase/lm/ngram_model.c +++ b/src/libsphinxbase/lm/ngram_model.c @@ -74,6 +74,14 @@ ngram_file_name_to_type(const char *file_name) return NGRAM_ARPA; /* Default file type */ } } + else if (0 == strcmp_nocase(ext, ".bz2")) { + while (--ext >= file_name) { + if (*ext == '.') break; + } + if (ext < file_name) { + return NGRAM_ARPA; /* Default file type */ + } + } /* We use strncmp because there might be a .gz on the end. */ if (0 == strncmp_nocase(ext, ".ARPA", 5)) return NGRAM_ARPA; diff --git a/src/libsphinxbase/util/pio.c b/src/libsphinxbase/util/pio.c index 9038974..41f72f6 100644 --- a/src/libsphinxbase/util/pio.c +++ b/src/libsphinxbase/util/pio.c @@ -47,8 +47,46 @@ #include "pio.h" #include "err.h" +#include "strfuncs.h" #include "ckd_alloc.h" +#ifndef EXEEXT +#define EXEEXT "" +#endif + +enum { + COMP_NONE, + COMP_COMPRESS, + COMP_GZIP, + COMP_BZIP2 +}; + +static void +guess_comptype(char const *file, int32 *ispipe, int32 *isgz) +{ + int k; + + k = strlen(file); + *ispipe = 0; + *isgz = COMP_NONE; + if ((k > 2) + && ((strcmp(file + k - 2, ".Z") == 0) + || (strcmp(file + k - 2, ".z") == 0))) { + *ispipe = 1; + *isgz = COMP_COMPRESS; + } + else if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) + || (strcmp(file + k - 3, ".GZ") == 0))) { + *ispipe = 1; + *isgz = COMP_GZIP; + } + else if ((k > 4) && ((strcmp(file + k - 4, ".bz2") == 0) + || (strcmp(file + k - 4, ".BZ2") == 0))) { + *ispipe = 1; + *isgz = COMP_BZIP2; + } +} + FILE * fopen_comp(const char *file, const char *mode, int32 * ispipe) { @@ -57,28 +95,8 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe) #ifndef HAVE_POPEN *ispipe = 0; /* No popen() on WinCE */ #else /* HAVE_POPEN */ - int32 k, isgz; - k = strlen(file); -#if defined(WIN32) - *ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0)); - isgz = *ispipe; -#else - *ispipe = 0; - isgz = 0; - if ((k > 2) - && ((strcmp(file + k - 2, ".Z") == 0) - || (strcmp(file + k - 2, ".z") == 0))) { - *ispipe = 1; - } - else { - if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0))) { - *ispipe = 1; - isgz = 1; - } - } -#endif /* NOT WIN32 */ + int32 isgz; + guess_comptype(file, ispipe, &isgz); #endif /* HAVE_POPEN */ if (*ispipe) { @@ -86,55 +104,53 @@ fopen_comp(const char *file, const char *mode, int32 * ispipe) /* Shouldn't get here, anyway */ E_FATAL("No popen() on WinCE\n"); #else - char command[16384]; -#if defined(WIN32) + char *command = NULL; + if (strcmp(mode, "r") == 0) { - sprintf(command, "gzip.exe -d -c %s", file); - if ((fp = _popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode); + switch (isgz) { + case COMP_GZIP: + command = string_join("gunzip" EXEEXT, " -c ", file, NULL); + break; + case COMP_COMPRESS: + command = string_join("zcat" EXEEXT, " ", file, NULL); + break; + case COMP_BZIP2: + command = string_join("bunzip2" EXEEXT, " -c ", file, NULL); + break; + default: + E_FATAL("Unknown compression type %d\n", isgz); + } + if ((fp = popen(command, mode)) == NULL) { + E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); + ckd_free(command); return NULL; } } else if (strcmp(mode, "w") == 0) { - sprintf(command, "gzip.exe > %s", file); - - if ((fp = _popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("_popen (%s,%s) failed\n", command, mode); + switch (isgz) { + case COMP_GZIP: + command = string_join("gzip" EXEEXT, " > ", file, NULL); + break; + case COMP_COMPRESS: + command = string_join("compress" EXEEXT, " -c > ", file, NULL); + break; + case COMP_BZIP2: + command = string_join("bzip2" EXEEXT, " > ", file, NULL); + break; + default: + E_FATAL("Unknown compression type %d\n", isgz); + } + if ((fp = popen(command, mode)) == NULL) { + E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); + ckd_free(command); return NULL; } + ckd_free(command); } else { E_ERROR("fopen_comp not implemented for mode = %s\n", mode); return NULL; } -#else - if (strcmp(mode, "r") == 0) { - if (isgz) - sprintf(command, "gunzip -c %s", file); - else - sprintf(command, "zcat %s", file); - - if ((fp = popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); - return NULL; - } - } - else if (strcmp(mode, "w") == 0) { - if (isgz) - sprintf(command, "gzip > %s", file); - else - sprintf(command, "compress -c > %s", file); - - if ((fp = popen(command, mode)) == NULL) { - E_ERROR_SYSTEM("popen (%s,%s) failed\n", command, mode); - return NULL; - } - } - else { - E_ERROR("fopen_comp not implemented for mode = %s\n", mode); - return NULL; - } -#endif /* NOT WIN32 */ #endif /* HAVE_POPEN */ } else { @@ -170,64 +186,58 @@ fopen_compchk(const char *file, int32 * ispipe) /* And therefore the rest of this function is useless. */ return (fopen_comp(file, "r", ispipe)); #else /* HAVE_POPEN */ - char tmpfile[16384]; - int32 k, isgz; - struct stat statbuf; + int32 isgz; + FILE *fh; - k = strlen(file); - -#if defined(WIN32) - *ispipe = (k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0)); - isgz = *ispipe; -#else - *ispipe = 0; - isgz = 0; - if ((k > 2) - && ((strcmp(file + k - 2, ".Z") == 0) - || (strcmp(file + k - 2, ".z") == 0))) { - *ispipe = 1; - } + /* First just try to fopen_comp() it */ + if ((fh = fopen_comp(file, "r", ispipe)) != NULL) + return fh; else { - if ((k > 3) && ((strcmp(file + k - 3, ".gz") == 0) - || (strcmp(file + k - 3, ".GZ") == 0))) { - *ispipe = 1; - isgz = 1; - } - } -#endif + char *tmpfile; + int k; - strcpy(tmpfile, file); - if (stat(tmpfile, &statbuf) != 0) { /* File doesn't exist; try other compressed/uncompressed form, as appropriate */ - E_ERROR_SYSTEM("stat(%s) failed\n", tmpfile); - - if (*ispipe) { - if (isgz) - tmpfile[k - 3] = '\0'; - else - tmpfile[k - 2] = '\0'; - - if (stat(tmpfile, &statbuf) != 0) - return NULL; - } - else { + guess_comptype(file, ispipe, &isgz); + k = strlen(file); + tmpfile = ckd_calloc(k+5, 1); + strcpy(tmpfile, file); + switch (isgz) { + case COMP_GZIP: + tmpfile[k - 3] = '\0'; + break; + case COMP_BZIP2: + tmpfile[k - 4] = '\0'; + break; + case COMP_COMPRESS: + tmpfile[k - 2] = '\0'; + break; + case COMP_NONE: strcpy(tmpfile + k, ".gz"); - if (stat(tmpfile, &statbuf) != 0) { -#if (! WIN32) - strcpy(tmpfile + k, ".Z"); - if (stat(tmpfile, &statbuf) != 0) - return NULL; -#else - return NULL; -#endif + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; } + strcpy(tmpfile + k, ".bz2"); + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; + } + strcpy(tmpfile + k, ".Z"); + if ((fh = fopen_comp(tmpfile, "r", ispipe)) != NULL) { + E_WARN("Using %s instead of %s\n", tmpfile, file); + ckd_free(tmpfile); + return fh; + } + ckd_free(tmpfile); + return NULL; } - E_WARN("Using %s instead of %s\n", tmpfile, file); + fh = fopen_comp(tmpfile, "r", ispipe); + ckd_free(tmpfile); + return NULL; } - - return (fopen_comp(tmpfile, "r", ispipe)); #endif /* HAVE_POPEN */ } diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 6c97ce9..af317ff 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -19,5 +19,6 @@ SUBDIRS=test_ad \ test_logmath \ test_ngram \ test_fsg \ + test_util \ $(maybe_threads) diff --git a/test/unit/test_hash/test_hash_iter.c b/test/unit/test_hash/test_hash_iter.c index e09c11b..5bc9f5b 100644 --- a/test/unit/test_hash/test_hash_iter.c +++ b/test/unit/test_hash/test_hash_iter.c @@ -1,5 +1,5 @@ /** - * @file test_config.c Test hash tables + * @file test_hash_iter.c Test hash table iterators * @author David Huggins-Daines */ diff --git a/test/unit/test_ngram/100.arpa.bz2 b/test/unit/test_ngram/100.arpa.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..4451d7e7bc4a3cbef83f16621f40bf8cd0598e68 GIT binary patch literal 14653 zcmV-DIl{(5T4*^jL0KkKS??#(wE!4Ie}G{80svV5|Mx!c-^9QF|Mg%}`(U)}V4*+& z;Xo?10av-px4xKr*gIAZfXSd43Iz&LN+cUn_$2O*miEBEwUk)*@i9mz8`u%RVw$rUXNBORS*?>&B#RB z0s;g~iIFMf(LYHKC};ox000`EAS94fX{69(Xa)#k8X5oq0ASMz38g(rsp*i_Jf@i$ zFq!}cKx6=D0A`5@P?bMSrlU<77(tT(4^U{x!3;(wAc+t{CYntNqMp)eCe+zRhCN0o z#2A6K4H|lZ2}pt>o<%(xqtrC@4eEnU01SWt00HCZ^sd1}_V7{=B2S-kEcl>obc@mU8i{+Ld1FH(i5$*6%PO{CfEk&_?vcyM;d z``M+9_n-7rUJ@bFjKrqaWldQDv9|jyTGCKTNG?BD{iSN~f1acM2}c)3(^WHX_IBC# znOhI{*`lx7d$)tpEbV|+%IK37BoF9StP0A7Oxykrq5+dZp`i#uw}6@2(KQDUnbN#r z(YJm7PSsoj{AFzY^`_0HaD*91>nvR-@jhPFDKtKen6J%O0cxo0M&;@rVqV(}OSko= z1!9wVeQ;yfhQ8m=LCODr(s!);O|g(Is0onp-ko8&7`- z+f636wnd?q?Q;;9QN|0_@G($}xg31D+~M+cb*vVF!erb;T8y&rOe2KP7OG>2&iL9C z1qk?b0EKSm0p{K96ob%6wZc^etK~~8gQuQ6h{g?@aZf8n1p?i{@NvQ%=NyLiOl$2o zc%tdW1t8k)RH$>KgOijrvKhYHX+yxNgk!EMXN#vUrnIcnrtD26D)A`ChT?Vut;pON z8?`$LGKo55_k*242^Nil%TK5wb=rv0PIPg}u9-Mpik-QD*hf=v87itg*bH#BY3!Xc zUh!~DWs+>5mKwDePLP`I)RLo6*b3_dSQ4)Au~!@%00BEMoIS!$5pOR!a$=LILXO^n zF?Ox1$7Uxe&o{CsBc}&nMZmR{*%7uF#w#Lm2{&;hx)3Gnq~!fCjq5^gA4?4Hzc-e(F)$EJGl0>M(j}7Hhj)13XQiKy zOjDe#ns+v`CapNkwV}~=gecW7NNBUB38=Df4IxqkP4UT*ey&LWRXZxO(__h#A0>Y5 zYF$(9B8n9~diy;6-o^9#3ldB8`TY^T!ylqa{}Z*J?Fm1RFxa=@4f%d}Ob^R-tj496 zfEKUyl~c3>h9uq_@VMY^Q6|S&TEFDbXC4@!A!|(8wvbJ&lWWxiGblUPT~I*HKMmG( zeLbPlE64I%QRF467Nq{9cNSj?Q(Uw?n32+`SDR+Ia_ku+Liuz+6}zQ%SqCOis_nL7 z<@s-!y9=K9x<)d8u=4+r;OOx3wdn)1KGp_})<>1~OItW3e!1sr>tjcj-gRebLaLR^ zGM|v)LPf1-#*KaE%yJcrLjz^T!f|f!}9>&Lj2=OqlEvB=BCb3tm2CMA_ zk5@x*w%rMo#*W)H`^GY}#~`qxDrAIwuu?bdj}>R*ZTG3kM*?bi)of-q5c$MiN#_vv z<~lN+Pk0rs)8DGh6h4fuc2hB}gOUt8qP!#^ld+R-3F;Ldu zr}5y;>tk=;Zq4}D1%AnhT&*U|S(}>nHh$MlxP2*Q-r}q4n0o1YU<1v3UnWi9%3cQ} zKC;8-TGKa2ltNfhlq)AH4q_~aI#CcHLU7!$8kCc;B{VI%yxpKqgMpE?=OPIa0va+V z=VTR^i$lhRkS$SR4qcQfJnp!yg>1%lJ5g`>Elsb+abce!fbyYBb>(KK zAmE1{J(n1n-H9yCUL4HS)?eAs>w=g8y51kHu9Cp(sG}+tt$KLap2k)eD=4-&7!%6I z@^TgA>&tGgw@SQNqF6LG44I_Tcg$|io{MsCYZAOyEXy;t3r}Y`Gk^dvPBmT-bwL#= zY#iR#l*I1GYaM$N46wpJO1{9j*{3@`Feo5##MMaLX!J)WNT*Z=yYYlKWPx;3g}*)n zbAW!o$@YHm``+*X4!%pzzw&n1!`MM5E3bfk-bMijdq4mfg)Z=af0M73TRA(sI@%n= zfS_*F{19HHyB?$D&|}!sorB6u>$<()feo3%&$H^?;5=JI-1hKRy)(r}zZq%1)XON4ve(C#iM&VNjg1uD_r2@Xs6k zmeREj7`vU*cC%kGXL%zJxadb6SqPzlzt9sy9GLVNLSV2yFN>R=r9wqhH$*eca~MoV z$?zPk@Q7IZG9>%9H@kiNNbpFkvPT-GNSAq1$pdACh=B&w**AkIH>TGX0ao9)+9`Y${SIvUyyXYgN z{XFqCuCYKGDm*|gFc$_0F=;5@!0G#YvlhRb_@~AaeErk~0sC>uCO7le{NtIguYA@G z+;w44)-tM^p)AAbjdTC6kG(GF0FK(tSr1c?qz3VohHtsD?&kG_oJKg<=P{G@G9cq^scQGy?uNqnL#0uP@`%5E?iwU`YRRv;lA6FIa{MXY37&XzdlpYqsjv}pM+i&yjN9A zgDK!5vON%R=td*!&tZ7SH1v7$Xt#*6qYOs3|y;8w) z_$WbWO{|Hlq+ae#d@C6-xtKG-YiWZhsmVpGrZB~7hF9BV-gWZ-58OU?^2HE@BnA;6 zK}Zr|M!%EneHs7(n_j7qUD{s7rWCw662eI)l1ZeLvPmS8OC^z=ci5H28j?ej**PVI zP9+z!DnzX&lUreC$XVX;FBayKh~1*7006ymW^$+p*RM~&p#>w3|H4Xr=giW+s^@g@ z&Tr~IWy^b!8JU>l$(M%l>hY2?^puz{1a(~T_U+`&aAn*VNmYP_4G|_)NQJg+GT=aH z0|A-SVu2L{AShx1G6MmS5)m^AC9icOBg&cXl;*zsQa$aZBPzlGX-P!*q9R&|m-|Sp zfL{C3sQ2fm`t|<5>qZEghc$V4B#I(PL}3<1k=4(3{?8jN;`cC!TWNqm;06ldFy4?w zZOauSGfN;HEiG7r^}z!#KYxU)5;D89sj?*bk%a<4a3BhRqhbRW4X;{qliu9`4p{(D zFUQuBqus)&01Y48V={F>_3hOQZUXE+JengLGoeM;RX_lJ_D84=90ZW600P03Er8>i zc0d7-8~_WJR8{WRBYXU$AVXI0$c_U#V;EXllZqlXs7!zi--i3}6e_G`M8H>s&5CG` z?FG>pX6OI`06;kVw%gnjy|&Xpu)saJi8Pu}A6gHrq?N2O+;#7gd$Q8>-n2=+8dKNe z2{Lx(5>Ku)l2MNmGF3r%9@gnfDk%>8|9jPOu#2~0RTW01p_ma zGCxa@?*j8Rrqk=b^(0dDEo`Kc>-{26sA~KAyvoerww3!nV{8V%1HB~+-jo15g^(Bu zkO=+hSUgeP;)T71m1g1U9fLmjGbL04`eQORbFDK8=7@>n zh=j@8aa`otk(}J#GO;tSdYsq|S43x}Ko8cNBkxAKT8Nu&q-J9b`pKE4kON9`@4ySt z^;X|~YfrkHeA92wO}}!Rd8XcJzRwBd5*WCdCG62!JWpPE$@BpO=XDUlSOV&>T-^$MyQ?$0mLvfp?iK)$Bnp54 z6;2B%Qbu@$7XWQ)BThZ&XLUdj4fdtZ$^e)v5uBbBNZ7r+r;=7x4-&~J5BxWR< zUH*?CPUA@JOC$PGvy6P0%oq&)hl*OsdLlbfkO6K8k)^fiY8VD~sR09jXlbve`RlQF zK1-4TVI5u$J8xt(bP5*r454fT0dNotfmpgJSh|{JxwZNFVGU7Iu)rfE6hd3(hw>hT zXN#cZ0GeBNH2ebZ*K>2R8BX7fGsmOhtsBk7lf|xhTNtsqu=#i#tH(sZs%0idde9OH zOt58T4S-5%a0L1xT_mo1V{Mlc=$QrHhG(j;1pvG1le5^1!~mg=otZO4Z^E1Gmr`+& z?Q$?QganYd>9P-lc5e?5q=t|n^JkTv&g4fL1Sa4>6#)PYh`=tkwpH#@B|tzED8=(s z&Nj|7v|*x~p-#BKeU-_b)x)cjDA*A?i+E>y zS>)SYshNLb_Tt27hEKWxfGCFo7tqZ8GS&J#29f?SunTD5MTkt5O z)uf``sh-+x=AXqAC;=xN4O1{dn~S3oODqyieI1E>fYP~`i)3iZ^Mto8$xrui{OX4!2dVK!cF;$Q@X zlCsxh-(<0|nc)z(z+!O5NDgL0%(wsv0k!ZYuvZaK1<{JbU>*Q~H}LLY4W&b{9HCdB zMhkX86(q8^>PRTe!ceJi*5pUNU?b?(re1<#8kzU9NRh02+~T)8x5y(Aq86Ki? zkqICQ>bLL=_J|`oG=LBb^-HAjM&jO-Mk%0U)hky@OPuyuKCeoJnGC#I5Q`%OIJ*Kv zk7~EW>Y5*#W)k#7iqW9O8MauWt=WCd0mC2)V`<)%X{BxOMpeQG^L_4M7!d8#5t*vm z)GXK{cj0J|f3HpD`lC7Gj`_~6U7qK*NdY@jWbsa(*|CwGl@Yxbd(lxlQ4>0$Hb4hjJN9=~w2}!VlEO@9 zXT+qD$k`r7<~WXk)B!*M01)*{rF+q!`_bJzQ4-N}GUlioo%l1Mk=GzfeSJ*NRX`VM zGc{BI_N7rZL`}Rycs7PcVpd=fcDtibUval^Xo&m(7zGgE7R#6}R0UAPhD>t6pW#Y& zrthm!oaT=D`?5w}l7Xk%l8IVvx^JH^BSegCtBGHuv=Jjpcu;L5d9AnA5j}fTJ4UGy zeyGrIM{=F`*LCNIFNNVenelXc2B@1{5?bqfpb`WD06-N1Km%(9U?2w%fG`7xPVjG@ z%B6#18udun+0>sLI`m|Bxix`H#AJvdjXfTG&XF#1=GUR)tnaT|*0|oMzJ#&!>)XnX zUJ;8=%?(E(i*a5+(1`+}xdJgBrM;sBFHU8Xp?S$~buTBV5(a<*+l6p5#rojY-;OuI zNm?S{b^@XRKnA)aj#iwbqybVFB?^1fbZ96eF8M$Z4=ZCcw4euK*Uh`%v8N?_RT`-E zkySAb<6<+VYlJG}RvE|u2hNEwuAulNK4(u z2msbl6h6NHrzQb#$^wAhY*#K1l0E#I8WRI_V|T4!2m#dj=rdFV0@k;T61$|)0RhMg z0KRF$B2f`Iq9QF3HPh?>5oHd*I&Nlas0J6Wk$^JFsMLicC$_Q2JOVcwUEFt`{p5ns zAd-wyOTtPw*dW!o{ovo)T$z`yeNd61bB~v+H*f zIE^m}Bdl6&L`(7Ke}q7R-2i4p%BI@++>G%k0qsacF6Yhs`SDwmYYDwy%C%9HtPiE))TfttrNXEz6HI#@lLImE89FbImZV#z@Pnpk|DbnyLVRdP<(#01+>6 z$qWO5ZrVJV09a%JMZg-pD*da2Gj%{UqU4N@Y=MAE_u+`ZtA3xWEnp6RGMSeZK-Xd@ zKm>bznWtXAf+2=hIhVd_VG+HvbeovG*95l7cQ8HEys=e?-TjdfSU@F%gss2=A$3F> z*b9McB?ZEu5CedKKmc7pxC#OMxwLR#(9uh1U=#)b3qTr*A{$f0B2f~*b6s8Qs=Xc7 zF!N2nH8}mhYc-?ey?$=wW{eWiS+!JJCmI<` zjwB0NJC*}n1_IdIXf7>7-nESEyYcj;XXj1+&#&lD=zr9-IspB%;zm!u(sr+Ku)F(x zKS}q|tdoRZztB+De($ZHqZs_hroX?kcIZA4LJ139*2LWkWz>Kq+WNtqSNwgk?gaY84`45Dl^d8UP|7 zkpLwbf`JHRCFc3tb;2kL745WfU5D5%05+M{Qz`-bB z(1Kv13jS|k5BRE!Z^!jhMAT{|gdw=^6!fIf0FtLd(6CELP)bRJ0a4}E!LWFSjI||J zqWW6Guv--pR6+F2{+z=aLKCBdTk0o7`sSgdX2r7w5m8JfpxX_hjU*d{Ov0%NBpOjk z5huQP9$_e^Dtj*0jwhb8!_`$uxL^p_EMvQzn85q*bi8mgCxc=eQe`sYhlfr-juL-h@cf5h(zSG5#ILh0kN_ICKe)UCWDd;r8qGIa|quF zh{TZ$iUFIpTI)0hVhPrHEE5`zBI8`=sT`1yLFHd!a|n8cS_SmJVK=TW6D|rNm|Osm zD@LXbW3I3}9syuFhc!aGir~JvDh>pozK1*HW`$_L9;7trS`9h~lbX_}zPmUi+D&H` z6*aQ5_PpyynDsRUE)@y%DWm}s!xFM!-8XvUIHs6k7SR$_F68ilTnT}2i>APqU+|N& zsBLt15IpG=!WN>7otJpLQ^8@1L2z~U&DEhORornHtyYOF6?9WARv`nZST5gVopc0< zj0c!W;ss(1B|+VRQUwAKAQNkkixSxMx_iW6h)pQuM=tde(iXHqh#sw>ib#hw6ubo$ zc#4)A<1=E=LoX9fjA{nqu#L^@Y#|6hkYozyXDsX0N1X3HnCO}#7}8?y4hp1ipmkIu zwPLGLMcxQ&YppP0#p~>TM1JD`Y3zcdP-GTFKqD1kp({#=QX<7bED0$6{hwXGQ#@a; zeRD65R81erA3UD~sqnE6%HI0#10Ws9e8A=sWN{1C--ETZcKGzX0Ix@La~j5>cH%r6 z?dabmfNzTQw0jT;o%4?P?Dw$G6c|yAos#k2xK&Y6>>AzDbBvtB4-g)4ToM`@c6WDc zlMLRssAn}#P%Ee)oCiEppaQl^l1fA_%xG4{3=9{AXCu}t8orh1n&R)XJ9ayBD(ax* zKPdy`#Bnurk>Z(RNF+Hzn`nkI@+8Ug2Hni+YRD)Yx++oHg9qrp7V{tlS>;ovl(_6vhA-At*ZvHhgTjv@CrIn3j=xOVmYSfHn+6LJ@@L5o!_WCk-NLILQL> zrGpu_tpW$e>e9H6BucDS_0ewE$Iw7~o;}ZX6i$6ke8Qc30(bC*IE&}o$Xi3})F*QJ zCLm@VVo``92!J9MCC)UAF$_!E9QfWkMB!%;dWKjN+f}vfF&ywPF^WLuHvkD{S4Eck(T^yxt8uyAwoH6)5Y; zRaI3}@cJus+tQ?5? zYl2KET8f!=i?`t}y)iD0G?tefrc7d#HobEmDzV#3coi_5oWreQ5g-dhd~?a#y;A|W z5L)%f3JH-UkYmcVGjjm~dFsog;KCq>D}h&_uQZLQY!E?64WLoL5E+}emg3rbT8I&j z$h#W^Bp^ajTD@n9Q53J?o{&oME;JVhQ&(55#<(nN@J ztKkxZ0D-m`6xxRQ!LBn>fT%SI0I1YJt=J>e6bZ#>N?1Z21|nt=cyTpGo_-h#0*HdD zhKi`BF1+{7WZo;4XaF}b<|d`5Pz)sz&R}pa3VXZoglm-;^wX^^bc{%(Doj<9s5_xEEpx$;@ws$aAKQVOp`3$e81vwRM$H zIf5VuT)Gf|bxG6<=nkOBXfTN|fD|mYL#vh4qP7?lQBDA~#`DJ0fn zHslTj<_Q{A!6@Gx^k|$K*7+boOAzD|_XRTVrNRIzqUl_f_>Rp2r8~yqk^^LfbIq8u z7R<=@0eetgnZvDO=Ji-S?zoowY9Y0xE%zG31R^M;8_Cny8ihoO3X-!kOxF1`jGUb7 zq_{W;mX<=H#Ig-hxMDHJc2$5D!ps>Sjw0ucg>VTX926mW)YfYdHM5GnL5NYP1VY5vwV;~tKg^){#NQtF=)CMFdUP<3F z#DT&w0L1ApJUHxQY=sdoRwO8W$Y}$`dh@!zL?|h5C~ueG4*g_kNmbLzgMiN)VFmzo z0qAuQkw7Wv5DcPKPP!GtEbDma4d<+mZ!)0@ogLiUzX8n6J4j4W6ki*^hSKbTH60s! zYE!@5J#&kIQ;t+-n9rxE5fMB@uYrDa8Oa$uP*z~m)_*STdJ63Fm4wfwd zj%CZH>l-8s=LQBHbnfqPE24(_odJM?5Ojt>4RH$%UGEqOK}ZlfnulEHt|o(}|bguAnAX&5><5}28a!E006o*7g#)x#fo8F>>+XqA09QKGis^hX4 zsvU^VK^_%*AaetAbOFLnDsIb;ag9c3R#1WB=wj(d3C^|-(wGRNE-gY_mQpyZ@Gz{V zVcbE2h^a`{&6p5_ClDs1R5*x`3yRW$A)|0C5QxR)#2P`_VgfHGB8a<3QZPzH+$$42 zYaA?L11DcK+?gqnJ8uNmA$LJFhDgdbT}1-|P*Q}jwh%=J9xH<*2?Z2+bwmjXjARYo zxEuSH1Swt?VG0;7MAY@Z%dZD~jIfi$8>WBma4G%_R0VruWo8_)!Rm|&M zJA^n_IS}Yz5p4&7LJPc2n47BDm{dUAKmn|51e(Qa^}ysf$6WHDbRdyYC`~PN)5ZjG zG!vc~8LeLF>T%bN&}qYcf*pQBVK?vMQci2m#U58WvKID7Y~PanF!L=D67KO<{ofS<|! zytPtMRZy{hX0m7Xe{ao@1{6tMOCV@aOIIc=qH9*?TGhMb(iw-TPBV#_8J>m=u`G&z zv|vBX{=wMjW`806#WtpYX$TS0*#Ed}e-W|$WEB3NLJ%W0AV@Kl`yu}v0m0};#DDO$ zt)m)=`@@?Uo?@McL)M6iAjSwm*8i-O5~-$|Xj2L%hiXuAF@k<`&{{z9QWU5RcWZ0A zhgTAI!7MV(-GdTPHmpMnjOCfO$pZ*e2QC_ErrU7SmfVj{W=@{ReIN>27G+;t`r|aw zF|5Hb7Q=v?!KjvGchGgzWRVELh=1d?gX+5;dr1NWLuemGKAn*TP>{&ClnoEB;)COYwy+e~oB3~f%D-~*n3O6C#}}3CO0Igg-;%9uM!S2koLs7{SJW2nNjp86S|x3T>!{qJD#TVpc41@vN{=nr)p*2G2(QHH+m17h(XmTU*7!C%&FlSPdsG>px(rw)SL1!9eDwZuAn8s(6d3~y%NbeM|BWh|Lp zp0;aY*2t}3^;_#R^Qy&BRRQmqGoHOquPF$;^JVmVt}$JmU0^Hi9f2|+qg@hpy1-#Z z#r7VxJFu-p5ad}v`})6acx(sHM~!>mzbsJqFpXejTaHfUHzH{70z+-+)&m$Kb%CM1v|27Hnd6wt;GWYS_r;e zw6v7Fd~rMwBi`2Q>H0LW#JB}{=+_IsxYLc@Q_HHcEdR9}%gF6rCzJqg%(V^q05;Z;d}t1Jk4jJv;YaWVxA+ z=6IgonEJ_<%WR@!^l_|0LDB#~n#~eGw})*freHz^_aQVk_H^^pAvqsW@H`EUhhhiN za4TvMQv>cW@MHt*JumOnl*~W-90BsGa0n`C5QCuWrp|akB36#Fu`W=~5eOJ^LJA5c zo9mg@pA}=@9DsZUCr`Jw*L?yleIz`#5lP%Yl=% z1u~|GNAAaw(w`V7xcNDOFgeN^Q`lVPPvSTmAMF3R&;~qxHB#;uAuj2R8*Pmbx)>MA zA@LXG#cNMcDv=0k5)33ekO_AQbto%HfNS<+uCd|`IGtif;UXrTNDu}%IqZT-@-u1kLB9M%Gbv%2WTWe=N9}fZGbk0Ef z50`|sFg)odBB82^ie_K8ugQyNGqV`dG5ZwI`rjoVOV`sG_MG_eXPVDAyOo=ukn!1h zC}wGnaIjFIjN%n_yI^N4m(DP4%b}+%Y2b7eJ9%xko>~)72`H)xN?DX9QVE!5K=L?I zX8~_>y0HKvUfK(U)WOx`jQq;}- zWj6pS?!+b0QN^Z`f|EvuAT7CQM_gnikmDj?m2)A-I0&c%*JERqM1bIZU~Z|XDS_Iq zMNN+1!J+I&!aa`jP(@S$`Hw^VFJu%$AQmpiLy_Hl{#R8xg%a+P$qQ)fm^qAS!MVEW zb1d9*hGu#bGisxvu7{X{pVA!4pKy9W`wXM__9X;OF$7AJB1{a-Ck5JP*Jx><7sHuA+)*g8OfbP`W_2 zYN!CriNWHYg%4ww>Mx>Nr3wU~Vo8*lR%H{=^f?3O0e}vONX7*Xj<~&Xc<0Z~&Z~{@ zi-w^%5=EIrDll?qC|)8AG{d%NOesN#0!~b1jGL$m+if=6oaSc~jPpbq2gr*Y{}?lnV^q0*g~n8+z3B7qp7cG&uls6N1V6yP6f zMN~vnLsKye5-d!}k$}Zuz+)7#JOk`DJz#%JV1Jqj{Qd+WN8J$6-~-6L{n!MONU@A# z7%xB+rw#;s4uw7CpAQo5Oi?P}Q)x}O_!|e@M#ttZcBf0c7&*M3m6|zfy4g(u*D%da zm#zk-rHW;7WR_cMg)NCZVr?jdBsPhWh_gA8dh7P_U%-EZ=(+yH{XRU^0N0YB7z2Sy zs18!96^wJz?j}JbH5_n8n`Rnmz`%omPeJ4!5VSc`QV#(*>G2Pp&T3+4HZa9a6|;j` z2n}l0SZ}|Y8dN>T4%d=~hQS|TXnqQ0b~+e@RqzQ0DW#>ZllEU9 zlaX}Cky@b#irY+<;uKX(M^RIm$WFxt(DdCAZis?spC1_g-P}O%6q1W1RSO(Y$g#Ts zBN_slaMD|BZKmvKZM-#r0w8x26N#`-Nywk8Z)?)a$gyEFGGe7I3sz$sPP}#2I_ZHS zF*=$^PpsjqScs99<6-3_U!eXaZn~S{RBDCI(l)K29Q>DvAk_ z@eUXYMu1vC7^xSv11=xv3=a1WJtt3Q6mBf=wSU6 z{&$ROkCpR`7FAs)TUJn*OlWanfCeQ20(kQqTtLH(*5n#K_S`rT6g;*t#ya6njs%>z zmA5XodVrv^=chP$GczzUNO8|k1WYje1dHDcw2dtkQUx@P3QYS~APfqs7|uHD$J+12 z^=r#Hc8okaM3QSIKniI>qSRO|Ozihi^3ksvB1If8d#uz145mZ`GTKQ_~1A+0zaM)3kT1sB#e8aoe#}_FqQlR z#JLEilW$ho|%79-WYpw24p0j*$xEuzuWQ-@qFdP zhJ-P&B+8h6sH8m)vHVxC4}Qu@5;^K5i6Y1F?x$gY_t5_g1aKjedF|y^5~8~#mXcH| ztCbar*IZ*5jOQ>_TpkX72jKens)iwEXTTv4JRm?a0fEqkj03~hqah%X^9RW6L!W$9 zg+P@}G}0d-$o&V`1T@TyKG*jv{9bqzMO8!gzXHTjGpt20$Y+DV@6X)W+IBK%X9jZ* zfn7mN&(#p@ef>Q}j6$erY0?%jX0`RAs_d0v=G+b+$NXpgB$%J$NFN0t`u)Jzgc}rp z&M0y=rHmL4a7Zx>{RAx(v{D24#bAjCz}PrHgcBT7Etgi@bqq~I5?sp$F})5BA`A(T zw#d{{4AB4wl1!fmiU<($V1l#(6hcuUTBP`g$4+{p?;B%U4XUcBAkJ_(NyCF7nZphY z=ZV3?Fm}V9In+@>4@^8~gLdph6Z9XrM1UDUh*11ch#MXRbFL~u5-AAB85rQ1gwB>^ zR8If|Tp(2)m01u75s|9Okc(ApLaAmhvuKNt0qNCHOwMzI2t(63<2YtQbDTfCGM$Kd zK!NlD=np^7@db@drvWpW`2BSdK>1NRQ_N9AKO0!lwTOd1nDnAx7YKpU{l*V+1>{ie z#AF!;J}D1j`VUQro})tV)+k7ln!UwK3BN+9uVjQxIB#wjHarZs-m@=;lT_r ze1Jk7DbSy!FIo^FU?Fimwvx={P%Wz3I+&_ffeM`-FldIT(uRGPzTKw>WVbS{$Nb(K z7!axG5$m_DS(478>mqXz(Cj?E4HDqsG^61m2tv@S@sCF6rhRo1Y>$Nd?2xQL^5flw z>XsVq4>z?(;l)9d(t{i!L=vEJkz%>oH-j~Po6xIq^u-x!v_+onkw9M(*hDtX5PJY zN-DJIKUQ=?a>K-uOKtRam4XvHeY(x}AL#wT7(kQ|(G^8YLq!2iMFkZ|!BGU16+lFa zB#MW?kIfbU9SOkFymra*ZiIxv~?05$!0&-N)(EOAPjszVb za19Fkii8J*NO=9{DVRsVGN%XQ6%kAjLom$21Wo8onH_;IPXI6(%z9O zhydE`!%v0MK#-q^T^U(ZJsn|#+v~`D!31gqf>7W{>eSA&z0PFosfJ|D#=G2L;F%+Q zOizo*h!*h7`g&(9=Bf*#M!e3_vdc?4BaujXK^;&5)cBu(BhD270SF33 z4t01U*)1E!I{4%1tagG2ramSoybuB(Vi3~3cuJL2@Y9CW-4cmQ^hk-w_y@S~uzBbp zLPJPT|GD3^$rY3>Dv%Lebv_Q8k3V0mB~dDdnsTQ@$o+;gRY8>+0}IeSds-S9fyx4& z##8CvWm{Vq#wkge&dzu!@MS*^w3Vc`W=A*$d<>6jAAtP_=h*ukjf15vYYQVFMhlRS zm>+*M1cy10(avHbkT}5cg!@Gm6#>8w>-Em7Mj+5maLhaa_Na$(3?Q0<`BMtL3=kmi z@`2;AMYflIS;riPCV(i5AqQ{I-iU-q9VwML5e*(^KMK6oEUZe6!Kf5%$=s^LJDXP2 zq0#gBd8j0ih=2F+KYE`fC}N(Y^**%7Mgs@nKLkh)$763%^#k|VrTho%VA!eBud=aE z);{WBeSzsy;x-lxH8CV=sHjV9hG>dxz-13Z^!t$n{|^E7r)a*uwG`7ZL{K$J1Oy-- zhtMggiXM;D!BsNFU$h^@c)VB0r{*#seZ+UZzdbb-8)KU_bC`&DMaVH4%*=!`&lq^k zs<0fJ@>z(XqVc=n}_z;b~7aA+2NKlT6VV2#|6x`jyp zTZKXWpdT2fnUQ><2n!5|a7BVNKh&Mo$cq&2r>CzKMQ0wj>yznJSPgoyJUF@ssPo0~ z0$RX~j5Hgw1W1;X?TU`aZs(V4G&emk(2^|j2%y9!2}XEj0yMBI1@fN=CCt&WC>kpg zBzAiKzb_q{swA$_>CpCc(BX|S;pq%E>EhW7%hx2P3*xGMdwySFbLH4O!9iWVgQj{O zk?3dFDvnsaj2#tW@t+P14+@+s0B_=8e$_Y~h-m>o<^8as|Mu#c|K literal 0 HcmV?d00001 diff --git a/test/unit/test_ngram/test_lm_read.c b/test/unit/test_ngram/test_lm_read.c index d4ec6f4..26eb07f 100644 --- a/test/unit/test_ngram/test_lm_read.c +++ b/test/unit/test_ngram/test_lm_read.c @@ -18,7 +18,7 @@ main(int argc, char *argv[]) /* Initialize a logmath object to pass to ngram_read */ lmath = logmath_init(1.0001, 0, 0); /* Read a language model */ - model = ngram_model_read(NULL, LMDIR "/100.arpa.gz", NGRAM_ARPA, lmath); + model = ngram_model_read(NULL, LMDIR "/100.arpa.bz2", NGRAM_ARPA, lmath); TEST_ASSERT(model); TEST_EQUAL(ngram_wid(model, ""), 0); TEST_EQUAL(strcmp(ngram_word(model, 0), ""), 0); diff --git a/test/unit/test_util/Makefile.am b/test/unit/test_util/Makefile.am new file mode 100644 index 0000000..9022892 --- /dev/null +++ b/test/unit/test_util/Makefile.am @@ -0,0 +1,15 @@ +check_PROGRAMS = \ + test_fopen + +TESTS = $(check_PROGRAMS) + +INCLUDES = \ + -I$(top_srcdir)/include \ + -I$(top_builddir)/include \ + -DLMDIR=\"${top_srcdir}/test/unit/test_ngram\" + +LDADD = ${top_builddir}/src/libsphinxbase/libsphinxbase.la + +noinst_HEADERS = test_macros.h + +CLEANFILES = *.log diff --git a/test/unit/test_util/test_fopen.c b/test/unit/test_util/test_fopen.c new file mode 100644 index 0000000..c3b4784 --- /dev/null +++ b/test/unit/test_util/test_fopen.c @@ -0,0 +1,44 @@ +/** + * @file test_fopen.c Test file opening + * @author David Huggins-Daines + */ + +#include "pio.h" +#include "test_macros.h" + +#include +#include + +int +main(int argc, char *argv[]) +{ + FILE *fh; + char line[256], *c; + int ispipe; + + fh = fopen_comp(LMDIR "/100.arpa.gz", "r", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa.gz", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa.bz2", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + fh = fopen_compchk(LMDIR "/100.arpa", &ispipe); + TEST_ASSERT(fh != NULL); + c = fgets(line, sizeof(line), fh); + TEST_EQUAL('#', line[0]); + fclose_comp(fh, ispipe); + + return 0; +} diff --git a/test/unit/test_util/test_macros.h b/test/unit/test_util/test_macros.h new file mode 100644 index 0000000..675d2e1 --- /dev/null +++ b/test/unit/test_util/test_macros.h @@ -0,0 +1,10 @@ +#include +#include + +#include "logmath.h" + +#define TEST_ASSERT(x) if (!(x)) { fprintf(stderr, "FAIL: %s\n", #x); exit(1); } +#define TEST_EQUAL(a,b) TEST_ASSERT((a) == (b)) +#define TEST_EQUAL_FLOAT(a,b) TEST_ASSERT(fabs((a) - (b)) < EPSILON) +#define LOG_EPSILON 20 +#define TEST_EQUAL_LOG(a,b) TEST_ASSERT(abs((a) - (b)) < LOG_EPSILON)