GitHub/httrack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UTF-8 filenames handling (based on HTML page charset)

Browse Source
This commit is contained in:
Xavier Roche
2012-05-06 14:20:29 +00:00
parent 78a6312516
commit fbfb68b7c7
21 changed files with 498 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configure
Vendored
+2 -2
View File
@@ -2992,7 +2992,7 @@ fi
# Define the identity of the package.
PACKAGE=httrack
VERSION=3.45.4
VERSION=3.46.1
cat >>confdefs.h <<_ACEOF
@@ -3134,7 +3134,7 @@ am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -'
VERSION_INFO="2:45:0"
VERSION_INFO="2:46:0"
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to enable maintainer-specific portions of Makefiles" >&5
$as_echo_n "checking whether to enable maintainer-specific portions of Makefiles... " >&6; }
# Check whether --enable-maintainer-mode was given.
configure.in
+2 -2
View File
@@ -1,7 +1,7 @@
AC_INIT(src/httrack.c)
AM_CONFIG_HEADER(config.h)
AM_INIT_AUTOMAKE(httrack, 3.45.4)
VERSION_INFO="2:45:0"
AM_INIT_AUTOMAKE(httrack, 3.46.1)
VERSION_INFO="2:46:0"
AM_MAINTAINER_MODE
AC_PREREQ(2.50)
debian/changelog
Vendored
+6
View File
@@ -1,3 +1,9 @@
httrack (3.46.1-1) unstable; urgency=low
* Updated to 3.46.1 (3.46-1)
-- Xavier Roche <xavier@debian.org> Sun, 06 May 2012 16:09:41 +0200
httrack (3.45.4-1) unstable; urgency=low
* Updated to 3.45.4 (3.45-4)
html/httrack.man.html
+1 -1
View File
@@ -1,5 +1,5 @@
<!-- Creator : groff version 1.20.1 -->
<!-- CreationDate: Tue May 1 21:34:14 2012 -->
<!-- CreationDate: Sun May 6 16:10:06 2012 -->
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
src/htsalias.c
+2
View File
@@ -176,6 +176,8 @@ const char* hts_optalias[][4] = {
{"disable-module","-%w","param1",""},
{"no-background-on-suspend","-y0","single",""},
{"background-on-suspend","-y","single",""},
{"utf8-conversion","-%T","single",""},
{"no-utf8-conversion","-%T0","single",""},
/* */
/* DEPRECATED */
src/htsback.c
+35 -34
View File
@@ -128,7 +128,7 @@ void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) {
#ifndef HTS_NO_BACK_ON_DISK
char *filename = (char*) item->value.ptr;
if (filename != NULL) {
(void) unlink(filename);
(void) UNLINK(filename);
}
#else
/* clear entry content (but not yet the entry) */
@@ -196,7 +196,7 @@ static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char*
FILE *fp;
char* fileback = (char*) ptr;
char catbuff[CATBUFF_SIZE];
if (( fp = fopen(fconv(catbuff, fileback), "rb") ) != NULL ) {
if (( fp = FOPEN(fconv(catbuff, fileback), "rb") ) != NULL ) {
if (back_unserialize(fp, &itemback) != 0) {
if (itemback != NULL) {
back_clear_entry(itemback);
@@ -217,7 +217,7 @@ static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char*
test_flush;
}
}
(void) unlink(fileback);
(void) UNLINK(fileback);
#else
itemback = (lien_back*) ptr;
#endif
@@ -293,10 +293,10 @@ int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback)
if (opt->getmode != 0) {
sprintf(filename, "%s.tmp", back[i].url_sav);
} else {
sprintf(filename, "%stmpfile%d.tmp", StringBuff(opt->path_html), opt->state.tmpnameid++);
sprintf(filename, "%stmpfile%d.tmp", StringBuff(opt->path_html_utf8), opt->state.tmpnameid++);
}
/* Security check */
if (fexist(filename)) {
if (fexist_utf8(filename)) {
if (opt->log != NULL) {
HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: temporary file %s already exists"LF, filename);
test_flush;
@@ -323,7 +323,7 @@ int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback)
} else {
if (opt->log != NULL) {
int last_errno = errno;
HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: serialize error for %s%s to %s: open error: %s (%s, %s)"LF, back[i].url_adr, back[i].url_fil, filename, strerror(last_errno), dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist(filename) ? "file already exists!" : "file does not exist");
HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: serialize error for %s%s to %s: open error: %s (%s, %s)"LF, back[i].url_adr, back[i].url_fil, filename, strerror(last_errno), dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist_utf8(filename) ? "file already exists!" : "file does not exist");
test_flush;
}
}
@@ -501,7 +501,7 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
back[p].tmpfile=tmpnam(back[p].tmpfile_buffer);
#endif
if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
back[p].r.out=fopen(back[p].tmpfile,"wb");
back[p].r.out=FOPEN(back[p].tmpfile,"wb");
if (back[p].r.out) {
if ((back[p].r.adr) && (back[p].r.size>0)) {
if (fwrite(back[p].r.adr,1,(size_t)back[p].r.size,back[p].r.out) != back[p].r.size) {
@@ -541,12 +541,12 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
back[p].r.statuscode=STATUSCODE_INVALID;
strcpybuff(back[p].r.msg,"Read error when decompressing");
}
unlink(back[p].url_sav);
UNLINK(back[p].url_sav);
}
}
}
/* encore that no remaining temporary file exists */
unlink(back[p].tmpfile);
UNLINK(back[p].tmpfile);
back[p].tmpfile = NULL;
}
// stats
@@ -578,10 +578,10 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
if (back[p].r.is_write) { // Written file
if (may_be_hypertext_mime(opt,back[p].r.contenttype, back[p].url_fil)) { // to parse!
off_t sz;
sz=fsize(back[p].url_sav);
sz=fsize_utf8(back[p].url_sav);
if (sz>0) { // ok, exists!
if (sz < 8192) { // ok, small file --> to parse!
FILE* fp=fopen(back[p].url_sav,"rb");
FILE* fp=FOPEN(back[p].url_sav,"rb");
if (fp) {
back[p].r.adr=malloct((int)sz + 2);
if (back[p].r.adr) {
@@ -599,7 +599,7 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
fclose(fp);
fp=NULL;
// remove (temporary) file!
unlink(fconv(catbuff,back[p].url_sav));
UNLINK(fconv(catbuff,back[p].url_sav));
}
if (fp)
fclose(fp);
@@ -909,6 +909,7 @@ int back_unserialize(FILE *fp, lien_back** dst) {
}
/* serialize a reference ; used to store references of files being downloaded in case of broken download */
/* Note: NOT utf-8 */
int back_serialize_ref(httrackp* opt, const lien_back* src) {
char *filename = url_savename_refname_fullpath(opt, src->url_adr, src->url_fil);
FILE *fp = fopen(filename, "wb");
@@ -934,7 +935,7 @@ int back_serialize_ref(httrackp* opt, const lien_back* src) {
/* unserialize a reference ; used to store references of files being downloaded in case of broken download */
int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst) {
char *filename = url_savename_refname_fullpath(opt, adr, fil);
FILE *fp = fopen(filename, "rb");
FILE *fp = FOPEN(filename, "rb");
if (fp != NULL) {
int ser = back_unserialize(fp, dst);
fclose(fp);
@@ -1174,7 +1175,7 @@ int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, int
/* écrire date "remote" */
if (strnotempty(back[p].url_sav)
&& strnotempty(back[p].r.lastmodified)
&& fexist(back[p].url_sav)) // normalement existe si on a un fichier de sortie
&& fexist_utf8(back[p].url_sav)) // normalement existe si on a un fichier de sortie
{
set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified);
}
@@ -1268,7 +1269,7 @@ int back_clear_entry(lien_back* back) {
// only for security
if (back->tmpfile && back->tmpfile[0] != '\0') {
(void) unlink(back->tmpfile);
(void) UNLINK(back->tmpfile);
back->tmpfile = NULL;
}
@@ -1471,7 +1472,7 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char*
if (pos<0) { // pas de mise en cache data, vérifier existence
#endif
/* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize(fconv(catbuff,save)) <= 0) { // fichier final n'existe pas ou est vide!
if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff,save)) <= 0) { // fichier final n'existe pas ou est vide!
int found=0;
/* It is possible that the file has been moved due to changes in build structure */
@@ -1483,9 +1484,9 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char*
/* Is supposed to be on disk only */
if (r.is_write && previous_save[0] != '\0') {
/* Exists, but with another (old) filename: rename (almost) silently */
if (strcmp(previous_save, save) != 0 && fexist(fconv(catbuff, previous_save))) {
if (strcmp(previous_save, save) != 0 && fexist_utf8(fconv(catbuff, previous_save))) {
rename(fconv(catbuff, previous_save), fconv(catbuff2,save));
if (fexist(fconv(catbuff,save))) {
if (fexist_utf8(fconv(catbuff,save))) {
found = 1;
if ((opt->debug>1) && (opt->log!=NULL)) {
HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush;
@@ -1511,8 +1512,8 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char*
// sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
// en être sûr
if (opt->norecatch) { // tester norecatch
if (!fexist(fconv(catbuff,save))) { // fichier existe pas mais déclaré: on l'a effacé
FILE* fp=fopen(fconv(catbuff,save),"wb");
if (!fexist_utf8(fconv(catbuff,save))) { // fichier existe pas mais déclaré: on l'a effacé
FILE* fp=FOPEN(fconv(catbuff,save),"wb");
if (fp) fclose(fp);
if (opt->log!=NULL) {
HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Previous file '%s' not found (erased by user ?), ignoring: %s%s"LF,save,back[p].url_adr,back[p].url_fil); test_flush;
@@ -1655,7 +1656,7 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char*
}
/* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
const long file_size = fsize(itemback->url_sav);
const long file_size = fsize_utf8(itemback->url_sav);
/* Found file on disk */
if (file_size > 0) {
char *send_too = back[p].send_too;
@@ -1686,8 +1687,8 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char*
itemback = NULL;
}
/* Not in cache or temporary cache ; found on disk ? (hack) */
else if (fexist(save)) {
off_t sz=fsize(save);
else if (fexist_utf8(save)) {
off_t sz=fsize_utf8(save);
// Bon, là il est possible que le fichier ait été partiellement transféré
// (s'il l'avait été en totalité il aurait été inscrit dans le cache ET existerait sur disque)
// PAS de If-Modified-Since, on a pas connaissance des données à la date du cache
@@ -2668,7 +2669,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
else if (back[i].status==STATUS_FTP_TRANSFER) { // en réception ftp
if (!fexist(back[i].location_buffer)) { // terminé
FILE* fp;
fp=fopen(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"rb");
fp=FOPEN(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"rb");
if (fp) {
int j=0;
fscanf(fp,"%d ",&(back[i].r.statuscode));
@@ -2679,7 +2680,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
}
back[i].r.msg[j++]='\0';
fclose(fp);
unlink(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"));
UNLINK(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"));
strcpybuff(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"");
} else {
strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok");
@@ -2772,7 +2773,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
back[i].tmpfile=tmpnam(back[p].tmpfile_buffer);
#endif
if (back[i].tmpfile != NULL && back[i].tmpfile[0]) {
if ((back[i].r.out=fopen(back[i].tmpfile,"wb")) == NULL) {
if ((back[i].r.out=FOPEN(back[i].tmpfile,"wb")) == NULL) {
last_errno = errno;
}
}
@@ -3292,7 +3293,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
// with an error : consider a 304 error
if (!opt->delete_old) {
if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update && !back[i].testmode) {
if (back[i].url_sav[0] && fexist(back[i].url_sav)) {
if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
if ((opt->debug>1) && (opt->log!=NULL)) {
HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Error ignored %d (%s) because of 'no purge' option for %s%s"LF,back[i].r.statuscode,back[i].r.msg,back[i].url_adr,back[i].url_fil); test_flush;
}
@@ -3350,7 +3351,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
if (back[i].r.statuscode==HTTP_OK && !back[i].testmode) { // 'OK'
if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) { // not HTML
if (strnotempty(back[i].url_sav)) { // target found
int size = fsize(back[i].url_sav); // target size
int size = fsize_utf8(back[i].url_sav); // target size
if (size >= 0) {
if (back[i].r.totalsize == size) { // same size!
deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
@@ -3473,11 +3474,11 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
// In case of 'if-unmodified-since' hack, a 304 status can be sent
// then, force 'ok' status
if (back[i].r.statuscode == STATUSCODE_INVALID) {
if (fexist(back[i].url_sav)) {
if (fexist_utf8(back[i].url_sav)) {
back[i].r.statuscode=HTTP_OK; // OK
strcpybuff(back[i].r.msg, "OK (cached)");
back[i].r.is_file=1;
back[i].r.totalsize = back[i].r.size = fsize(back[i].url_sav);
back[i].r.totalsize = back[i].r.size = fsize_utf8(back[i].url_sav);
get_httptype(opt,back[i].r.contenttype, back[i].url_sav, 1);
if ((opt->debug>0) && (opt->log!=NULL)) {
HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Not-modified status without cache guessed: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
@@ -3544,7 +3545,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
// traiter 206 (partial content)
// xxc SI CHUNK VERIFIER QUE CA MARCHE??
if (back[i].r.statuscode==206) { // on nous envoie un morceau (la fin) coz une partie sur disque!
off_t sz=fsize(back[i].url_sav);
off_t sz=fsize_utf8(back[i].url_sav);
#if HDEBUG
printf("partial content: "LLintP" on disk..\n",(LLint)sz);
#endif
@@ -3553,7 +3554,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
if (opt->getmode&2) { // on peut ecrire des non html **sinon ben euhh sera intercepté plus loin, donc rap sur ce qui va sortir**
filenote(&opt->state.strc,back[i].url_sav,NULL); // noter fichier comme connu
file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 1, back[i].r.notmodified);
back[i].r.out=fopen(fconv(catbuff,back[i].url_sav),"ab"); // append
back[i].r.out=FOPEN(fconv(catbuff,back[i].url_sav),"ab"); // append
if (back[i].r.out) {
back[i].r.is_write=1; // écrire
back[i].r.size=sz; // déja écrit
@@ -3577,7 +3578,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
}
}
} else { // mémoire
FILE* fp=fopen(fconv(catbuff,back[i].url_sav),"rb");
FILE* fp=FOPEN(fconv(catbuff,back[i].url_sav),"rb");
if (fp) {
LLint alloc_mem=sz + 1;
if (back[i].r.totalsize>0)
@@ -3703,7 +3704,7 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti
#if HTS_REMOVE_BAD_FILES
if (back[i].status<0) {
if (!back[i].testmode) { // pas en test
unlink(back[i].url_sav); // éliminer fichier (endommagé)
UNLINK(back[i].url_sav); // éliminer fichier (endommagé)
//printf("&& %s\n",back[i].url_sav);
}
}
src/htscache.c
+19 -16
View File
@@ -147,7 +147,7 @@ void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,const char* url_adr,
// stocker fichiers (et robots.txt)
if ( url_save == NULL || (strnotempty(url_save)) || (strcmp(url_fil,"/robots.txt")==0)) {
// ajouter le fichier au cache
cache_add(opt,cache,r,url_adr,url_fil,url_save,opt->all_in_cache,StringBuff(opt->path_html));
cache_add(opt,cache,r,url_adr,url_fil,url_save,opt->all_in_cache,StringBuff(opt->path_html_utf8));
//
// store a reference NOT to redo the same test zillions of times!
// (problem reported by Lars Clausen)
@@ -367,9 +367,9 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_a
} else {
FILE* fp;
// On recopie le fichier->.
off_t file_size=fsize(fconv(catbuff, url_save));
off_t file_size=fsize_utf8(fconv(catbuff, url_save));
if (file_size>=0) {
fp=fopen(fconv(catbuff, url_save),"rb");
fp=FOPEN(fconv(catbuff, url_save),"rb");
if (fp!=NULL) {
char BIGSTK buff[32768];
size_t nl;
@@ -498,10 +498,10 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,char* url_adr,cha
} else { // recopier fichier dans cache
FILE* fp;
// On recopie le fichier->.
off_t file_size=fsize(fconv(catbuff, url_save));
off_t file_size=fsize_utf8(fconv(catbuff, url_save));
if (file_size>=0) {
if (cache_wLLint(cache_dat,file_size)!=-1) {
fp=fopen(fconv(catbuff, url_save),"rb");
fp=FOPEN(fconv(catbuff, url_save),"rb");
if (fp!=NULL) {
char BIGSTK buff[32768];
ssize_t nl;
@@ -678,9 +678,9 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c
/* Previous entry */
if (previous_save_[0] != '\0') {
int pathLen = (int) strlen(StringBuff(opt->path_html));
if (pathLen != 0 && strncmp(previous_save_, StringBuff(opt->path_html), pathLen) != 0) { // old (<3.40) buggy format
sprintf(previous_save, "%s%s", StringBuff(opt->path_html), previous_save_);
int pathLen = (int) strlen(StringBuff(opt->path_html_utf8));
if (pathLen != 0 && strncmp(previous_save_, StringBuff(opt->path_html_utf8), pathLen) != 0) { // old (<3.40) buggy format
sprintf(previous_save, "%s%s", StringBuff(opt->path_html_utf8), previous_save_);
} else {
strcpy(previous_save, previous_save_);
}
@@ -710,8 +710,8 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c
r.is_write=1; // écrire
if (!dataincache) {
if (fexist(fconv(catbuff, save))) { // un fichier existe déja
//if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
if (fexist_utf8(fconv(catbuff, save))) { // un fichier existe déja
//if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
ok=1; // plus rien à faire
filenote(&opt->state.strc,save,NULL); // noter comme connu
file_notify(opt,adr, fil, save, 0, 0, 1); // data in cache
@@ -797,8 +797,8 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c
strcpybuff(r.msg,"Previous cache file not found (2)");
}
} else { /* Read in memory from cache */
if (strnotempty(previous_save) && fexist(previous_save)) {
FILE* fp = fopen(fconv(catbuff, previous_save), "rb");
if (strnotempty(previous_save) && fexist_utf8(previous_save)) {
FILE* fp = FOPEN(fconv(catbuff, previous_save), "rb");
if (fp != NULL) {
r.adr = (char*) malloct((int) r.size + 4);
if (r.adr != NULL) {
@@ -1016,8 +1016,8 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c
int ok=0;
r.is_write=1; // écrire
if (fexist(fconv(catbuff, save))) { // un fichier existe déja
//if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
if (fexist_utf8(fconv(catbuff, save))) { // un fichier existe déja
//if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
ok=1; // plus rien à faire
filenote(&opt->state.strc,save,NULL); // noter comme connu
file_notify(opt,adr, fil, save, 0, 0, 0);
@@ -1082,8 +1082,8 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c
r.statuscode=STATUSCODE_INVALID;
strcpybuff(r.msg,"Previous cache file not found (2)");
} else { /* Read in memory from cache */
if (strnotempty(return_save) && fexist(return_save)) {
FILE* fp = fopen(fconv(catbuff, return_save), "rb");
if (strnotempty(return_save) && fexist_utf8(return_save)) {
FILE* fp = FOPEN(fconv(catbuff, return_save), "rb");
if (fp != NULL) {
r.adr = (char*) malloct((size_t)r.size + 4);
if (r.adr != NULL) {
@@ -1685,10 +1685,12 @@ void cache_init(cache_back* cache,httrackp* opt) {
// lire un fichier.. (compatible \0)
/* Note: NOT utf-8 */
char* readfile(char* fil) {
return readfile2(fil, NULL);
}
/* Note: NOT utf-8 */
char* readfile2(char* fil, LLint* size) {
char* adr=NULL;
char catbuff[CATBUFF_SIZE];
@@ -1714,6 +1716,7 @@ char* readfile2(char* fil, LLint* size) {
return adr;
}
/* Note: NOT utf-8 */
char* readfile_or(char* fil,char* defaultdata) {
char* realfile=fil;
char* ret;
src/htscore.c
+166 -13
View File
@@ -64,6 +64,9 @@ Please visit our Website: http://www.httrack.com
/* Cache */
#include "htszlib.h"
/* Charset handling */
#include "htscharset.h"
/* END specific definitions */
@@ -256,7 +259,7 @@ if (makeindex_fp) { \
fflush(makeindex_fp); \
fclose(makeindex_fp); /* à ne pas oublier sinon on passe une nuit blanche */ \
makeindex_fp=NULL; \
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt),StringBuff(opt->path_html),"index.html"),"",""); \
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt),StringBuff(opt->path_html_utf8),"index.html"),"",""); \
} \
} \
makeindex_done=1; /* ok c'est fait */ \
@@ -601,7 +604,7 @@ int httpmirror(char* url1, httrackp* opt) {
// lien primaire
liens_record("primary","/primary",fslash(OPT_GET_BUFF(opt),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html")),"","",opt->urlhack);
liens_record("primary","/primary",fslash(OPT_GET_BUFF(opt),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),"index.html")),"","",opt->urlhack);
if (liens[lien_tot]==NULL) { // erreur, pas de place réservée
printf("PANIC! : Not enough memory [%d]\n",__LINE__);
if (opt->log) {
@@ -890,6 +893,8 @@ int httpmirror(char* url1, httrackp* opt) {
str.lien_size_ = &lien_size;
str.lien_buffer_ = &lien_buffer;
/* */
str.page_charset_ = NULL;
/* */
/* */
stre.r_ = &r;
/* */
@@ -1048,6 +1053,33 @@ int httpmirror(char* url1, httrackp* opt) {
(is_hypertext_mime(opt,r.contenttype, urlfil) /* Is HTML or Js, .. */
|| may_be_hypertext_mime(opt,r.contenttype, urlfil)) /* Is real media, .. */
) {
/* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
//if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
// char *charset;
// char *pos;
// if (r.charset[0] != '\0') {
// charset = strdup(r.charset);
// } else {
// charset = hts_getCharsetFromMeta(r.adr, r.size);
// }
// if (charset != NULL) {
// char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
// /* Use new buffer */
// if (utf8 != NULL) {
// freet(r.adr);
// r.size = strlen(utf8);
// r.adr = utf8;
// /* New UTF-8 charset */
// r.charset[0] = '\0';
// strcpy(r.charset, "utf-8");
// }
// /* Free charset */
// free(charset);
// }
//}
/* Check bogus chars */
if ((r.adr) && (r.size)) {
unsigned int map[256];
int i;
@@ -1199,10 +1231,10 @@ int httpmirror(char* url1, httrackp* opt) {
// if (r.adr==NULL) { // Written file
// if (may_be_hypertext_mime(r.contenttype, urlfil)) { // to parse!
// LLint sz;
// sz=fsize(savename);
// sz=fsize_utf8(savename);
// if (sz>0) { // ok, exists!
// if (sz < 8192) { // ok, small file --> to parse!
// FILE* fp=fopen(savename,"rb");
// FILE* fp=FOPEN(savename,"rb");
// if (fp) {
// r.adr=malloct((int)sz + 2);
// if (r.adr) {
@@ -1285,6 +1317,8 @@ int httpmirror(char* url1, httrackp* opt) {
str.lien_size_ = &lien_size;
str.lien_buffer_ = &lien_buffer;
/* */
str.page_charset_ = NULL;
/* */
/* */
stre.r_ = &r;
/* */
@@ -1401,6 +1435,7 @@ int httpmirror(char* url1, httrackp* opt) {
// -- -- -- --
// Parsing HTML
if (!error) {
char page_charset[32];
/* Remove file if being processed */
if (is_loaded_from_file) {
@@ -1408,6 +1443,23 @@ int httpmirror(char* url1, httrackp* opt) {
is_loaded_from_file = 0;
}
/* Detect charset to convert links into proper UTF8 filenames */
page_charset[0] = '\0';
if (opt->convert_utf8) {
if (r.charset[0] != '\0') {
if (strlen(r.charset) < sizeof(page_charset)) {
strcpy(page_charset, r.charset);
}
} else if (is_html_mime_type(r.contenttype)) {
char *const charset = hts_getCharsetFromMeta(r.adr, r.size);
if (charset != NULL && strlen(charset) < sizeof(page_charset)) {
strcpy(page_charset, charset);
}
if (charset != NULL)
free(charset);
}
}
/* Info for wrappers */
if ( (opt->debug>0) && (opt->log!=NULL) ) {
HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: check-html: %s%s"LF,urladr,urlfil);
@@ -1442,6 +1494,8 @@ int httpmirror(char* url1, httrackp* opt) {
str.lien_size_ = &lien_size;
str.lien_buffer_ = &lien_buffer;
/* */
str.page_charset_ = page_charset[0] != '\0' ? page_charset : NULL;
/* */
/* */
stre.r_ = &r;
/* */
@@ -1750,7 +1804,7 @@ int httpmirror(char* url1, httrackp* opt) {
HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Real Media): parsing %s"LF,savename); test_flush;
}
if (fexist(savename)) { // ok, existe bien!
FILE* fp=fopen(savename,"r+b");
FILE* fp=FOPEN(savename,"r+b");
if (fp) {
if (!fseek(fp,0,SEEK_SET)) {
char BIGSTK line[HTS_URLMAXSIZE*2];
@@ -2328,7 +2382,7 @@ static int mkdir_compat(const char *pathname) {
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
HTSEXT_API int dir_exists(const char* path) {
struct stat st;
STRUCT_STAT st;
char BIGSTK file[HTS_URLMAXSIZE*2];
int i = 0;
if (strnotempty(path) == 0) {
@@ -2356,7 +2410,7 @@ HTSEXT_API int dir_exists(const char* path) {
file[i + 1] = '\0';
/* Check the final dir */
if (stat(file, &st) == 0 && S_ISDIR(st.st_mode)) {
if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) {
errno = 0;
return 1; /* EXISTS */
}
@@ -2365,6 +2419,7 @@ HTSEXT_API int dir_exists(const char* path) {
}
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
/* Note: *not* UTF-8 */
HTSEXT_API int structcheck(const char* path) {
struct stat st;
char BIGSTK tmpbuf[HTS_URLMAXSIZE*2];
@@ -2459,6 +2514,102 @@ HTSEXT_API int structcheck(const char* path) {
return 0;
}
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
/* Note: UTF-8 */
HTSEXT_API int structcheck_utf8(const char* path) {
STRUCT_STAT st;
char BIGSTK tmpbuf[HTS_URLMAXSIZE*2];
char BIGSTK file[HTS_URLMAXSIZE*2];
int i = 0;
int npaths;
if (strnotempty(path) == 0)
return 0;
if (strlen(path) > HTS_URLMAXSIZE) {
errno = EINVAL;
return -1;
}
/* Get a copy */
strcpybuff(file, path);
#ifdef _WIN32
/* To system name */
for(i = 0 ; file[i] != 0 ; i++) {
if (file[i] == '/') {
file[i] = PATH_SEPARATOR;
}
}
#endif
/* Get prefix (note: file can not be empty here) */
for(i = (int) strlen(file) - 1 ; i > 0 && file[i] != PATH_SEPARATOR ; i--);
for( ; i > 0 && file[i] == PATH_SEPARATOR ; i--);
file[i + 1] = '\0';
/* First check the final dir */
if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) {
return 0; /* OK */
}
/* Start from the beginning */
i = 0;
/* Skip irrelevant part (the root slash, or the drive path) */
#ifdef _WIN32
if (file[0] != 0 && file[1] == ':') { /* f:\ */
i+= 2;
if (file[i] == PATH_SEPARATOR) { /* f:\ */
i++;
}
} else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */
i+= 2;
}
#endif
/* Check paths */
for(npaths = 1 ; ; npaths++) {
char end_char;
/* Go to next path */
/* Skip separator(s) */
for( ; file[i] == PATH_SEPARATOR ; i++);
/* Next separator */
for( ; file[i] != 0 && file[i] != PATH_SEPARATOR ; i++);
/* Check */
end_char = file[i];
if (end_char != 0) {
file[i] = '\0';
}
if (STAT(file, &st) == 0) { /* Something exists */
if (!S_ISDIR(st.st_mode)) {
#if HTS_REMOVE_ANNOYING_INDEX
if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */
sprintf(tmpbuf, "%s.txt", file);
if (RENAME(file, tmpbuf) != 0) { /* Can't rename regular file */
return -1;
}
if (MKDIR(file) != 0) { /* Can't create directory */
return -1;
}
}
#else
#error Not implemented
#endif
}
} else { /* Nothing exists ; create directory */
if (MKDIR(file) != 0) { /* Can't create directory */
return -1;
}
}
if (end_char == 0) { /* End */
break;
} else {
file[i] = end_char; /* Restore / */
}
}
return 0;
}
// sauver un fichier
int filesave(httrackp* opt,const char* adr,int len,const char* s,const char* url_adr,const char* url_fil) {
FILE* fp;
@@ -2497,6 +2648,7 @@ int check_fatal_io_errno(void) {
// ouvrir un fichier (avec chemin Un*x)
/* Note: utf-8 */
FILE* filecreate(filenote_strc *strc, const char* s) {
char BIGSTK fname[HTS_URLMAXSIZE*2];
FILE* fp;
@@ -2523,17 +2675,17 @@ FILE* filecreate(filenote_strc *strc, const char* s) {
#endif
/* Try to open the file */
fp = fopen(fname, "wb");
fp = FOPEN(fname, "wb");
/* Error ? Check the directory structure and retry. */
if (fp == NULL) {
last_errno = errno;
if (structcheck(s) != 0) {
if (structcheck_utf8(s) != 0) {
last_errno = errno;
} else {
last_errno = 0;
}
fp = fopen(fname, "wb");
fp = FOPEN(fname, "wb");
}
if (fp == NULL && last_errno != 0) {
errno = last_errno;
@@ -2571,7 +2723,7 @@ FILE* fileappend(filenote_strc *strc,const char* s) {
#endif
// ouvrir
fp=fopen(fname,"ab");
fp=FOPEN(fname,"ab");
#ifndef _WIN32
if (fp!=NULL) chmod(fname,HTS_ACCESS_FILE);
@@ -2616,6 +2768,7 @@ int filenote(filenote_strc *strc, const char* s, filecreate_params* params) {
return 1;
}
/* Note: utf-8 */
void file_notify(httrackp* opt,const char* adr,const char* fil,const char* save,int create,int modify,int not_updated) {
RUN_CALLBACK6(opt, filesave2, adr, fil, save, create, modify, not_updated);
}
@@ -2681,7 +2834,7 @@ static void postprocess_file(httrackp* opt,const char* save, const char* adr, co
int n;
if (rsc_fil == NULL)
rsc_fil = fil;
if (strncmp(fslash(OPT_GET_BUFF(opt),save), fslash(OPT_GET_BUFF(opt),StringBuff(opt->path_html)), (n = (int)strlen(StringBuff(opt->path_html)))) == 0) {
if (strncmp(fslash(OPT_GET_BUFF(opt),save), fslash(OPT_GET_BUFF(opt),StringBuff(opt->path_html_utf8)), (n = (int)strlen(StringBuff(opt->path_html_utf8)))) == 0) {
rsc_save += n;
}
@@ -2716,7 +2869,7 @@ static void postprocess_file(httrackp* opt,const char* save, const char* adr, co
}
}
if (opt->state.mimehtml_created == 1 && opt->state.mimefp != NULL) {
FILE* fp = fopen(save, "rb");
FILE* fp = FOPEN(save, "rb");
if (fp != NULL) {
char buff[60*100 + 2];
char mimebuff[256];
src/htscore.h
+1
View File
@@ -340,6 +340,7 @@ void usercommand_exe(const char* cmd,const char* file);
int filters_init(char*** ptrfilters, int maxfilter, int filterinc);
#ifndef HTTRACK_DEFLIB
HTSEXT_API int structcheck(const char* path);
HTSEXT_API int structcheck_utf8(const char* path);
HTSEXT_API int dir_exists(const char* path);
#endif
HTS_INLINE int fspc(httrackp *opt,FILE* fp,const char* type);
src/htscoremain.c
+31 -3
View File
@@ -47,6 +47,7 @@ Please visit our Website: http://www.httrack.com
#include "htswrap.h"
#include "htsmodules.h"
#include "htszlib.h"
#include "htscharset.h"
#include <ctype.h>
#if USE_BEGINTHREAD
@@ -394,6 +395,22 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) {
} // for
// Convert path to UTF-8
#ifdef _WIN32
{
char *const path = hts_convertStringSystemToUTF8(StringBuff(opt->path_html), (int) StringLength(opt->path_html));
if (path != NULL) {
StringCopy(opt->path_html_utf8, path);
free(path);
} else {
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), StringLength(opt->path_html));
}
}
#else
// Assume UTF-8 filesystem.
StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), StringLength(opt->path_html));
#endif
/* if doit.log exists, or if new URL(s) defined,
then DO NOT load standard config files */
/* (config files are added in doit.log) */
@@ -1058,6 +1075,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) {
case 'i': opt->dir_topindex = 1; if (*(com+1)=='0') { opt->dir_topindex=0; com++; } break;
case 'N': opt->savename_delayed = 2; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->savename_delayed); while(isdigit((unsigned char)*(com+1))) com++; } break;
case 'D': opt->delayed_cached=1; if (*(com+1)=='0') { opt->delayed_cached=0; com++; } break; // url hack
case 'T': opt->convert_utf8=1; if (*(com+1)=='0') { opt->convert_utf8=0; com++; } break; // convert to utf-8
case '!': opt->bypass_limits = 1; if (*(com+1)=='0') { opt->bypass_limits=0; com++; } break;
#if HTS_USEMMS
case 'm': sscanf(com+1,"%d",&opt->mms_maxtime); while(isdigit((unsigned char)*(com+1))) com++; break;
@@ -2114,10 +2132,20 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) {
t, url);
fprintf(opt->log,"(");
for(i=0;i<argc;i++) {
if (strchr(argv[i],' ') == NULL || strchr(argv[i],'\"') != NULL)
fprintf(opt->log,"%s ",argv[i]);
#ifdef _WIN32
char *carg = hts_convertStringSystemToUTF8(argv[i], (int) strlen(argv[i]));
char *arg = carg != NULL ? carg : argv[i];
#else
const char *arg = argv[i];
#endif
if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL)
fprintf(opt->log,"%s ", arg);
else // entre "" (si espace(s) et pas déja de ")
fprintf(opt->log,"\"%s\" ",argv[i]);
fprintf(opt->log,"\"%s\" ", arg);
#ifdef _WIN32
if (carg != NULL)
free(carg);
#endif
}
fprintf(opt->log,")"LF);
fprintf(opt->log,LF);
src/htsglobal.h
+2 -2
View File
@@ -40,8 +40,8 @@ Please visit our Website: http://www.httrack.com
#define HTTRACK_GLOBAL_DEFH
// Version (also check external version information)
#define HTTRACK_VERSION "3.45-4"
#define HTTRACK_VERSIONID "3.45.4"
#define HTTRACK_VERSION "3.46-1"
#define HTTRACK_VERSIONID "3.46.1"
#define HTTRACK_AFF_VERSION "3.x"
#define HTTRACK_LIB_VERSION "2.0"
src/htshelp.c
+2 -1
View File
@@ -262,7 +262,7 @@ void help_wizard(httrackp* opt) {
linput(stdin,str,250);
if (strnotempty(str)) {
if (!((str[0]=='y') || (str[0]=='Y')))
return 0;
return ;
}
printf("\n");
@@ -468,6 +468,7 @@ void help(char* app,int more) {
infomsg(" o *generate output html file in case of error (404..) (o0 don't generate)");
infomsg(" X *purge old files after update (X0 keep delete)");
infomsg(" %p preserve html files 'as is' (identical to '-K4 -%F \"\"')");
infomsg(" %T links conversion to UTF-8");
infomsg("");
infomsg("Spider options:");
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
src/htsindex.c
+4 -2
View File
@@ -142,6 +142,7 @@ void index_init(const char* indexpath) {
But should be okay on most cases
Tags and javascript handled (ignored)
*/
/* Note: utf-8 */
int index_keyword(const char* html_data,LLint size,const char* mime,const char* filename,const char* indexpath) {
#if HTS_MAKE_KEYWORD_INDEX
char catbuff[CATBUFF_SIZE];
@@ -166,8 +167,8 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
// Init ?
if (hts_index_init) {
remove(concat(catbuff,indexpath,"index.txt"));
remove(concat(catbuff,indexpath,"sindex.html"));
UNLINK(concat(catbuff,indexpath,"index.txt"));
UNLINK(concat(catbuff,indexpath,"sindex.html"));
hts_index_init=0;
}
@@ -338,6 +339,7 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char*
/*
Sort index!
*/
/* Note: NOT utf-8 */
void index_finish(const char* indexpath,int mode) {
#if HTS_MAKE_KEYWORD_INDEX
char catbuff[CATBUFF_SIZE];
src/htslib.c
+125 -23
View File
@@ -56,6 +56,7 @@ Please visit our Website: http://www.httrack.com
#include "htswrap.h"
#include "htsmd5.h"
#include "htsmodules.h"
#include "htscharset.h"
#ifdef _WIN32
#ifndef _WIN32_WCE
@@ -745,7 +746,7 @@ int http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,cha
else {
// Note: On passe par un FILE* (plus propre)
//soc=open(fil,O_RDONLY,0); // en lecture seule!
retour->fp=fopen(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil)),"rb"); // ouvrir
retour->fp=FOPEN(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil)),"rb"); // ouvrir
if (retour->fp==NULL)
soc=INVALID_SOCKET;
else
@@ -861,7 +862,7 @@ int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr,
search_tag=strstr(fil,POSTTOK"file:");
if (search_tag) { // postfile
if (mode==0) { // GET!
FILE* fp=fopen(unescape_http(OPT_GET_BUFF(opt),search_tag+strlen(POSTTOK)+5),"rb");
FILE* fp=FOPEN(unescape_http(OPT_GET_BUFF(opt),search_tag+strlen(POSTTOK)+5),"rb");
if (fp) {
char BIGSTK line[1100];
char BIGSTK protocol[256],url[HTS_URLMAXSIZE*2],method[256];
@@ -2710,9 +2711,9 @@ int set_filetime_rfc822(const char* file, const char* date) {
}
int get_filetime_rfc822(const char* file, char* date) {
struct stat buf;
STRUCT_STAT buf;
date[0] = '\0';
if (stat(file, &buf) == 0) {
if (STAT(file, &buf) == 0) {
struct tm* A;
time_t tt = buf.st_mtime;
A=gmtime(&tt);
@@ -4306,6 +4307,7 @@ void fprintfio(FILE* fp,char* buff,char* prefix) {
}
/* Le fichier existe-t-il? (ou est-il accessible?) */
/* Note: NOT utf-8 */
int fexist(const char* s) {
char catbuff[CATBUFF_SIZE];
struct stat st;
@@ -4318,27 +4320,44 @@ int fexist(const char* s) {
return 0;
}
/* Taille d'un fichier, -1 si n'existe pas */
/* fp->_cnt ne fonctionne pas sur toute les plate-formes :-(( */
/* Note: NOT YET READY FOR 64-bit */
off_t fsize(const char* s) {
/* Le fichier existe-t-il? (ou est-il accessible?) */
/* Note: utf-8 */
int fexist_utf8(const char* s) {
char catbuff[CATBUFF_SIZE];
FILE* fp;
if (strnotempty(s)==0) // nom vide: erreur
STRUCT_STAT st;
memset(&st, 0, sizeof(st));
if (STAT(fconv(catbuff,s), &st) == 0) {
if (S_ISREG(st.st_mode)) {
return 1;
}
}
return 0;
}
/* Taille d'un fichier, -1 si n'existe pas */
/* Note: NOT utf-8 */
off_t fsize(const char* s) {
struct stat st;
if (!strnotempty(s)) // nom vide: erreur
return -1;
fp=fopen(fconv(catbuff,s),"rb");
if (fp!=NULL) {
off_t i;
fseek(fp,0,SEEK_END);
#ifdef HTS_FSEEKO
i=ftello(fp);
#else
i=ftell(fp);
#endif
fclose(fp);
return i;
} else
if (stat(s, &st) == 0) {
return st.st_size;
} else {
return -1;
}
}
/* Taille d'un fichier, -1 si n'existe pas */
/* Note: utf-8 */
off_t fsize_utf8(const char* s) {
STRUCT_STAT st;
if (!strnotempty(s)) // nom vide: erreur
return -1;
if (STAT(s, &st) == 0) {
return st.st_size;
} else {
return -1;
}
}
off_t fpsize(FILE* fp) {
@@ -5029,7 +5048,7 @@ FILE *hts_dgb_(void) {
#ifdef _WIN32_WCE
hts_dgb_init_fp = fopen("\\Temp\\hts-debug.txt", "wb");
#else
hts_dgb_init_fp = fopen("hts-debug.txt", "wb");
hts_dgb_init_fp = FOPEN("hts-debug.txt", "wb");
#endif
if (hts_dgb_init_fp != NULL) {
fprintf(hts_dgb_init_fp, "* Creating file\r\n");
@@ -5296,6 +5315,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
opt->urlhack=1; // url hack (normalizer)
StringCopy(opt->footer,HTS_DEFAULT_FOOTER);
opt->ftp_proxy=1; // proxy http pour ftp
opt->convert_utf8 = 1; // convert html to UTF-8
StringCopy(opt->filelist,"");
StringCopy(opt->lang_iso,"en, *");
StringCopy(opt->mimedefs,"\n"); // aucun filtre mime (\n IMPORTANT)
@@ -5308,6 +5328,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
opt->keyboard=0;
//
StringCopy(opt->path_html,"");
StringCopy(opt->path_html_utf8,"");
StringCopy(opt->path_log,"");
StringCopy(opt->path_bin,"");
//
@@ -5420,6 +5441,7 @@ HTSEXT_API void hts_free_opt(httrackp *opt) {
StringFree(opt->mod_blacklist);
StringFree(opt->path_html);
StringFree(opt->path_html_utf8);
StringFree(opt->path_log);
StringFree(opt->path_bin);
@@ -5690,6 +5712,86 @@ int closedir(DIR *dir) {
errno = EBADF;
return -1;
}
// UTF-8 aware FILE API
static void copyWchar(LPWSTR dest, const char *src) {
int i;
for(i = 0 ; src[i] ; i++) {
dest[i] = src[i];
}
dest[i] = '\0';
}
FILE* hts_fopen_utf8(const char *path, const char *mode) {
WCHAR wmode[32];
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, strlen(path), NULL);
assertf(strlen(mode) < sizeof(wmode) / sizeof(WCHAR));
copyWchar(wmode, mode);
if (wpath != NULL) {
FILE *const fp = _wfopen(wpath, wmode);
free(wpath);
return fp;
} else {
// Fallback on conversion error.
return fopen(path, mode);
}
}
int hts_stat_utf8(const char *path, STRUCT_STAT *buf) {
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, strlen(path), NULL);
if (wpath != NULL) {
const int result = _wstat(wpath, buf);
free(wpath);
return result;
} else {
// Fallback on conversion error.
return stat(path, buf);
}
}
int hts_unlink_utf8(const char *path) {
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, strlen(path), NULL);
if (wpath != NULL) {
const int result = _wunlink(wpath);
free(wpath);
return result;
} else {
// Fallback on conversion error.
return unlink(path);
}
}
int hts_rename_utf8(const char *oldpath, const char *newpath) {
LPWSTR woldpath = hts_convertUTF8StringToUCS2(oldpath, strlen(oldpath), NULL);
LPWSTR wnewpath = hts_convertUTF8StringToUCS2(newpath, strlen(newpath), NULL);
if (woldpath != NULL && wnewpath != NULL) {
const int result = _wrename(woldpath, wnewpath);
free(woldpath);
free(wnewpath);
return result;
} else {
if (woldpath != NULL)
free(woldpath);
if (wnewpath != NULL)
free(wnewpath);
// Fallback on conversion error.
return rename(oldpath, newpath);
}
}
int hts_mkdir_utf8(const char *path) {
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, strlen(path), NULL);
if (wpath != NULL) {
const int result = _wmkdir(wpath);
free(wpath);
return result;
} else {
// Fallback on conversion error.
return mkdir(path);
}
}
#endif
// Fin
src/htslib.h
+25
View File
@@ -411,9 +411,11 @@ int sig_ignore_flag( int setflag ); // flag ignore
void cut_path(char* fullpath,char* path,char* pname);
int fexist(const char* s);
int fexist_utf8(const char* s);
/*LLint fsize(const char* s); */
off_t fpsize(FILE* fp);
off_t fsize(const char* s);
off_t fsize_utf8(const char* s);
/* root dir */
#ifndef HTTRACK_DEFLIB
HTSEXT_API char* hts_rootdir(char* file);
@@ -488,6 +490,29 @@ void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name);
)
*/
/* UTF-8 aware FILE operations */
#ifdef _WIN32
#define FOPEN hts_fopen_utf8
extern FILE* hts_fopen_utf8(const char *path, const char *mode);
#define STAT hts_stat_utf8
typedef struct _stat STRUCT_STAT;
extern int hts_stat_utf8(const char *path, STRUCT_STAT *buf);
#define UNLINK hts_unlink_utf8
extern int hts_unlink_utf8(const char *pathname);
#define RENAME hts_rename_utf8
extern int hts_rename_utf8(const char *oldpath, const char *newpath);
#define MKDIR(F) hts_mkdir_utf8(F)
extern int hts_mkdir_utf8(const char *pathname);
#else
/* The underlying filesystem charset is supposed to be UTF-8 */
#define FOPEN fopen
#define STAT stat
typedef struct stat STRUCT_STAT;
#define UNLINK unlink
#define RENAME rename
#define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER)
#endif
#endif // internals
#undef PATH_SEPARATOR
src/htsmodules.h
+1
View File
@@ -119,6 +119,7 @@ struct htsmoduleStruct {
int* ptr_;
size_t* lien_size_;
char** lien_buffer_;
const char *page_charset_;
/* Internal use - please don't touch */
};
src/htsname.c
+32 -10
View File
@@ -43,6 +43,7 @@ Please visit our Website: http://www.httrack.com
#include "md5.h"
#include "htsmd5.h"
#include "htstools.h"
#include "htscharset.h"
#include <ctype.h>
#undef test_flush
@@ -119,16 +120,28 @@ static void cleanDoubleSlash(char *s) {
}
}
// legacy version, without page charset
int url_savename(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback, cache_back* cache, hash_struct* hash,
int ptr, int numero_passe, const lien_back* headers) {
return url_savename2(adr_complete, fil_complete, save, former_adr, former_fil,
referer_adr, referer_fil, opt,
liens, lien_tot, sback, cache, hash, ptr, numero_passe, headers, /* unknown */ NULL);
}
// forme le nom du fichier à sauver (save) à partir de fil et adr
// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
int url_savename(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback, cache_back* cache, hash_struct* hash,
int ptr, int numero_passe, const lien_back* headers) {
int url_savename2(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback, cache_back* cache, hash_struct* hash,
int ptr, int numero_passe, const lien_back* headers, const char *charset) {
char catbuff[CATBUFF_SIZE];
const char* mime_type = ( headers && !HTTP_IS_REDIRECT(headers->r.statuscode) ) ? headers->r.contenttype : NULL;
/*const char* mime_type = ( headers && HTTP_IS_OK(headers->r.statuscode) ) ? headers->r.contenttype : NULL;*/
@@ -1306,6 +1319,15 @@ int url_savename(char* adr_complete, char* fil_complete, char* save,
/* ensure that there is no ../ (potential vulnerability) */
fil_simplifie(save);
/* convert name to UTF-8 ? */
if (charset != NULL && charset[0] != '\0') {
char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
if (s != NULL) {
strcpy(save, s);
free(s);
}
}
/* callback */
RUN_CALLBACK5(opt, savename, adr_complete,fil_complete,referer_adr,referer_fil,save);
@@ -1333,9 +1355,9 @@ int url_savename(char* adr_complete, char* fil_complete, char* save,
}
// chemin primaire éventuel A METTRE AVANT
if (strnotempty(StringBuff(opt->path_html))) {
if (strnotempty(StringBuff(opt->path_html_utf8))) {
char BIGSTK tempo[HTS_URLMAXSIZE*2];
strcpybuff(tempo,StringBuff(opt->path_html));
strcpybuff(tempo,StringBuff(opt->path_html_utf8));
strcatbuff(tempo,save);
strcpybuff(save,tempo);
}
@@ -1531,7 +1553,7 @@ char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *
/* remove refname if any */
void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil) {
char *filename = url_savename_refname_fullpath(opt, adr, fil);
(void) unlink(filename);
(void) UNLINK(filename);
}
#undef test_flush
src/htsname.h
+11
View File
@@ -96,6 +96,17 @@ int url_savename(char* adr_complete, char* fil_complete, char* save,
hash_struct* hash,
int ptr, int numero_passe,
const lien_back* headers);
int url_savename2(char* adr_complete, char* fil_complete, char* save,
char* former_adr, char* former_fil,
char* referer_adr, char* referer_fil,
httrackp* opt,
lien_url** liens, int lien_tot,
struct_back* sback,
cache_back* cache,
hash_struct* hash,
int ptr, int numero_passe,
const lien_back* headers,
const char *charset);
void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver);
void url_savename_addstr(char* d,char* s);
char* url_md5(char* digest_buffer, char* fil_complete);
src/htsopt.h
+2
View File
@@ -309,6 +309,7 @@ struct httrackp {
String from; // from
String path_log; // chemin pour cache et log
String path_html; // chemin pour miroir
String path_html_utf8; // chemin pour miroir, UTF-8
String path_bin; // chemin pour templates
int retry; // nombre d'essais supplémentaires en cas d'échec
int makestat; // mettre à jour un fichier log de statistiques de transfert
@@ -349,6 +350,7 @@ struct httrackp {
String lang_iso; // en, fr ..
String mimedefs; // ext1=mimetype1\next2=mimetype2..
String mod_blacklist; // (3.41)
int convert_utf8; // UTF-8 conversion ; 3.46
//
int maxlink; // nombre max de liens
int maxfilter; // nombre max de filtres
src/htsparse.c
+24 -23
View File
@@ -239,7 +239,7 @@ Please visit our Website: http://www.httrack.com
fflush(makeindex_fp); \
fclose(makeindex_fp); /* à ne pas oublier sinon on passe une nuit blanche */ \
makeindex_fp=NULL; \
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html"),"primary","primary"); \
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),"index.html"),"primary","primary"); \
} \
} \
makeindex_done=1; /* ok c'est fait */ \
@@ -429,7 +429,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
// Indexing!
#if HTS_MAKE_KEYWORD_INDEX
if (opt->kindex) {
if (index_keyword(r->adr,r->size,r->contenttype,savename,StringBuff(opt->path_html))) {
if (index_keyword(r->adr,r->size,r->contenttype,savename,StringBuff(opt->path_html_utf8))) {
if ( (opt->debug>1) && (opt->log!=NULL) ) {
HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"indexing file..done"LF); test_flush;
}
@@ -656,9 +656,9 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
if (p) { // ok center
if (makeindex_fp==NULL) {
file_notify(opt,"", "", fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html"), 1, 1, 0);
verif_backblue(opt,StringBuff(opt->path_html)); // générer gif
makeindex_fp=filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html"));
file_notify(opt,"", "", fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),"index.html"), 1, 1, 0);
verif_backblue(opt,StringBuff(opt->path_html_utf8)); // générer gif
makeindex_fp=filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),"index.html"));
if (makeindex_fp!=NULL) {
// Header
@@ -683,7 +683,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
b=strchr(a,'<'); // prochain tag
}
}
if (lienrelatif(tempo,liens[ptr]->sav,concat(OPT_GET_BUFF(opt),StringBuff(opt->path_html),"index.html"))==0) {
if (lienrelatif(tempo,liens[ptr]->sav,concat(OPT_GET_BUFF(opt),StringBuff(opt->path_html_utf8),"index.html"))==0) {
detect_title=1; // ok détecté pour cette page!
makeindex_links++; // un de plus
strcpybuff(makeindex_firstlink,tempo);
@@ -753,6 +753,8 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
if (len > 0) {
if (strfield(token, "content-type")) {
intag_ctype=1;
//NOPE-we do not convert the whole page actually
//intag_start[1] = 'X';
}
else if (strfield(token, "refresh")) {
intag_ctype=2;
@@ -1104,7 +1106,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
) {
chpos++;
while(is_space(*chpos)) chpod++;
chpos
//chpos
}
}
#endif
@@ -2381,7 +2383,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
//char last_fil[HTS_URLMAXSIZE*2]="";
strcpybuff(last_adr,adr); // ancienne adresse
//strcpybuff(last_fil,fil); // ancien chemin
r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL);
r_sv=url_savename2(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL,str->page_charset_);
if (strcmp(jump_identification(last_adr),jump_identification(adr)) != 0) { // a changé
// 2e test si moved
@@ -2578,7 +2580,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
if (patch_it) {
char BIGSTK save[HTS_URLMAXSIZE*2];
char BIGSTK tempo[HTS_URLMAXSIZE*2];
strcpybuff(save,StringBuff(opt->path_html));
strcpybuff(save,StringBuff(opt->path_html_utf8));
strcatbuff(save,cat_name);
if (lienrelatif(tempo,save, relativesavename)==0) {
/* Never escape high-chars (we don't know the encoding!!) */
@@ -2626,17 +2628,16 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
// écrire fichier?
if (verif_external(opt,cat_nb,1)) {
//if (!fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),cat_name))) {
FILE* fp = filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),cat_name));
FILE* fp = filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),cat_name));
if (fp) {
if (cat_data_len==0) { // texte
verif_backblue(opt,StringBuff(opt->path_html));
verif_backblue(opt,StringBuff(opt->path_html_utf8));
fprintf(fp,"%s%s","<!-- Created by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->"LF,cat_data);
} else { // data
fwrite(cat_data,cat_data_len,1,fp);
}
fclose(fp);
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),cat_name),"","");
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),cat_name),"","");
}
}
} else { // écrire normalement le nom de fichier
@@ -2769,8 +2770,8 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) {
uri = save;
// .. after stripping the path prefix (ex: "www.example.com\foo4242.html)
if (strnotempty(StringBuff(opt->path_html))) {
uri += StringLength(opt->path_html);
if (strnotempty(StringBuff(opt->path_html_utf8))) {
uri += StringLength(opt->path_html_utf8);
for( ; uri[0] == '/' || uri[0] == '\\' ; uri++) ;
}
@@ -3383,7 +3384,7 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre)
{
char BIGSTK mov_sav[HTS_URLMAXSIZE*2];
// calculer lien et éventuellement modifier addresse/fichier
if (url_savename(mov_adr,mov_fil,mov_sav,NULL,NULL,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) {
if (url_savename2(mov_adr,mov_fil,mov_sav,NULL,NULL,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL,str->page_charset_)!=-1) {
if (hash_read(hash,mov_sav,"",0,0)<0) { // n'existe pas déja
// enregistrer lien (MACRO) avec SAV IDENTIQUE
liens_record(mov_adr,mov_fil,liens[ptr]->sav,"","");
@@ -3480,9 +3481,9 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre)
(r->statuscode==HTTP_PRECONDITION_FAILED)
|| (r->statuscode==HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)
) { // Precondition Failed, c'est à dire pour nous redemander TOUT le fichier
if (fexist(liens[ptr]->sav)) {
if (fexist_utf8(liens[ptr]->sav)) {
remove(liens[ptr]->sav); // Eliminer
if (!fexist(liens[ptr]->sav)) { // Bien éliminé? (sinon on boucle..)
if (!fexist_utf8(liens[ptr]->sav)) { // Bien éliminé? (sinon on boucle..)
#if HDEBUG
printf("Partial content NOT up-to-date, reget all file for %s\n",liens[ptr]->sav);
#endif
@@ -3800,7 +3801,7 @@ void hts_mirror_process_user_interaction(htsmoduleStruct* str, htsmoduleStructEx
// noter NOUVEAU lien
char BIGSTK add_sav[HTS_URLMAXSIZE*2];
// calculer lien et éventuellement modifier addresse/fichier
if (url_savename(add_adr,add_fil,add_sav,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) {
if (url_savename2(add_adr,add_fil,add_sav,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL,str->page_charset_)!=-1) {
if (hash_read(hash,add_sav,"",0,0)<0) { // n'existe pas déja
// enregistrer lien (MACRO)
liens_record(add_adr,add_fil,add_sav,"","");
@@ -4296,7 +4297,7 @@ int hts_wait_delayed(htsmoduleStruct* str,
/* Recompute filename with MIME type */
save[0] = '\0';
r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&back);
r_sv=url_savename2(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&back,str->page_charset_);
/* Recompute authorization with MIME type */
{
@@ -4364,7 +4365,7 @@ int hts_wait_delayed(htsmoduleStruct* str,
/* Recompute filename with MIME type */
save[0] = '\0';
r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back);
r_sv=url_savename2(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back,str->page_charset_);
/* Recompute authorization with MIME type */
{
@@ -4482,7 +4483,7 @@ int hts_wait_delayed(htsmoduleStruct* str,
strcpybuff(mov_url, back[b].r.location); // copier URL
/* Remove (temporarily created) file if it was created */
unlink(fconv(OPT_GET_BUFF(opt),back[b].url_sav));
UNLINK(fconv(OPT_GET_BUFF(opt),back[b].url_sav));
/* Remove slot! */
if (back[b].status == STATUS_READY) {
@@ -4553,7 +4554,7 @@ int hts_wait_delayed(htsmoduleStruct* str,
/* Recompute filename for hash lookup */
save[0] = '\0';
r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back);
r_sv=url_savename2(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back,str->page_charset_);
} else {
if ( opt->log!=NULL ) {
HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Unable to test %s%s (loop to same filename)"LF,adr,fil);
src/htstools.c
+5 -3
View File
@@ -67,7 +67,7 @@ struct find_handle_struct {
struct find_handle_struct {
DIR * hdir;
struct dirent* dirp;
struct stat filestat;
STRUCT_STAT filestat;
char path[2048];
};
#endif
@@ -481,6 +481,7 @@ void longfile_to_83(int mode,char* n83,char* save) {
}
// écrire backblue.gif
/* Note: utf-8 */
int verif_backblue(httrackp* opt, const char* base) {
int* done = &opt->state.verif_backblue_done;
int ret=0;
@@ -490,7 +491,7 @@ int verif_backblue(httrackp* opt, const char* base) {
return 0;
}
if ( (!*done)
|| (fsize(fconcat(OPT_GET_BUFF(opt), base,"backblue.gif")) != HTS_DATA_BACK_GIF_LEN)) {
|| (fsize_utf8(fconcat(OPT_GET_BUFF(opt), base,"backblue.gif")) != HTS_DATA_BACK_GIF_LEN)) {
FILE* fp = filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), base,"backblue.gif"));
*done=1;
if (fp) {
@@ -683,6 +684,7 @@ static int sortTopIndexFnc(const void * a_, const void * b_) {
HTSEXT_API char* hts_getcategory(const char* filename);
/* Note: NOT utf-8 */
HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath) {
FILE* fpo;
int retval=0;
@@ -1006,7 +1008,7 @@ HTSEXT_API int hts_findnext(find_handle find) {
memset(&(find->filestat), 0, sizeof(find->filestat));
if ((find->dirp=readdir(find->hdir)))
if (find->dirp->d_name)
if (!stat(concat(catbuff, find->path,find->dirp->d_name),&find->filestat))
if (!STAT(concat(catbuff, find->path,find->dirp->d_name),&find->filestat))
return 1;
#endif
}