From 59f10b2c9cc757f41519165af39ef7ec9411e374 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Wed, 10 Aug 2011 23:57:20 -0400 Subject: [PATCH 01/30] move functions+defines out of main.c --- lrzip.c | 61 +++++++++++++++++++++++++++++++++++ lrzip.h | 2 ++ lrzip_private.h | 24 ++++++++++++++ main.c | 85 +------------------------------------------------ 4 files changed, 88 insertions(+), 84 deletions(-) diff --git a/lrzip.c b/lrzip.c index b1d5a4bc..bd8c9d3f 100644 --- a/lrzip.c +++ b/lrzip.c @@ -45,6 +45,7 @@ #ifdef HAVE_ARPA_INET_H # include #endif +#include #include "md5.h" #include "rzip.h" @@ -69,6 +70,66 @@ static i64 fdout_seekto(rzip_control *control, i64 pos) return lseek(control->fd_out, pos, SEEK_SET); } +#ifdef __APPLE__ +# include +inline i64 get_ram(void) +{ + int mib[2]; + size_t len; + i64 *p, ramsize; + + mib[0] = CTL_HW; + mib[1] = HW_MEMSIZE; + sysctl(mib, 2, NULL, &len, NULL, 0); + p = malloc(len); + sysctl(mib, 2, p, &len, NULL, 0); + ramsize = *p; + + return ramsize; +} +#else /* __APPLE__ */ +inline i64 get_ram(void) +{ + i64 ramsize; + FILE *meminfo; + char aux[256]; + + ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE; + if (ramsize > 0) + return ramsize; + + /* Workaround for uclibc which doesn't properly support sysconf */ + if(!(meminfo = fopen("/proc/meminfo", "r"))) + fatal("fopen\n"); + + while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { + if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) + fatal("Failed to fgets in get_ram\n"); + } + if (fclose(meminfo) == -1) + fatal("fclose"); + ramsize *= 1000; + + return ramsize; +} +#endif + +i64 nloops(i64 seconds, uchar *b1, uchar *b2) +{ + i64 nloops; + int nbits; + + nloops = ARBITRARY_AT_EPOCH * pow(MOORE_TIMES_PER_SECOND, seconds); + if (nloops < ARBITRARY) + nloops = ARBITRARY; + for (nbits = 0; nloops > 255; nbits ++) + nloops = nloops >> 1; + *b1 = nbits; + *b2 = nloops; + return nloops << nbits; +} + + void write_magic(rzip_control *control) { char magic[MAGIC_LEN]; diff --git a/lrzip.h b/lrzip.h index 6b54c99c..19399573 100644 --- a/lrzip.h +++ b/lrzip.h @@ -21,6 +21,8 @@ #include "lrzip_private.h" +inline i64 get_ram(void); +i64 nloops(i64 seconds, uchar *b1, uchar *b2); void write_magic(rzip_control *control); void read_magic(rzip_control *control, int fd_in, i64 *expected_size); void preserve_perms(rzip_control *control, int fd_in, int fd_out); diff --git a/lrzip_private.h b/lrzip_private.h index fda47d25..8512bf08 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -177,6 +177,30 @@ typedef struct md5_ctx md5_ctx; #define SALT_LEN 8 #define CBC_LEN 16 +#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN) +# define PROCESSORS (1) +#else +# define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN)) +#endif + +#ifdef _SC_PAGE_SIZE +# define PAGE_SIZE (sysconf(_SC_PAGE_SIZE)) +#else +# define PAGE_SIZE (4096) +#endif + +/* Determine how many times to hash the password when encrypting, based on + * the date such that we increase the number of loops according to Moore's + * law relative to when the data is encrypted. It is then stored as a two + * byte value in the header */ +#define MOORE 1.835 // world constant [TIMES per YEAR] +#define ARBITRARY 1000000 // number of sha2 calls per one second in 2011 +#define T_ZERO 1293840000 // seconds since epoch in 2011 + +#define SECONDS_IN_A_YEAR (365*86400) +#define MOORE_TIMES_PER_SECOND pow (MOORE, 1.0 / SECONDS_IN_A_YEAR) +#define ARBITRARY_AT_EPOCH (ARBITRARY * pow (MOORE_TIMES_PER_SECOND, -T_ZERO)) + #define print_err(format, args...) do {\ fprintf(stderr, format, ##args); \ } while (0) diff --git a/main.c b/main.c index ae99f13d..e190a942 100644 --- a/main.c +++ b/main.c @@ -41,7 +41,7 @@ #ifdef HAVE_SYS_STAT_H # include #endif -#include + #include #ifdef HAVE_ENDIAN_H # include @@ -115,63 +115,6 @@ print_output(format, ##args); \ } while (0) - -#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN) -# define PROCESSORS (1) -#else -# define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN)) -#endif - -#ifdef _SC_PAGE_SIZE -# define PAGE_SIZE (sysconf(_SC_PAGE_SIZE)) -#else -# define PAGE_SIZE (4096) -#endif - -#ifdef __APPLE__ -# include -static inline i64 get_ram(void) -{ - int mib[2]; - size_t len; - i64 *p, ramsize; - - mib[0] = CTL_HW; - mib[1] = HW_MEMSIZE; - sysctl(mib, 2, NULL, &len, NULL, 0); - p = malloc(len); - sysctl(mib, 2, p, &len, NULL, 0); - ramsize = *p; - - return ramsize; -} -#else /* __APPLE__ */ -static inline i64 get_ram(void) -{ - i64 ramsize; - FILE *meminfo; - char aux[256]; - - ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE; - if (ramsize > 0) - return ramsize; - - /* Workaround for uclibc which doesn't properly support sysconf */ - if(!(meminfo = fopen("/proc/meminfo", "r"))) - fatal("fopen\n"); - - while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { - if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) - fatal("Failed to fgets in get_ram\n"); - } - if (fclose(meminfo) == -1) - fatal("fclose"); - ramsize *= 1000; - - return ramsize; -} -#endif - static rzip_control control; static void usage(void) @@ -457,32 +400,6 @@ static void read_config(rzip_control *control) */ } -/* Determine how many times to hash the password when encrypting, based on - * the date such that we increase the number of loops according to Moore's - * law relative to when the data is encrypted. It is then stored as a two - * byte value in the header */ -#define MOORE 1.835 // world constant [TIMES per YEAR] -#define ARBITRARY 1000000 // number of sha2 calls per one second in 2011 -#define T_ZERO 1293840000 // seconds since epoch in 2011 - -#define SECONDS_IN_A_YEAR (365*86400) -#define MOORE_TIMES_PER_SECOND pow (MOORE, 1.0 / SECONDS_IN_A_YEAR) -#define ARBITRARY_AT_EPOCH (ARBITRARY * pow (MOORE_TIMES_PER_SECOND, -T_ZERO)) - -static i64 nloops(i64 seconds, uchar *b1, uchar *b2) -{ - i64 nloops; - int nbits; - - nloops = ARBITRARY_AT_EPOCH * pow(MOORE_TIMES_PER_SECOND, seconds); - if (nloops < ARBITRARY) - nloops = ARBITRARY; - for (nbits = 0; nloops > 255; nbits ++) - nloops = nloops >> 1; - *b1 = nbits; - *b2 = nloops; - return nloops << nbits; -} int main(int argc, char *argv[]) { From 87120e0f75b6882c09046e811d290d5c8aa23ede Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 00:24:33 -0400 Subject: [PATCH 02/30] use liblrzip.h defines in main.c --- main.c | 307 ++++++++++++++++++++++++--------------------------------- 1 file changed, 129 insertions(+), 178 deletions(-) diff --git a/main.c b/main.c index e190a942..e96aacdd 100644 --- a/main.c +++ b/main.c @@ -52,6 +52,7 @@ # include #endif +#include "liblrzip.h" #include "rzip.h" #include "lrzip.h" #include "util.h" @@ -64,58 +65,7 @@ #define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) #define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) -/* main() defines, different from liblrzip defines */ -#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) -#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) -#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA)) - -#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS) -#define KEEP_FILES (control.flags & FLAG_KEEP_FILES) -#define TEST_ONLY (control.flags & FLAG_TEST_ONLY) -#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE) -#define DECOMPRESS (control.flags & FLAG_DECOMPRESS) -#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS) -#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS) -#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS) -#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS) -#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS) -#define VERBOSE (control.flags & FLAG_VERBOSE) -#define VERBOSITY (control.flags & FLAG_VERBOSITY) -#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX) -#define STDIN (control.flags & FLAG_STDIN) -#define STDOUT (control.flags & FLAG_STDOUT) -#define INFO (control.flags & FLAG_INFO) -#define UNLIMITED (control.flags & FLAG_UNLIMITED) -#define HASH_CHECK (control.flags & FLAG_HASH) -#define HAS_MD5 (control.flags & FLAG_MD5) -#define CHECK_FILE (control.flags & FLAG_CHECK) -#define KEEP_BROKEN (control.flags & FLAG_KEEP_BROKEN) -#define LZO_TEST (control.flags & FLAG_THRESHOLD) -#define TMP_OUTBUF (control.flags & FLAG_TMP_OUTBUF) -#define TMP_INBUF (control.flags & FLAG_TMP_INBUF) -#define ENCRYPT (control.flags & FLAG_ENCRYPT) - -#define print_output(format, args...) do {\ - fprintf(control.msgout, format, ##args); \ - fflush(control.msgout); \ -} while (0) - -#define print_progress(format, args...) do {\ - if (SHOW_PROGRESS) \ - print_output(format, ##args); \ -} while (0) - -#define print_verbose(format, args...) do {\ - if (VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -#define print_maxverbose(format, args...) do {\ - if (MAX_VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -static rzip_control control; +static rzip_control controlstaticvariablehaha, *control; static void usage(void) { @@ -183,11 +133,11 @@ static void show_summary(void) if (!TEST_ONLY) print_verbose("The following options are in effect for this %s.\n", DECOMPRESS ? "DECOMPRESSION" : "COMPRESSION"); - print_verbose("Threading is %s. Number of CPUs detected: %d\n", control.threads > 1? "ENABLED" : "DISABLED", - control.threads); - print_verbose("Detected %lld bytes ram\n", control.ramsize); - print_verbose("Compression level %d\n", control.compression_level); - print_verbose("Nice Value: %d\n", control.nice_val); + print_verbose("Threading is %s. Number of CPUs detected: %d\n", control->threads > 1? "ENABLED" : "DISABLED", + control->threads); + print_verbose("Detected %lld bytes ram\n", control->ramsize); + print_verbose("Compression level %d\n", control->compression_level); + print_verbose("Nice Value: %d\n", control->nice_val); print_verbose("Show Progress\n"); print_maxverbose("Max "); print_verbose("Verbose\n"); @@ -195,14 +145,14 @@ static void show_summary(void) print_verbose("Overwrite Files\n"); if (!KEEP_FILES) print_verbose("Remove input files on completion\n"); - if (control.outdir) - print_verbose("Output Directory Specified: %s\n", control.outdir); - else if (control.outname) - print_verbose("Output Filename Specified: %s\n", control.outname); + if (control->outdir) + print_verbose("Output Directory Specified: %s\n", control->outdir); + else if (control->outname) + print_verbose("Output Filename Specified: %s\n", control->outname); if (TEST_ONLY) print_verbose("Test file integrity\n"); - if (control.tmpdir) - print_verbose("Temporary Directory set as: %s\n", control.tmpdir); + if (control->tmpdir) + print_verbose("Temporary Directory set as: %s\n", control->tmpdir); /* show compression options */ if (!DECOMPRESS && !TEST_ONLY) { @@ -219,16 +169,16 @@ static void show_summary(void) print_verbose("ZPAQ. LZO Compressibility testing %s\n", (LZO_TEST? "enabled" : "disabled")); else if (NO_COMPRESS) print_verbose("RZIP pre-processing only\n"); - if (control.window) - print_verbose("Compression Window: %lld = %lldMB\n", control.window, control.window * 100ull); + if (control->window) + print_verbose("Compression Window: %lld = %lldMB\n", control->window, control->window * 100ull); /* show heuristically computed window size */ - if (!control.window && !UNLIMITED) { + if (!control->window && !UNLIMITED) { i64 temp_chunk, temp_window; if (STDOUT || STDIN) - temp_chunk = control.maxram; + temp_chunk = control->maxram; else - temp_chunk = control.ramsize * 2 / 3; + temp_chunk = control->ramsize * 2 / 3; temp_window = temp_chunk / (100 * 1024 * 1024); print_verbose("Heuristically Computed Compression Window: %lld = %lldMB\n", temp_window, temp_window * 100ull); } @@ -236,9 +186,9 @@ static void show_summary(void) print_verbose("Using Unlimited Window size\n"); } if (!DECOMPRESS && !TEST_ONLY) - print_maxverbose("Storage time in seconds %lld\n", control.secs); + print_maxverbose("Storage time in seconds %lld\n", control->secs); if (ENCRYPT) - print_maxverbose("Encryption hash loops %lld\n", control.encloops); + print_maxverbose("Encryption hash loops %lld\n", control->encloops); } } @@ -411,35 +361,36 @@ int main(int argc, char *argv[]) extern int optind; char *eptr; /* for environment */ - memset(&control, 0, sizeof(control)); + control = &controlstaticvariablehaha; + memset(control, 0, sizeof(rzip_control)); - control.msgout = stderr; - register_outputfile(control.msgout); - control.flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; - control.suffix = ".lrz"; - control.outdir = NULL; - control.tmpdir = NULL; + control->msgout = stderr; + register_outputfile(control->msgout); + control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; + control->suffix = ".lrz"; + control->outdir = NULL; + control->tmpdir = NULL; if (strstr(argv[0], "lrunzip")) - control.flags |= FLAG_DECOMPRESS; + control->flags |= FLAG_DECOMPRESS; else if (strstr(argv[0], "lrzcat")) - control.flags |= FLAG_DECOMPRESS | FLAG_STDOUT; + control->flags |= FLAG_DECOMPRESS | FLAG_STDOUT; - control.compression_level = 7; - control.ramsize = get_ram(); + control->compression_level = 7; + control->ramsize = get_ram(); /* for testing single CPU */ - control.threads = PROCESSORS; /* get CPUs for LZMA */ - control.page_size = PAGE_SIZE; - control.nice_val = 19; + control->threads = PROCESSORS; /* get CPUs for LZMA */ + control->page_size = PAGE_SIZE; + control->nice_val = 19; /* The first 5 bytes of the salt is the time in seconds. * The next 2 bytes encode how many times to hash the password. * The last 9 bytes are random data, making 16 bytes of salt */ if (unlikely(gettimeofday(&tv, NULL))) fatal("Failed to gettimeofday in main\n"); - control.secs = tv.tv_sec; - control.encloops = nloops(control.secs, control.salt, control.salt + 1); - get_rand(control.salt + 2, 6); + control->secs = tv.tv_sec; + control->encloops = nloops(control->secs, control->salt, control->salt + 1); + get_rand(control->salt + 2, 6); /* generate crc table */ CrcGenerateTable(); @@ -447,12 +398,12 @@ int main(int argc, char *argv[]) /* Get Temp Dir */ eptr = getenv("TMP"); if (eptr != NULL) { - control.tmpdir = malloc(strlen(eptr)+2); - if (control.tmpdir == NULL) + control->tmpdir = malloc(strlen(eptr)+2); + if (control->tmpdir == NULL) fatal("Failed to allocate for tmpdir\n"); - strcpy(control.tmpdir, eptr); + strcpy(control->tmpdir, eptr); if (strcmp(eptr+strlen(eptr) - 1, "/")) /* need a trailing slash */ - strcat(control.tmpdir, "/"); + strcat(control->tmpdir, "/"); } /* Get Preloaded Defaults from lrzip.conf @@ -461,126 +412,126 @@ int main(int argc, char *argv[]) */ eptr = getenv("LRZIP"); if (eptr == NULL) - read_config(&control); + read_config(control); else if (!strstr(eptr,"NOCONFIG")) - read_config(&control); + read_config(control); while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) { switch (c) { case 'b': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_BZIP2_COMPRESS; + control->flags |= FLAG_BZIP2_COMPRESS; break; case 'c': - control.flags |= FLAG_CHECK; - control.flags |= FLAG_HASH; + control->flags |= FLAG_CHECK; + control->flags |= FLAG_HASH; break; case 'd': - control.flags |= FLAG_DECOMPRESS; + control->flags |= FLAG_DECOMPRESS; break; case 'D': - control.flags &= ~FLAG_KEEP_FILES; + control->flags &= ~FLAG_KEEP_FILES; break; case 'e': - control.flags |= FLAG_ENCRYPT; + control->flags |= FLAG_ENCRYPT; break; case 'f': - control.flags |= FLAG_FORCE_REPLACE; + control->flags |= FLAG_FORCE_REPLACE; break; case 'g': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_ZLIB_COMPRESS; + control->flags |= FLAG_ZLIB_COMPRESS; break; case 'h': case '?': usage(); return -1; case 'H': - control.flags |= FLAG_HASH; + control->flags |= FLAG_HASH; break; case 'i': - control.flags |= FLAG_INFO; + control->flags |= FLAG_INFO; break; case 'k': - control.flags |= FLAG_KEEP_BROKEN; + control->flags |= FLAG_KEEP_BROKEN; break; case 'l': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_LZO_COMPRESS; + control->flags |= FLAG_LZO_COMPRESS; break; case 'L': - control.compression_level = atoi(optarg); - if (control.compression_level < 1 || control.compression_level > 9) + control->compression_level = atoi(optarg); + if (control->compression_level < 1 || control->compression_level > 9) failure("Invalid compression level (must be 1-9)\n"); break; case 'n': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_NO_COMPRESS; + control->flags |= FLAG_NO_COMPRESS; break; case 'N': - control.nice_val = atoi(optarg); - if (control.nice_val < -20 || control.nice_val > 19) + control->nice_val = atoi(optarg); + if (control->nice_val < -20 || control->nice_val > 19) failure("Invalid nice value (must be -20..19)\n"); break; case 'o': - if (control.outdir) + if (control->outdir) failure("Cannot have -o and -O together\n"); if (unlikely(STDOUT)) failure("Cannot specify an output filename when outputting to stdout\n"); - control.outname = optarg; - control.suffix = ""; + control->outname = optarg; + control->suffix = ""; break; case 'O': - if (control.outname) /* can't mix -o and -O */ + if (control->outname) /* can't mix -o and -O */ failure("Cannot have options -o and -O together\n"); if (unlikely(STDOUT)) failure("Cannot specify an output directory when outputting to stdout\n"); - control.outdir = malloc(strlen(optarg) + 2); - if (control.outdir == NULL) + control->outdir = malloc(strlen(optarg) + 2); + if (control->outdir == NULL) fatal("Failed to allocate for outdir\n"); - strcpy(control.outdir,optarg); + strcpy(control->outdir,optarg); if (strcmp(optarg+strlen(optarg) - 1, "/")) /* need a trailing slash */ - strcat(control.outdir, "/"); + strcat(control->outdir, "/"); break; case 'p': - control.threads = atoi(optarg); - if (control.threads < 1) + control->threads = atoi(optarg); + if (control->threads < 1) failure("Must have at least one thread\n"); break; case 'q': - control.flags &= ~FLAG_SHOW_PROGRESS; + control->flags &= ~FLAG_SHOW_PROGRESS; break; case 'S': - if (control.outname) + if (control->outname) failure("Specified output filename already, can't specify an extension.\n"); if (unlikely(STDOUT)) failure("Cannot specify a filename suffix when outputting to stdout\n"); - control.suffix = optarg; + control->suffix = optarg; break; case 't': - if (control.outname) + if (control->outname) failure("Cannot specify an output file name when just testing.\n"); if (!KEEP_FILES) failure("Doubt that you want to delete a file when just testing.\n"); - control.flags |= FLAG_TEST_ONLY; + control->flags |= FLAG_TEST_ONLY; break; case 'T': - control.flags &= ~FLAG_THRESHOLD; + control->flags &= ~FLAG_THRESHOLD; break; case 'U': - control.flags |= FLAG_UNLIMITED; + control->flags |= FLAG_UNLIMITED; break; case 'v': /* set verbosity flag */ - if (!(control.flags & FLAG_VERBOSITY) && !(control.flags & FLAG_VERBOSITY_MAX)) - control.flags |= FLAG_VERBOSITY; - else if ((control.flags & FLAG_VERBOSITY)) { - control.flags &= ~FLAG_VERBOSITY; - control.flags |= FLAG_VERBOSITY_MAX; + if (!(control->flags & FLAG_VERBOSITY) && !(control->flags & FLAG_VERBOSITY_MAX)) + control->flags |= FLAG_VERBOSITY; + else if ((control->flags & FLAG_VERBOSITY)) { + control->flags &= ~FLAG_VERBOSITY; + control->flags |= FLAG_VERBOSITY_MAX; } break; case 'V': @@ -588,12 +539,12 @@ int main(int argc, char *argv[]) exit(0); break; case 'w': - control.window = atol(optarg); + control->window = atol(optarg); break; case 'z': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_ZPAQ_COMPRESS; + control->flags |= FLAG_ZPAQ_COMPRESS; break; } } @@ -601,61 +552,61 @@ int main(int argc, char *argv[]) argc -= optind; argv += optind; - if (control.outname && argc > 1) + if (control->outname && argc > 1) failure("Cannot specify output filename with more than 1 file\n"); if (VERBOSE && !SHOW_PROGRESS) { print_err("Cannot have -v and -q options. -v wins.\n"); - control.flags |= FLAG_SHOW_PROGRESS; + control->flags |= FLAG_SHOW_PROGRESS; } - if (UNLIMITED && control.window) { + if (UNLIMITED && control->window) { print_err("If -U used, cannot specify a window size with -w.\n"); - control.window = 0; + control->window = 0; } if (argc < 1) - control.flags |= FLAG_STDIN; + control->flags |= FLAG_STDIN; if (UNLIMITED && STDIN) { print_err("Cannot have -U and stdin, unlimited mode disabled.\n"); - control.flags &= ~FLAG_UNLIMITED; + control->flags &= ~FLAG_UNLIMITED; } /* Work out the compression overhead per compression thread for the * compression back-ends that need a lot of ram */ if (LZMA_COMPRESS) { - int level = control.compression_level * 7 / 9 ? : 1; + int level = control->compression_level * 7 / 9 ? : 1; i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); - control.overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); + control->overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); } else if (ZPAQ_COMPRESS) - control.overhead = 112 * 1024 * 1024; + control->overhead = 112 * 1024 * 1024; /* Set the main nice value to half that of the backend threads since * the rzip stage is usually the rate limiting step */ - if (control.nice_val > 0 && !NO_COMPRESS) { - if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val / 2) == -1)) + if (control->nice_val > 0 && !NO_COMPRESS) { + if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val / 2) == -1)) print_err("Warning, unable to set nice value\n"); } else { - if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val) == -1)) + if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1)) print_err("Warning, unable to set nice value\n"); } /* One extra iteration for the case of no parameters means we will default to stdin/out */ for (i = 0; i <= argc; i++) { if (i < argc) - control.infile = argv[i]; + control->infile = argv[i]; else if (!(i == 0 && STDIN)) break; - if (control.infile) { - if ((strcmp(control.infile, "-") == 0)) - control.flags |= FLAG_STDIN; + if (control->infile) { + if ((strcmp(control->infile, "-") == 0)) + control->flags |= FLAG_STDIN; else { struct stat infile_stat; - stat(control.infile, &infile_stat); + stat(control->infile, &infile_stat); if (unlikely(S_ISDIR(infile_stat.st_mode))) failure("lrzip only works directly on FILES.\n" "Use lrztar or pipe through tar for compressing directories.\n"); @@ -665,23 +616,23 @@ int main(int argc, char *argv[]) if (INFO && STDIN) failure("Will not get file info from STDIN\n"); - if (control.outname && (strcmp(control.outname, "-") == 0)) { - control.flags |= FLAG_STDOUT; - control.msgout = stderr; - register_outputfile(control.msgout); + if (control->outname && (strcmp(control->outname, "-") == 0)) { + control->flags |= FLAG_STDOUT; + control->msgout = stderr; + register_outputfile(control->msgout); } /* If no output filename is specified, and we're using stdin, * use stdout */ - if (!control.outname && STDIN) { - control.flags |= FLAG_STDOUT; - control.msgout = stderr; - register_outputfile(control.msgout); + if (!control->outname && STDIN) { + control->flags |= FLAG_STDOUT; + control->msgout = stderr; + register_outputfile(control->msgout); } if (!STDOUT) { - control.msgout = stdout; - register_outputfile(control.msgout); + control->msgout = stdout; + register_outputfile(control->msgout); } /* Implement signal handler only once flags are set */ handler.sa_handler = &sighandler; @@ -704,19 +655,19 @@ int main(int argc, char *argv[]) if (CHECK_FILE) { if (!DECOMPRESS) { print_err("Can only check file written on decompression.\n"); - control.flags &= ~FLAG_CHECK; + control->flags &= ~FLAG_CHECK; } else if (STDOUT) { print_err("Can't check file written when writing to stdout. Checking disabled.\n"); - control.flags &= ~FLAG_CHECK; + control->flags &= ~FLAG_CHECK; } } /* Use less ram when using STDOUT to store the temporary output * file. */ if (STDOUT && ((STDIN && DECOMPRESS) || !(DECOMPRESS || TEST_ONLY))) - control.maxram = control.ramsize * 2 / 9; + control->maxram = control->ramsize * 2 / 9; else - control.maxram = control.ramsize / 3; + control->maxram = control->ramsize / 3; if (BITS32) { /* Decrease usable ram size on 32 bits due to kernel / * userspace split. Cannot allocate larger than a 1 @@ -725,12 +676,12 @@ int main(int argc, char *argv[]) * 2/3 of that makes for a total of 2GB to be split * into thirds. */ - control.usable_ram = MAX(control.ramsize - 900000000ll, 900000000ll); - control.maxram = MIN(control.maxram, control.usable_ram); - control.maxram = MIN(control.maxram, one_g * 2 / 3); + control->usable_ram = MAX(control->ramsize - 900000000ll, 900000000ll); + control->maxram = MIN(control->maxram, control->usable_ram); + control->maxram = MIN(control->maxram, one_g * 2 / 3); } else - control.usable_ram = control.maxram; - round_to_page(&control.maxram); + control->usable_ram = control->maxram; + round_to_page(&control->maxram); show_summary(); @@ -740,11 +691,11 @@ int main(int argc, char *argv[]) failure("Unable to work from STDIN while reading password\n"); if (DECOMPRESS || TEST_ONLY) - decompress_file(&control); + decompress_file(control); else if (INFO) - get_fileinfo(&control); + get_fileinfo(control); else - compress_file(&control); + compress_file(control); /* compute total time */ gettimeofday(&end_time, NULL); From 4642e68d6f4ba62903783828ae28e3644202d108 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 00:35:41 -0400 Subject: [PATCH 03/30] abort() instead of exit() --- util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util.c b/util.c index 2179abc8..b80eb251 100644 --- a/util.c +++ b/util.c @@ -102,7 +102,7 @@ static void fatal_exit(void) unlink_files(); fprintf(outputfile, "Fatal error - exiting\n"); fflush(outputfile); - exit(1); + abort(); } /* Failure when there is likely to be a meaningful error in perror */ From cfd6915b206cc342600339f8ab7c1ea272ea7c69 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 00:41:21 -0400 Subject: [PATCH 04/30] move some initialization stuff out of main() --- lrzip.c | 40 ++++++++++++++++++++++++++++++++++++++++ lrzip.h | 1 + main.c | 37 ++----------------------------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/lrzip.c b/lrzip.c index bd8c9d3f..aa03de01 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1143,3 +1143,43 @@ void compress_file(rzip_control *control) free(control->outfile); } + +void initialize_control(rzip_control *control) +{ + struct timeval tv; + char *eptr; /* for environment */ + + memset(control, 0, sizeof(rzip_control)); + control->msgout = stderr; + register_outputfile(control->msgout); + control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; + control->suffix = ".lrz"; + control->compression_level = 7; + control->ramsize = get_ram(); + /* for testing single CPU */ + control->threads = PROCESSORS; /* get CPUs for LZMA */ + control->page_size = PAGE_SIZE; + control->nice_val = 19; + + /* The first 5 bytes of the salt is the time in seconds. + * The next 2 bytes encode how many times to hash the password. + * The last 9 bytes are random data, making 16 bytes of salt */ + if (unlikely(gettimeofday(&tv, NULL))) + fatal("Failed to gettimeofday in main\n"); + control->secs = tv.tv_sec; + control->encloops = nloops(control->secs, control->salt, control->salt + 1); + get_rand(control->salt + 2, 6); + + /* Get Temp Dir */ + eptr = getenv("TMP"); + if (eptr != NULL) { + size_t len = strlen(eptr); + control->tmpdir = malloc(len+2); + if (control->tmpdir == NULL) + fatal("Failed to allocate for tmpdir\n"); + strcpy(control->tmpdir, eptr); + if (eptr[len - 2] != '/') + eptr[len - 2] = '/'; /* need a trailing slash */ + eptr[len - 1] = 0; + } +} diff --git a/lrzip.h b/lrzip.h index 19399573..e263f204 100644 --- a/lrzip.h +++ b/lrzip.h @@ -41,4 +41,5 @@ void close_tmpoutbuf(rzip_control *control); void clear_tmpinbuf(rzip_control *control); inline void clear_tmpinfile(rzip_control *control); void close_tmpinbuf(rzip_control *control); +void initialize_control(rzip_control *control); #endif diff --git a/main.c b/main.c index e96aacdd..7812f543 100644 --- a/main.c +++ b/main.c @@ -353,7 +353,7 @@ static void read_config(rzip_control *control) int main(int argc, char *argv[]) { - struct timeval start_time, end_time, tv; + struct timeval start_time, end_time; struct sigaction handler; double seconds,total_time; // for timers int c, i; @@ -362,50 +362,17 @@ int main(int argc, char *argv[]) char *eptr; /* for environment */ control = &controlstaticvariablehaha; - memset(control, 0, sizeof(rzip_control)); - control->msgout = stderr; - register_outputfile(control->msgout); - control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; - control->suffix = ".lrz"; - control->outdir = NULL; - control->tmpdir = NULL; + initialize_control(control); if (strstr(argv[0], "lrunzip")) control->flags |= FLAG_DECOMPRESS; else if (strstr(argv[0], "lrzcat")) control->flags |= FLAG_DECOMPRESS | FLAG_STDOUT; - control->compression_level = 7; - control->ramsize = get_ram(); - /* for testing single CPU */ - control->threads = PROCESSORS; /* get CPUs for LZMA */ - control->page_size = PAGE_SIZE; - control->nice_val = 19; - - /* The first 5 bytes of the salt is the time in seconds. - * The next 2 bytes encode how many times to hash the password. - * The last 9 bytes are random data, making 16 bytes of salt */ - if (unlikely(gettimeofday(&tv, NULL))) - fatal("Failed to gettimeofday in main\n"); - control->secs = tv.tv_sec; - control->encloops = nloops(control->secs, control->salt, control->salt + 1); - get_rand(control->salt + 2, 6); - /* generate crc table */ CrcGenerateTable(); - /* Get Temp Dir */ - eptr = getenv("TMP"); - if (eptr != NULL) { - control->tmpdir = malloc(strlen(eptr)+2); - if (control->tmpdir == NULL) - fatal("Failed to allocate for tmpdir\n"); - strcpy(control->tmpdir, eptr); - if (strcmp(eptr+strlen(eptr) - 1, "/")) /* need a trailing slash */ - strcat(control->tmpdir, "/"); - } - /* Get Preloaded Defaults from lrzip.conf * Look in ., $HOME/.lrzip/, /etc/lrzip. * If LRZIP=NOCONFIG is set, then ignore config From 1c64dcfb312b94b2fa563a9cd24c3756d4d1bfe6 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 00:52:59 -0400 Subject: [PATCH 05/30] more initialization stuff out of main(), remove weird coincidentally successful one_g variable --- lrzip_private.h | 2 ++ main.c | 22 +--------------------- stream.c | 2 -- stream.h | 1 - util.c | 23 +++++++++++++++++++++++ util.h | 1 + 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/lrzip_private.h b/lrzip_private.h index 8512bf08..02e2ac52 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -177,6 +177,8 @@ typedef struct md5_ctx md5_ctx; #define SALT_LEN 8 #define CBC_LEN 16 +#define one_g (1000 * 1024 * 1024) + #if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN) # define PROCESSORS (1) #else diff --git a/main.c b/main.c index 7812f543..da1608ea 100644 --- a/main.c +++ b/main.c @@ -629,27 +629,7 @@ int main(int argc, char *argv[]) } } - /* Use less ram when using STDOUT to store the temporary output - * file. */ - if (STDOUT && ((STDIN && DECOMPRESS) || !(DECOMPRESS || TEST_ONLY))) - control->maxram = control->ramsize * 2 / 9; - else - control->maxram = control->ramsize / 3; - if (BITS32) { - /* Decrease usable ram size on 32 bits due to kernel / - * userspace split. Cannot allocate larger than a 1 - * gigabyte chunk due to 32 bit signed long being - * used in alloc, and at most 3GB can be malloced, and - * 2/3 of that makes for a total of 2GB to be split - * into thirds. - */ - control->usable_ram = MAX(control->ramsize - 900000000ll, 900000000ll); - control->maxram = MIN(control->maxram, control->usable_ram); - control->maxram = MIN(control->maxram, one_g * 2 / 3); - } else - control->usable_ram = control->maxram; - round_to_page(&control->maxram); - + setup_ram(control); show_summary(); gettimeofday(&start_time, NULL); diff --git a/stream.c b/stream.c index a745ad58..9a6f923f 100644 --- a/stream.c +++ b/stream.c @@ -662,8 +662,6 @@ static int lzo_decompress_buf(rzip_control *control __UNUSED__, struct uncomp_th /* WORK FUNCTIONS */ -i64 one_g = 1000 * 1024 * 1024; - /* Look at whether we're writing to a ram location or physical files and write * the data accordingly. */ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret) diff --git a/stream.h b/stream.h index 9e4600a6..e91a170f 100644 --- a/stream.h +++ b/stream.h @@ -38,6 +38,5 @@ i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len int close_stream_out(rzip_control *control, void *ss); int close_stream_in(void *ss); ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret); -i64 one_g; #endif diff --git a/util.c b/util.c index b80eb251..930a37fb 100644 --- a/util.c +++ b/util.c @@ -133,6 +133,29 @@ void failure(const char *format, ...) fatal_exit(); } +void setup_ram(rzip_control *control) +{ + /* Use less ram when using STDOUT to store the temporary output file. */ + if (STDOUT && ((STDIN && DECOMPRESS) || !(DECOMPRESS || TEST_ONLY))) + control->maxram = control->ramsize * 2 / 9; + else + control->maxram = control->ramsize / 3; + if (BITS32) { + /* Decrease usable ram size on 32 bits due to kernel / + * userspace split. Cannot allocate larger than a 1 + * gigabyte chunk due to 32 bit signed long being + * used in alloc, and at most 3GB can be malloced, and + * 2/3 of that makes for a total of 2GB to be split + * into thirds. + */ + control->usable_ram = MAX(control->ramsize - 900000000ll, 900000000ll); + control->maxram = MIN(control->maxram, control->usable_ram); + control->maxram = MIN(control->maxram, one_g * 2 / 3); + } else + control->usable_ram = control->maxram; + round_to_page(&control->maxram); +} + void round_to_page(i64 *size) { *size -= *size % PAGE_SIZE; diff --git a/util.h b/util.h index 74fe8199..8b8bf905 100644 --- a/util.h +++ b/util.h @@ -27,6 +27,7 @@ void unlink_files(void); void register_outputfile(FILE *f); void fatal(const char *format, ...); void failure(const char *format, ...); +void setup_ram(rzip_control *control); void round_to_page(i64 *size); void get_rand(uchar *buf, int len); void lrz_stretch(rzip_control *control); From e85e4df0efbe6e7e19969c8d5dbec5bdeac5276a Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 00:57:43 -0400 Subject: [PATCH 06/30] move compression overhead calcs out of main() --- main.c | 11 +---------- util.c | 14 ++++++++++++++ util.h | 1 + 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/main.c b/main.c index da1608ea..654c12b1 100644 --- a/main.c +++ b/main.c @@ -540,16 +540,7 @@ int main(int argc, char *argv[]) control->flags &= ~FLAG_UNLIMITED; } - /* Work out the compression overhead per compression thread for the - * compression back-ends that need a lot of ram */ - if (LZMA_COMPRESS) { - int level = control->compression_level * 7 / 9 ? : 1; - i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) : - (level == 6 ? (1 << 25) : (1 << 26))); - - control->overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); - } else if (ZPAQ_COMPRESS) - control->overhead = 112 * 1024 * 1024; + setup_overhead(control); /* Set the main nice value to half that of the backend threads since * the rzip stage is usually the rate limiting step */ diff --git a/util.c b/util.c index 930a37fb..a92fa7ed 100644 --- a/util.c +++ b/util.c @@ -133,6 +133,20 @@ void failure(const char *format, ...) fatal_exit(); } +void setup_overhead(rzip_control *control) +{ + /* Work out the compression overhead per compression thread for the + * compression back-ends that need a lot of ram */ + if (LZMA_COMPRESS) { + int level = control->compression_level * 7 / 9 ? : 1; + i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) : + (level == 6 ? (1 << 25) : (1 << 26))); + + control->overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); + } else if (ZPAQ_COMPRESS) + control->overhead = 112 * 1024 * 1024; +} + void setup_ram(rzip_control *control) { /* Use less ram when using STDOUT to store the temporary output file. */ diff --git a/util.h b/util.h index 8b8bf905..d70a7a96 100644 --- a/util.h +++ b/util.h @@ -27,6 +27,7 @@ void unlink_files(void); void register_outputfile(FILE *f); void fatal(const char *format, ...); void failure(const char *format, ...); +void setup_overhead(rzip_control *control); void setup_ram(rzip_control *control); void round_to_page(i64 *size); void get_rand(uchar *buf, int len); From cbff8faef745322439a32402798c89747a7601a2 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 03:06:32 -0400 Subject: [PATCH 07/30] move liblrzip.h contents to lrzip_private.h global variables----- --- liblrzip.h | 50 ------------- lrzip.c | 182 ++++++++++++++++++++++++------------------------ lrzip.h | 2 +- lrzip_private.h | 56 +++++++++++++++ main.c | 68 +++++++++--------- runzip.c | 60 ++++++++-------- rzip.c | 66 ++++++++++-------- rzip.h | 1 + stream.c | 166 +++++++++++++++++++++---------------------- util.c | 58 +++++++-------- util.h | 14 ++-- 11 files changed, 366 insertions(+), 357 deletions(-) diff --git a/liblrzip.h b/liblrzip.h index d817af0f..d683b957 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -20,54 +20,4 @@ #ifndef LIBLRZIP_H #define LIBLRZIP_H -#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) -#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) -#define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA)) - -#define SHOW_PROGRESS (control->flags & FLAG_SHOW_PROGRESS) -#define KEEP_FILES (control->flags & FLAG_KEEP_FILES) -#define TEST_ONLY (control->flags & FLAG_TEST_ONLY) -#define FORCE_REPLACE (control->flags & FLAG_FORCE_REPLACE) -#define DECOMPRESS (control->flags & FLAG_DECOMPRESS) -#define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS) -#define LZO_COMPRESS (control->flags & FLAG_LZO_COMPRESS) -#define BZIP2_COMPRESS (control->flags & FLAG_BZIP2_COMPRESS) -#define ZLIB_COMPRESS (control->flags & FLAG_ZLIB_COMPRESS) -#define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS) -#define VERBOSE (control->flags & FLAG_VERBOSE) -#define VERBOSITY (control->flags & FLAG_VERBOSITY) -#define MAX_VERBOSE (control->flags & FLAG_VERBOSITY_MAX) -#define STDIN (control->flags & FLAG_STDIN) -#define STDOUT (control->flags & FLAG_STDOUT) -#define INFO (control->flags & FLAG_INFO) -#define UNLIMITED (control->flags & FLAG_UNLIMITED) -#define HASH_CHECK (control->flags & FLAG_HASH) -#define HAS_MD5 (control->flags & FLAG_MD5) -#define CHECK_FILE (control->flags & FLAG_CHECK) -#define KEEP_BROKEN (control->flags & FLAG_KEEP_BROKEN) -#define LZO_TEST (control->flags & FLAG_THRESHOLD) -#define TMP_OUTBUF (control->flags & FLAG_TMP_OUTBUF) -#define TMP_INBUF (control->flags & FLAG_TMP_INBUF) -#define ENCRYPT (control->flags & FLAG_ENCRYPT) - -#define print_output(format, args...) do {\ - fprintf(control->msgout, format, ##args); \ - fflush(control->msgout); \ -} while (0) - -#define print_progress(format, args...) do {\ - if (SHOW_PROGRESS) \ - print_output(format, ##args); \ -} while (0) - -#define print_verbose(format, args...) do {\ - if (VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -#define print_maxverbose(format, args...) do {\ - if (MAX_VERBOSE) \ - print_output(format, ##args); \ -} while (0) - #endif diff --git a/lrzip.c b/lrzip.c index aa03de01..2de89203 100644 --- a/lrzip.c +++ b/lrzip.c @@ -88,7 +88,7 @@ inline i64 get_ram(void) return ramsize; } #else /* __APPLE__ */ -inline i64 get_ram(void) +inline i64 get_ram(rzip_control *control) { i64 ramsize; FILE *meminfo; @@ -100,14 +100,14 @@ inline i64 get_ram(void) /* Workaround for uclibc which doesn't properly support sysconf */ if(!(meminfo = fopen("/proc/meminfo", "r"))) - fatal("fopen\n"); + fatal(control, "fopen\n"); while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) - fatal("Failed to fgets in get_ram\n"); + fatal(control, "Failed to fgets in get_ram\n"); } if (fclose(meminfo) == -1) - fatal("fclose"); + fatal(control, "fclose"); ramsize *= 1000; return ramsize; @@ -168,10 +168,10 @@ void write_magic(rzip_control *control) magic[22] = 1; if (unlikely(fdout_seekto(control, 0))) - fatal("Failed to seek to BOF to write Magic Header\n"); + fatal(control, "Failed to seek to BOF to write Magic Header\n"); if (unlikely(put_fdout(control, magic, MAGIC_LEN) != MAGIC_LEN)) - fatal("Failed to write magic header\n"); + fatal(control, "Failed to write magic header\n"); control->magic_written = 1; } @@ -187,7 +187,7 @@ static void get_magic(rzip_control *control, char *magic) uint32_t v; if (unlikely(strncmp(magic, "LRZI", 4))) - failure("Not an lrzip file\n"); + failure(control, "Not an lrzip file\n"); memcpy(&control->major_version, &magic[4], 1); memcpy(&control->minor_version, &magic[5], 1); @@ -230,7 +230,7 @@ static void get_magic(rzip_control *control, char *magic) if (encrypted == 1) control->flags |= FLAG_ENCRYPT; else - failure("Unknown encryption\n"); + failure(control, "Unknown encryption\n"); /* In encrypted files, the size field is used to store the salt * instead and the size is unknown, just like a STDOUT chunked * file */ @@ -251,7 +251,7 @@ void read_magic(rzip_control *control, int fd_in, i64 *expected_size) memset(magic, 0, sizeof(magic)); /* Initially read only st_size; @@ -263,7 +263,7 @@ void preserve_perms(rzip_control *control, int fd_in, int fd_out) struct stat st; if (unlikely(fstat(fd_in, &st))) - fatal("Failed to fstat input file\n"); + fatal(control, "Failed to fstat input file\n"); if (unlikely(fchmod(fd_out, (st.st_mode & 0666)))) print_err("Warning, unable to set permissions on %s\n", control->outfile); @@ -282,24 +282,24 @@ int open_tmpoutfile(rzip_control *control) if (control->tmpdir) { control->outfile = realloc(NULL, strlen(control->tmpdir) + 16); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal(control, "Failed to allocate outfile name\n"); strcpy(control->outfile, control->tmpdir); strcat(control->outfile, "lrzipout.XXXXXX"); } else { control->outfile = realloc(NULL, 16); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal(control, "Failed to allocate outfile name\n"); strcpy(control->outfile, "lrzipout.XXXXXX"); } fd_out = mkstemp(control->outfile); if (unlikely(fd_out == -1)) - fatal("Failed to create out tmpfile: %s\n", control->outfile); - register_outfile(control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); + fatal(control, "Failed to create out tmpfile: %s\n", control->outfile); + register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); return fd_out; } -static void fwrite_stdout(void *buf, i64 len) +static void fwrite_stdout(rzip_control *control, void *buf, i64 len) { uchar *offset_buf = buf; ssize_t ret; @@ -313,7 +313,7 @@ static void fwrite_stdout(void *buf, i64 len) ret = len; ret = fwrite(offset_buf, 1, ret, stdout); if (unlikely(ret <= 0)) - fatal("Failed to fwrite in fwrite_stdout\n"); + fatal(control, "Failed to fwrite in fwrite_stdout\n"); len -= ret; offset_buf += ret; total += ret; @@ -330,7 +330,7 @@ void write_fdout(rzip_control *control, void *buf, i64 len) ret = MIN(len, one_g); ret = write(control->fd_out, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal("Failed to write to fd_out in write_fdout\n"); + fatal(control, "Failed to write to fd_out in write_fdout\n"); len -= ret; offset_buf += ret; } @@ -341,7 +341,7 @@ void flush_tmpoutbuf(rzip_control *control) if (!TEST_ONLY) { print_maxverbose("Dumping buffer to physical file.\n"); if (STDOUT) - fwrite_stdout(control->tmp_outbuf, control->out_len); + fwrite_stdout(control, control->tmp_outbuf, control->out_len); else write_fdout(control, control->tmp_outbuf, control->out_len); } @@ -359,7 +359,7 @@ void dump_tmpoutfile(rzip_control *control, int fd_out) fsync(fd_out); tmpoutfp = fdopen(fd_out, "r"); if (unlikely(tmpoutfp == NULL)) - fatal("Failed to fdopen out tmpfile\n"); + fatal(control, "Failed to fdopen out tmpfile\n"); rewind(tmpoutfp); if (!TEST_ONLY) { @@ -371,7 +371,7 @@ void dump_tmpoutfile(rzip_control *control, int fd_out) } if (unlikely(ftruncate(fd_out, 0))) - fatal("Failed to ftruncate fd_out in dump_tmpoutfile\n"); + fatal(control, "Failed to ftruncate fd_out in dump_tmpoutfile\n"); } /* Used if we're unable to read STDIN into the temporary buffer, shunts data @@ -386,7 +386,7 @@ void write_fdin(rzip_control *control) ret = MIN(len, one_g); ret = write(control->fd_in, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal("Failed to write to fd_in in write_fdin\n"); + fatal(control, "Failed to write to fd_in in write_fdin\n"); len -= ret; offset_buf += ret; } @@ -400,24 +400,24 @@ int open_tmpinfile(rzip_control *control) if (control->tmpdir) { control->infile = malloc(strlen(control->tmpdir) + 15); if (unlikely(!control->infile)) - fatal("Failed to allocate infile name\n"); + fatal(control, "Failed to allocate infile name\n"); strcpy(control->infile, control->tmpdir); strcat(control->infile, "lrzipin.XXXXXX"); } else { control->infile = malloc(15); if (unlikely(!control->infile)) - fatal("Failed to allocate infile name\n"); + fatal(control, "Failed to allocate infile name\n"); strcpy(control->infile, "lrzipin.XXXXXX"); } fd_in = mkstemp(control->infile); if (unlikely(fd_in == -1)) - fatal("Failed to create in tmpfile: %s\n", control->infile); - register_infile(control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); + fatal(control, "Failed to create in tmpfile: %s\n", control->infile); + register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); /* Unlink temporary file immediately to minimise chance of files left * lying around in cases of failure. */ if (unlikely(unlink(control->infile))) - fatal("Failed to unlink tmpfile: %s\n", control->infile); + fatal(control, "Failed to unlink tmpfile: %s\n", control->infile); return fd_in; } @@ -430,7 +430,7 @@ static void read_tmpinmagic(rzip_control *control) for (i = 0; i < 24; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure("Reached end of file on STDIN prematurely on v05 magic read\n"); + failure(control, "Reached end of file on STDIN prematurely on v05 magic read\n"); magic[i] = (char)tmpchar; } get_magic(control, magic); @@ -446,7 +446,7 @@ void read_tmpinfile(rzip_control *control, int fd_in) fprintf(control->msgout, "Copying from stdin.\n"); tmpinfp = fdopen(fd_in, "w+"); if (unlikely(tmpinfp == NULL)) - fatal("Failed to fdopen in tmpfile\n"); + fatal(control, "Failed to fdopen in tmpfile\n"); while ((tmpchar = getchar()) != EOF) fputc(tmpchar, tmpinfp); @@ -465,7 +465,7 @@ static void open_tmpoutbuf(rzip_control *control) * fall back to a real temporary file */ control->tmp_outbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_outbuf)) - fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); + fatal(control, "Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); if (!DECOMPRESS && !TEST_ONLY) control->out_ofs = control->out_len = MAGIC_LEN; } @@ -482,7 +482,7 @@ static void open_tmpinbuf(rzip_control *control) control->in_maxlen = control->maxram; control->tmp_inbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_inbuf)) - fatal("Failed to malloc tmp_inbuf in open_tmpinbuf\n"); + fatal(control, "Failed to malloc tmp_inbuf in open_tmpinbuf\n"); } inline void clear_tmpinbuf(rzip_control *control) @@ -493,9 +493,9 @@ inline void clear_tmpinbuf(rzip_control *control) void clear_tmpinfile(rzip_control *control) { if (unlikely(lseek(control->fd_in, 0, SEEK_SET))) - fatal("Failed to lseek on fd_in in clear_tmpinfile\n"); + fatal(control, "Failed to lseek on fd_in in clear_tmpinfile\n"); if (unlikely(ftruncate(control->fd_in, 0))) - fatal("Failed to truncate fd_in in clear_tmpinfile\n"); + fatal(control, "Failed to truncate fd_in in clear_tmpinfile\n"); } void close_tmpinbuf(rzip_control *control) @@ -504,13 +504,13 @@ void close_tmpinbuf(rzip_control *control) free(control->tmp_inbuf); } -static int get_pass(char *s) +static int get_pass(rzip_control *control, char *s) { int len; memset(s, 0, PASS_LEN - SALT_LEN); if (unlikely(fgets(s, PASS_LEN - SALT_LEN, stdin) == NULL)) - failure("Failed to retrieve passphrase\n"); + failure(control, "Failed to retrieve passphrase\n"); len = strlen(s); if (len > 0 && ('\r' == s[len - 1] || '\n' == s[len - 1])) s[len - 1] = '\0'; @@ -518,7 +518,7 @@ static int get_pass(char *s) s[len - 2] = '\0'; len = strlen(s); if (unlikely(0 == len)) - failure("Empty passphrase\n"); + failure(control, "Empty passphrase\n"); return len; } @@ -532,7 +532,7 @@ static void get_hash(rzip_control *control, int make_hash) control->salt_pass = calloc(PASS_LEN, 1); control->hash = calloc(HASH_LEN, 1); if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) - fatal("Failed to calloc encrypt buffers in compress_file\n"); + fatal(control, "Failed to calloc encrypt buffers in compress_file\n"); mlock(passphrase, PASS_LEN); mlock(testphrase, PASS_LEN); mlock(control->salt_pass, PASS_LEN); @@ -544,11 +544,11 @@ static void get_hash(rzip_control *control, int make_hash) tcsetattr(fileno(stdin), 0, &termios_p); retry_pass: print_output("Enter passphrase: "); - control->salt_pass_len = get_pass(passphrase) + SALT_LEN; + control->salt_pass_len = get_pass(control, passphrase) + SALT_LEN; print_output("\n"); if (make_hash) { print_output("Re-enter passphrase: "); - get_pass(testphrase); + get_pass(control, testphrase); print_output("\n"); if (strcmp(passphrase, testphrase)) { print_output("Passwords do not match. Try again.\n"); @@ -600,7 +600,7 @@ void decompress_file(rzip_control *control) */ infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); if (unlikely(infilecopy == NULL)) - fatal("Failed to allocate memory for infile suffix\n"); + fatal(control, "Failed to allocate memory for infile suffix\n"); else { strcpy(infilecopy, control->infile); strcat(infilecopy, control->suffix); @@ -630,7 +630,7 @@ void decompress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpoutfile) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal(control, "Failed to allocate outfile name\n"); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); @@ -648,13 +648,13 @@ void decompress_file(rzip_control *control) fd_in = open_tmpinfile(control); read_tmpinmagic(control); if (ENCRYPT) - failure("Cannot decompress encrypted file from STDIN\n"); + failure(control, "Cannot decompress encrypted file from STDIN\n"); expected_size = control->st_size; open_tmpinbuf(control); } else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) { - fatal("Failed to open %s\n", infilecopy); + fatal(control, "Failed to open %s\n", infilecopy); } } control->fd_in = fd_in; @@ -663,18 +663,18 @@ void decompress_file(rzip_control *control) fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink an existing file: %s\n", control->outfile); + fatal(control, "Failed to unlink an existing file: %s\n", control->outfile); fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal("Failed to create %s\n", control->outfile); + fatal(control, "Failed to create %s\n", control->outfile); } fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control->outfile); + fatal(control, "Failed to open history file %s\n", control->outfile); /* Can't copy permissions from STDIN */ if (!STDIN) @@ -682,13 +682,13 @@ void decompress_file(rzip_control *control) } else { fd_out = open_tmpoutfile(control); if (unlikely(fd_out == -1)) - fatal("Failed to create %s\n", control->outfile); + fatal(control, "Failed to create %s\n", control->outfile); fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control->outfile); + fatal(control, "Failed to open history file %s\n", control->outfile); /* Unlink temporary file as soon as possible */ if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink tmpfile: %s\n", control->outfile); + fatal(control, "Failed to unlink tmpfile: %s\n", control->outfile); } open_tmpoutbuf(control); @@ -700,13 +700,13 @@ void decompress_file(rzip_control *control) /* Check if there's enough free space on the device chosen to fit the * decompressed file. */ if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal("Failed to fstatvfs in decompress_file\n"); + fatal(control, "Failed to fstatvfs in decompress_file\n"); free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < expected_size) { if (FORCE_REPLACE) print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n"); else - failure("Inadequate free space to decompress file, use -f to override.\n"); + failure(control, "Inadequate free space to decompress file, use -f to override.\n"); } } control->fd_out = fd_out; @@ -742,13 +742,13 @@ void decompress_file(rzip_control *control) print_progress("[OK] \n"); if (unlikely(close(fd_hist) || close(fd_out))) - fatal("Failed to close files\n"); + fatal(control, "Failed to close files\n"); close(fd_in); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal("Failed to unlink %s\n", infilecopy); + fatal(control, "Failed to unlink %s\n", infilecopy); } if (ENCRYPT) @@ -762,18 +762,18 @@ void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head, int chunk_bytes) { if (unlikely(read(fd_in, ctype, 1) != 1)) - fatal("Failed to read in get_header_info\n"); + fatal(control, "Failed to read in get_header_info\n"); *c_len = *u_len = *last_head = 0; if (control->major_version == 0 && control->minor_version < 4) { u32 c_len32, u_len32, last_head32; if (unlikely(read(fd_in, &c_len32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal(control, "Failed to read in get_header_info"); if (unlikely(read(fd_in, &u_len32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal(control, "Failed to read in get_header_info"); if (unlikely(read(fd_in, &last_head32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal(control, "Failed to read in get_header_info"); c_len32 = le32toh(c_len32); u_len32 = le32toh(u_len32); last_head32 = le32toh(last_head32); @@ -788,11 +788,11 @@ void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, else read_len = chunk_bytes; if (unlikely(read(fd_in, c_len, read_len) != read_len)) - fatal("Failed to read in get_header_info"); + fatal(control, "Failed to read in get_header_info"); if (unlikely(read(fd_in, u_len, read_len) != read_len)) - fatal("Failed to read in get_header_info"); + fatal(control, "Failed to read in get_header_info"); if (unlikely(read(fd_in, last_head, read_len) != read_len)) - fatal("Failed to read_i64 in get_header_info"); + fatal(control, "Failed to read_i64 in get_header_info"); *c_len = le64toh(*c_len); *u_len = le64toh(*u_len); *last_head = le64toh(*last_head); @@ -833,7 +833,7 @@ void get_fileinfo(rzip_control *control) strcmp(tmp,control->suffix)) { infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); if (unlikely(infilecopy == NULL)) - fatal("Failed to allocate memory for infile suffix\n"); + fatal(control, "Failed to allocate memory for infile suffix\n"); else { strcpy(infilecopy, control->infile); strcat(infilecopy, control->suffix); @@ -847,12 +847,12 @@ void get_fileinfo(rzip_control *control) else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) - fatal("Failed to open %s\n", infilecopy); + fatal(control, "Failed to open %s\n", infilecopy); } /* Get file size */ if (unlikely(fstat(fd_in, &st))) - fatal("bad magic file descriptor!?\n"); + fatal(control, "bad magic file descriptor!?\n"); infile_size = st.st_size; /* Get decompressed size */ @@ -865,12 +865,12 @@ void get_fileinfo(rzip_control *control) if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal("Failed to read chunk_byte in get_fileinfo\n"); + fatal(control, "Failed to read chunk_byte in get_fileinfo\n"); if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal("Failed to read eof in get_fileinfo\n"); + fatal(control, "Failed to read eof in get_fileinfo\n"); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal("Failed to read chunk_size in get_fileinfo\n"); + fatal(control, "Failed to read chunk_size in get_fileinfo\n"); chunk_size = le64toh(chunk_size); } } @@ -907,7 +907,7 @@ void get_fileinfo(rzip_control *control) int block = 1; if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET)) == -1) - fatal("Failed to seek to header data in get_fileinfo\n"); + fatal(control, "Failed to seek to header data in get_fileinfo\n"); get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte); print_verbose("Stream: %d\n", stream); @@ -917,13 +917,13 @@ void get_fileinfo(rzip_control *control) i64 head_off; if (unlikely(last_head + ofs > infile_size)) - failure("Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure(control, "Offset greater than archive size, likely corrupted/truncated archive.\n"); if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1) - fatal("Failed to seek to header data in get_fileinfo\n"); + fatal(control, "Failed to seek to header data in get_fileinfo\n"); get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte); if (unlikely(last_head < 0 || c_len < 0 || u_len < 0)) - failure("Entry negative, likely corrupted archive.\n"); + failure(control, "Entry negative, likely corrupted archive.\n"); print_verbose("%d\t", block); if (ctype == CTYPE_NONE) print_verbose("none"); @@ -950,20 +950,20 @@ void get_fileinfo(rzip_control *control) } if (unlikely((ofs = lseek(fd_in, c_len, SEEK_CUR)) == -1)) - fatal("Failed to lseek c_len in get_fileinfo\n"); + fatal(control, "Failed to lseek c_len in get_fileinfo\n"); if (ofs >= infile_size - (HAS_MD5 ? MD5_DIGEST_SIZE : 0)) goto done; /* Chunk byte entry */ if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal("Failed to read chunk_byte in get_fileinfo\n"); + fatal(control, "Failed to read chunk_byte in get_fileinfo\n"); ofs++; if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal("Failed to read eof in get_fileinfo\n"); + fatal(control, "Failed to read eof in get_fileinfo\n"); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal("Failed to read chunk_size in get_fileinfo\n"); + fatal(control, "Failed to read chunk_size in get_fileinfo\n"); chunk_size = le64toh(chunk_size); ofs += 1 + chunk_byte; header_length = 1 + (chunk_byte * 3); @@ -972,7 +972,7 @@ void get_fileinfo(rzip_control *control) goto next_chunk; done: if (unlikely(ofs > infile_size)) - failure("Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure(control, "Offset greater than archive size, likely corrupted/truncated archive.\n"); if (chunk_total > expected_size) expected_size = chunk_total; print_verbose("Rzip compression: %.1f%% %lld / %lld\n", @@ -1014,9 +1014,9 @@ void get_fileinfo(rzip_control *control) print_output("MD5 used for integrity testing\n"); if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1) - fatal("Failed to seek to md5 data in runzip_fd\n"); + fatal(control, "Failed to seek to md5 data in runzip_fd\n"); if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to read md5 data in runzip_fd\n"); + fatal(control, "Failed to read md5 data in runzip_fd\n"); print_output("MD5: "); for (i = 0; i < MD5_DIGEST_SIZE; i++) print_output("%02x", md5_stored[i] & 0xFF); @@ -1024,7 +1024,7 @@ void get_fileinfo(rzip_control *control) } else print_output("CRC32 used for integrity testing\n"); if (unlikely(close(fd_in))) - fatal("Failed to close fd_in in get_fileinfo\n"); + fatal(control, "Failed to close fd_in in get_fileinfo\n"); out: free(control->outfile); @@ -1057,7 +1057,7 @@ void compress_file(rzip_control *control) fd_in = open(control->infile, O_RDONLY); if (unlikely(fd_in == -1)) - fatal("Failed to open %s\n", control->infile); + fatal(control, "Failed to open %s\n", control->infile); } else fd_in = 0; @@ -1069,7 +1069,7 @@ void compress_file(rzip_control *control) else if ((tmp=strrchr(control->outname, '.')) && strcmp(tmp, control->suffix)) { control->outfile = malloc(strlen(control->outname) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal(control, "Failed to allocate outfile name\n"); strcpy(control->outfile, control->outname); strcat(control->outfile, control->suffix); print_output("Suffix added to %s.\nFull pathname is: %s\n", control->outname, control->outfile); @@ -1087,7 +1087,7 @@ void compress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpinfile) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal(control, "Failed to allocate outfile name\n"); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); @@ -1101,14 +1101,14 @@ void compress_file(rzip_control *control) fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink an existing file: %s\n", control->outfile); + fatal(control, "Failed to unlink an existing file: %s\n", control->outfile); fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal("Failed to create %s\n", control->outfile); + fatal(control, "Failed to create %s\n", control->outfile); } control->fd_out = fd_out; if (!STDIN) @@ -1118,7 +1118,7 @@ void compress_file(rzip_control *control) /* Write zeroes to header at beginning of file */ if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header))) - fatal("Cannot write file header\n"); + fatal(control, "Cannot write file header\n"); rzip_fd(control, fd_in, fd_out); @@ -1130,15 +1130,15 @@ void compress_file(rzip_control *control) release_hashes(control); if (unlikely(close(fd_in))) - fatal("Failed to close fd_in\n"); + fatal(control, "Failed to close fd_in\n"); if (unlikely(!STDOUT && close(fd_out))) - fatal("Failed to close fd_out\n"); + fatal(control, "Failed to close fd_out\n"); if (TMP_OUTBUF) close_tmpoutbuf(control); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal("Failed to unlink %s\n", control->infile); + fatal(control, "Failed to unlink %s\n", control->infile); } free(control->outfile); @@ -1151,11 +1151,11 @@ void initialize_control(rzip_control *control) memset(control, 0, sizeof(rzip_control)); control->msgout = stderr; - register_outputfile(control->msgout); + register_outputfile(control, control->msgout); control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; control->suffix = ".lrz"; control->compression_level = 7; - control->ramsize = get_ram(); + control->ramsize = get_ram(control); /* for testing single CPU */ control->threads = PROCESSORS; /* get CPUs for LZMA */ control->page_size = PAGE_SIZE; @@ -1165,10 +1165,10 @@ void initialize_control(rzip_control *control) * The next 2 bytes encode how many times to hash the password. * The last 9 bytes are random data, making 16 bytes of salt */ if (unlikely(gettimeofday(&tv, NULL))) - fatal("Failed to gettimeofday in main\n"); + fatal(control, "Failed to gettimeofday in main\n"); control->secs = tv.tv_sec; control->encloops = nloops(control->secs, control->salt, control->salt + 1); - get_rand(control->salt + 2, 6); + get_rand(control, control->salt + 2, 6); /* Get Temp Dir */ eptr = getenv("TMP"); @@ -1176,7 +1176,7 @@ void initialize_control(rzip_control *control) size_t len = strlen(eptr); control->tmpdir = malloc(len+2); if (control->tmpdir == NULL) - fatal("Failed to allocate for tmpdir\n"); + fatal(control, "Failed to allocate for tmpdir\n"); strcpy(control->tmpdir, eptr); if (eptr[len - 2] != '/') eptr[len - 2] = '/'; /* need a trailing slash */ diff --git a/lrzip.h b/lrzip.h index e263f204..9ff6649f 100644 --- a/lrzip.h +++ b/lrzip.h @@ -21,7 +21,7 @@ #include "lrzip_private.h" -inline i64 get_ram(void); +inline i64 get_ram(rzip_control *control); i64 nloops(i64 seconds, uchar *b1, uchar *b2); void write_magic(rzip_control *control); void read_magic(rzip_control *control, int fd_in, i64 *expected_size); diff --git a/lrzip_private.h b/lrzip_private.h index 02e2ac52..f0803234 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -203,6 +203,57 @@ typedef struct md5_ctx md5_ctx; #define MOORE_TIMES_PER_SECOND pow (MOORE, 1.0 / SECONDS_IN_A_YEAR) #define ARBITRARY_AT_EPOCH (ARBITRARY * pow (MOORE_TIMES_PER_SECOND, -T_ZERO)) +#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) +#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) +#define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA)) + +#define SHOW_PROGRESS (control->flags & FLAG_SHOW_PROGRESS) +#define KEEP_FILES (control->flags & FLAG_KEEP_FILES) +#define TEST_ONLY (control->flags & FLAG_TEST_ONLY) +#define FORCE_REPLACE (control->flags & FLAG_FORCE_REPLACE) +#define DECOMPRESS (control->flags & FLAG_DECOMPRESS) +#define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS) +#define LZO_COMPRESS (control->flags & FLAG_LZO_COMPRESS) +#define BZIP2_COMPRESS (control->flags & FLAG_BZIP2_COMPRESS) +#define ZLIB_COMPRESS (control->flags & FLAG_ZLIB_COMPRESS) +#define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS) +#define VERBOSE (control->flags & FLAG_VERBOSE) +#define VERBOSITY (control->flags & FLAG_VERBOSITY) +#define MAX_VERBOSE (control->flags & FLAG_VERBOSITY_MAX) +#define STDIN (control->flags & FLAG_STDIN) +#define STDOUT (control->flags & FLAG_STDOUT) +#define INFO (control->flags & FLAG_INFO) +#define UNLIMITED (control->flags & FLAG_UNLIMITED) +#define HASH_CHECK (control->flags & FLAG_HASH) +#define HAS_MD5 (control->flags & FLAG_MD5) +#define CHECK_FILE (control->flags & FLAG_CHECK) +#define KEEP_BROKEN (control->flags & FLAG_KEEP_BROKEN) +#define LZO_TEST (control->flags & FLAG_THRESHOLD) +#define TMP_OUTBUF (control->flags & FLAG_TMP_OUTBUF) +#define TMP_INBUF (control->flags & FLAG_TMP_INBUF) +#define ENCRYPT (control->flags & FLAG_ENCRYPT) + +#define print_output(format, args...) do {\ + fprintf(control->msgout, format, ##args); \ + fflush(control->msgout); \ +} while (0) + +#define print_progress(format, args...) do {\ + if (SHOW_PROGRESS) \ + print_output(format, ##args); \ +} while (0) + +#define print_verbose(format, args...) do {\ + if (VERBOSE) \ + print_output(format, ##args); \ +} while (0) + +#define print_maxverbose(format, args...) do {\ + if (MAX_VERBOSE) \ + print_output(format, ##args); \ +} while (0) + + #define print_err(format, args...) do {\ fprintf(stderr, format, ##args); \ } while (0) @@ -268,6 +319,11 @@ struct rzip_control { unsigned char magic_written; md5_ctx ctx; i64 md5_read; // How far into the file the md5 has done so far + const char *util_infile; + char delete_infile; + const char *util_outfile; + char delete_outfile; + FILE *outputfile; }; struct stream { diff --git a/main.c b/main.c index 654c12b1..9aae29f4 100644 --- a/main.c +++ b/main.c @@ -122,7 +122,7 @@ static void sighandler(int sig __UNUSED__) termios_p.c_lflag |= ECHO; tcsetattr(fileno(stdin), 0, &termios_p); - unlink_files(); + unlink_files(control); exit(0); } @@ -204,7 +204,7 @@ static void read_config(rzip_control *control) line = malloc(255); homeconf = malloc(255); if (line == NULL || homeconf == NULL) - fatal("Fatal Memory Error in read_config"); + fatal(control, "Fatal Memory Error in read_config"); fp = fopen("lrzip.conf", "r"); if (fp) @@ -255,11 +255,11 @@ static void read_config(rzip_control *control) } else if (isparameter(parameter, "compressionlevel")) { control->compression_level = atoi(parametervalue); if ( control->compression_level < 1 || control->compression_level > 9 ) - failure("CONF.FILE error. Compression Level must between 1 and 9"); + failure(control, "CONF.FILE error. Compression Level must between 1 and 9"); } else if (isparameter(parameter, "compressionmethod")) { /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ if (control->flags & FLAG_NOT_LZMA) - failure("CONF.FILE error. Can only specify one compression method"); + failure(control, "CONF.FILE error. Can only specify one compression method"); if (isparameter(parametervalue, "bzip2")) control->flags |= FLAG_BZIP2_COMPRESS; else if (isparameter(parametervalue, "gzip")) @@ -271,7 +271,7 @@ static void read_config(rzip_control *control) else if (isparameter(parametervalue, "zpaq")) control->flags |= FLAG_ZPAQ_COMPRESS; else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ - failure("CONF.FILE error. Invalid compression method %s specified\n",parametervalue); + failure(control, "CONF.FILE error. Invalid compression method %s specified\n",parametervalue); } else if (isparameter(parameter, "lzotest")) { /* default is yes */ if (isparameter(parametervalue, "no")) @@ -287,13 +287,13 @@ static void read_config(rzip_control *control) } else if (isparameter(parameter, "outputdirectory")) { control->outdir = malloc(strlen(parametervalue) + 2); if (!control->outdir) - fatal("Fatal Memory Error in read_config"); + fatal(control, "Fatal Memory Error in read_config"); strcpy(control->outdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->outdir, "/"); } else if (isparameter(parameter,"verbosity")) { if (control->flags & FLAG_VERBOSE) - failure("CONF.FILE error. Verbosity already defined."); + failure(control, "CONF.FILE error. Verbosity already defined."); if (isparameter(parametervalue, "yes")) control->flags |= FLAG_VERBOSITY; else if (isparameter(parametervalue,"max")) @@ -307,7 +307,7 @@ static void read_config(rzip_control *control) } else if (isparameter(parameter,"nice")) { control->nice_val = atoi(parametervalue); if (control->nice_val < -20 || control->nice_val > 19) - failure("CONF.FILE error. Nice must be between -20 and 19"); + failure(control, "CONF.FILE error. Nice must be between -20 and 19"); } else if (isparameter(parameter, "keepbroken")) { if (isparameter(parametervalue, "yes" )) control->flags |= FLAG_KEEP_BROKEN; @@ -322,7 +322,7 @@ static void read_config(rzip_control *control) } else if (isparameter(parameter, "tmpdir")) { control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); if (!control->tmpdir) - fatal("Fatal Memory Error in read_config"); + fatal(control, "Fatal Memory Error in read_config"); strcpy(control->tmpdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->tmpdir, "/"); @@ -336,7 +336,7 @@ static void read_config(rzip_control *control) } if (unlikely(fclose(fp))) - fatal("Failed to fclose fp in read_config\n"); + fatal(control, "Failed to fclose fp in read_config\n"); out: /* clean up */ free(line); @@ -387,7 +387,7 @@ int main(int argc, char *argv[]) switch (c) { case 'b': if (control->flags & FLAG_NOT_LZMA) - failure("Can only use one of -l, -b, -g, -z or -n\n"); + failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_BZIP2_COMPRESS; break; case 'c': @@ -408,7 +408,7 @@ int main(int argc, char *argv[]) break; case 'g': if (control->flags & FLAG_NOT_LZMA) - failure("Can only use one of -l, -b, -g, -z or -n\n"); + failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_ZLIB_COMPRESS; break; case 'h': @@ -426,40 +426,40 @@ int main(int argc, char *argv[]) break; case 'l': if (control->flags & FLAG_NOT_LZMA) - failure("Can only use one of -l, -b, -g, -z or -n\n"); + failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_LZO_COMPRESS; break; case 'L': control->compression_level = atoi(optarg); if (control->compression_level < 1 || control->compression_level > 9) - failure("Invalid compression level (must be 1-9)\n"); + failure(control, "Invalid compression level (must be 1-9)\n"); break; case 'n': if (control->flags & FLAG_NOT_LZMA) - failure("Can only use one of -l, -b, -g, -z or -n\n"); + failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_NO_COMPRESS; break; case 'N': control->nice_val = atoi(optarg); if (control->nice_val < -20 || control->nice_val > 19) - failure("Invalid nice value (must be -20..19)\n"); + failure(control, "Invalid nice value (must be -20..19)\n"); break; case 'o': if (control->outdir) - failure("Cannot have -o and -O together\n"); + failure(control, "Cannot have -o and -O together\n"); if (unlikely(STDOUT)) - failure("Cannot specify an output filename when outputting to stdout\n"); + failure(control, "Cannot specify an output filename when outputting to stdout\n"); control->outname = optarg; control->suffix = ""; break; case 'O': if (control->outname) /* can't mix -o and -O */ - failure("Cannot have options -o and -O together\n"); + failure(control, "Cannot have options -o and -O together\n"); if (unlikely(STDOUT)) - failure("Cannot specify an output directory when outputting to stdout\n"); + failure(control, "Cannot specify an output directory when outputting to stdout\n"); control->outdir = malloc(strlen(optarg) + 2); if (control->outdir == NULL) - fatal("Failed to allocate for outdir\n"); + fatal(control, "Failed to allocate for outdir\n"); strcpy(control->outdir,optarg); if (strcmp(optarg+strlen(optarg) - 1, "/")) /* need a trailing slash */ strcat(control->outdir, "/"); @@ -467,23 +467,23 @@ int main(int argc, char *argv[]) case 'p': control->threads = atoi(optarg); if (control->threads < 1) - failure("Must have at least one thread\n"); + failure(control, "Must have at least one thread\n"); break; case 'q': control->flags &= ~FLAG_SHOW_PROGRESS; break; case 'S': if (control->outname) - failure("Specified output filename already, can't specify an extension.\n"); + failure(control, "Specified output filename already, can't specify an extension.\n"); if (unlikely(STDOUT)) - failure("Cannot specify a filename suffix when outputting to stdout\n"); + failure(control, "Cannot specify a filename suffix when outputting to stdout\n"); control->suffix = optarg; break; case 't': if (control->outname) - failure("Cannot specify an output file name when just testing.\n"); + failure(control, "Cannot specify an output file name when just testing.\n"); if (!KEEP_FILES) - failure("Doubt that you want to delete a file when just testing.\n"); + failure(control, "Doubt that you want to delete a file when just testing.\n"); control->flags |= FLAG_TEST_ONLY; break; case 'T': @@ -510,7 +510,7 @@ int main(int argc, char *argv[]) break; case 'z': if (control->flags & FLAG_NOT_LZMA) - failure("Can only use one of -l, -b, -g, -z or -n\n"); + failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_ZPAQ_COMPRESS; break; } @@ -520,7 +520,7 @@ int main(int argc, char *argv[]) argv += optind; if (control->outname && argc > 1) - failure("Cannot specify output filename with more than 1 file\n"); + failure(control, "Cannot specify output filename with more than 1 file\n"); if (VERBOSE && !SHOW_PROGRESS) { print_err("Cannot have -v and -q options. -v wins.\n"); @@ -566,18 +566,18 @@ int main(int argc, char *argv[]) stat(control->infile, &infile_stat); if (unlikely(S_ISDIR(infile_stat.st_mode))) - failure("lrzip only works directly on FILES.\n" + failure(control, "lrzip only works directly on FILES.\n" "Use lrztar or pipe through tar for compressing directories.\n"); } } if (INFO && STDIN) - failure("Will not get file info from STDIN\n"); + failure(control, "Will not get file info from STDIN\n"); if (control->outname && (strcmp(control->outname, "-") == 0)) { control->flags |= FLAG_STDOUT; control->msgout = stderr; - register_outputfile(control->msgout); + register_outputfile(control, control->msgout); } /* If no output filename is specified, and we're using stdin, @@ -585,12 +585,12 @@ int main(int argc, char *argv[]) if (!control->outname && STDIN) { control->flags |= FLAG_STDOUT; control->msgout = stderr; - register_outputfile(control->msgout); + register_outputfile(control, control->msgout); } if (!STDOUT) { control->msgout = stdout; - register_outputfile(control->msgout); + register_outputfile(control, control->msgout); } /* Implement signal handler only once flags are set */ handler.sa_handler = &sighandler; @@ -626,7 +626,7 @@ int main(int argc, char *argv[]) gettimeofday(&start_time, NULL); if (unlikely(STDIN && ENCRYPT)) - failure("Unable to work from STDIN while reading password\n"); + failure(control, "Unable to work from STDIN while reading password\n"); if (DECOMPRESS || TEST_ONLY) decompress_file(control); diff --git a/runzip.c b/runzip.c index 62e090d8..e882c53f 100644 --- a/runzip.c +++ b/runzip.c @@ -54,7 +54,7 @@ static inline uchar read_u8(rzip_control *control, void *ss, int stream) uchar b; if (unlikely(read_stream(control, ss, stream, &b, 1) != 1)) - fatal("Stream read u8 failed\n"); + fatal(control, "Stream read u8 failed\n"); return b; } @@ -63,7 +63,7 @@ static inline u32 read_u32(rzip_control *control, void *ss, int stream) u32 ret; if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4)) - fatal("Stream read u32 failed\n"); + fatal(control, "Stream read u32 failed\n"); ret = le32toh(ret); return ret; } @@ -74,7 +74,7 @@ static inline i64 read_vchars(rzip_control *control, void *ss, int stream, int l i64 s = 0; if (unlikely(read_stream(control, ss, stream, (uchar *)&s, length) != length)) - fatal("Stream read of %d bytes failed\n", length); + fatal(control, "Stream read of %d bytes failed\n", length); s = le64toh(s); return s; } @@ -128,7 +128,7 @@ static i64 seekto_fdinend(rzip_control *control) while ((tmpchar = getchar()) != EOF) { control->tmp_inbuf[control->in_len++] = (char)tmpchar; if (unlikely(control->in_len > control->in_maxlen)) - failure("Trying to read greater than max_len\n"); + failure(control, "Trying to read greater than max_len\n"); } control->in_ofs = control->in_len; return control->in_ofs; @@ -151,18 +151,18 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, uint32 *cksum uchar *buf; if (unlikely(len < 0)) - failure("len %lld is negative in unzip_literal!\n",len); + failure(control, "len %lld is negative in unzip_literal!\n",len); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal("Failed to malloc literal buffer of size %lld\n", len); + fatal(control, "Failed to malloc literal buffer of size %lld\n", len); stream_read = read_stream(control, ss, 1, buf, len); if (unlikely(stream_read == -1 )) - fatal("Failed to read_stream in unzip_literal\n"); + fatal(control, "Failed to read_stream in unzip_literal\n"); if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) - fatal("Failed to write literal buffer of size %lld\n", stream_read); + fatal(control, "Failed to write literal buffer of size %lld\n", stream_read); if (!HAS_MD5) *cksum = CrcUpdate(*cksum, buf, stream_read); @@ -191,32 +191,32 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, uchar *buf, *off_buf; if (unlikely(len < 0)) - failure("len %lld is negative in unzip_match!\n",len); + failure(control, "len %lld is negative in unzip_match!\n",len); total = 0; cur_pos = seekcur_fdout(control); if (unlikely(cur_pos == -1)) - fatal("Seek failed on out file in unzip_match.\n"); + fatal(control, "Seek failed on out file in unzip_match.\n"); /* Note the offset is in a different format v0.40+ */ offset = read_vchars(control, ss, 0, chunk_bytes); if (unlikely(seekto_fdhist(control, cur_pos - offset) == -1)) - fatal("Seek failed by %d from %d on history file in unzip_match\n", + fatal(control, "Seek failed by %d from %d on history file in unzip_match\n", offset, cur_pos); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal("Failed to malloc match buffer of size %lld\n", len); + fatal(control, "Failed to malloc match buffer of size %lld\n", len); off_buf = buf; while (len) { n = MIN(len, offset); if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal("Failed to read %d bytes in unzip_match\n", n); + fatal(control, "Failed to read %d bytes in unzip_match\n", n); if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal("Failed to write %d bytes in unzip_match\n", n); + fatal(control, "Failed to write %d bytes in unzip_match\n", n); if (!HAS_MD5) *cksum = CrcUpdate(*cksum, off_buf, n); @@ -233,7 +233,7 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, return total; } -/* decompress a section of an open file. Call fatal() on error +/* decompress a section of an open file. Call fatal(control, ) on error return the number of bytes that have been retrieved */ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 tally) @@ -273,9 +273,9 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 else { /* Read in the stored chunk byte width from the file */ if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1)) - fatal("Failed to read chunk_bytes size in runzip_chunk\n"); + fatal(control, "Failed to read chunk_bytes size in runzip_chunk\n"); if (unlikely(chunk_bytes < 1 || chunk_bytes > 8)) - failure("chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes); + failure(control, "chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes); } if (!tally && expected_size) print_maxverbose("Expected size: %lld\n", expected_size); @@ -283,14 +283,14 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 ofs = seekcur_fdin(control); if (unlikely(ofs == -1)) - fatal("Failed to seek input file in runzip_fd\n"); + fatal(control, "Failed to seek input file in runzip_fd\n"); if (fstat(fd_in, &st) || st.st_size - ofs == 0) return 0; ss = open_stream_in(control, fd_in, NUM_STREAMS, chunk_bytes); if (unlikely(!ss)) - failure("Failed to open_stream_in in runzip_chunk\n"); + failure(control, "Failed to open_stream_in in runzip_chunk\n"); while ((len = read_header(control, ss, &head)) || head) { switch (head) { @@ -316,17 +316,17 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 if (!HAS_MD5) { good_cksum = read_u32(control, ss, 0); if (unlikely(good_cksum != cksum)) - failure("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum); + failure(control, "Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum); print_maxverbose("Checksum for block: 0x%08x\n", cksum); } if (unlikely(close_stream_in(ss))) - fatal("Failed to close stream!\n"); + fatal(control, "Failed to close stream!\n"); return total; } -/* Decompress an open file. Call fatal() on error +/* Decompress an open file. Call fatal(control, ) on error return the number of bytes that have been retrieved */ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size) @@ -365,12 +365,12 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp i64 fdinend = seekto_fdinend(control); if (unlikely(fdinend == -1)) - failure("Failed to seekto_fdinend in rzip_fd\n"); + failure(control, "Failed to seekto_fdinend in rzip_fd\n"); if (unlikely(seekto_fdin(control, fdinend - MD5_DIGEST_SIZE) == -1)) - failure("Failed to seekto_fdin in rzip_fd\n"); + failure(control, "Failed to seekto_fdin in rzip_fd\n"); if (unlikely(read_1g(control, fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to read md5 data in runzip_fd\n"); + fatal(control, "Failed to read md5 data in runzip_fd\n"); if (ENCRYPT) lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass); for (i = 0; i < MD5_DIGEST_SIZE; i++) @@ -381,7 +381,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_resblock[j] & 0xFF); - failure("\n"); + failure(control, "\n"); } } @@ -400,11 +400,11 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp close_tmpoutbuf(control); memcpy(md5_stored, md5_resblock, MD5_DIGEST_SIZE); if (unlikely(seekto_fdhist(control, 0) == -1)) - fatal("Failed to seekto_fdhist in runzip_fd\n"); + fatal(control, "Failed to seekto_fdhist in runzip_fd\n"); if (unlikely((md5_fstream = fdopen(fd_hist, "r")) == NULL)) - fatal("Failed to fdopen fd_hist in runzip_fd\n"); + fatal(control, "Failed to fdopen fd_hist in runzip_fd\n"); if (unlikely(md5_stream(md5_fstream, md5_resblock))) - fatal("Failed to md5_stream in runzip_fd\n"); + fatal(control, "Failed to md5_stream in runzip_fd\n"); /* We don't close the file here as it's closed in main */ for (i = 0; i < MD5_DIGEST_SIZE; i++) if (md5_stored[i] != md5_resblock[i]) { @@ -414,7 +414,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_resblock[j] & 0xFF); - failure("\n"); + failure(control, "\n"); } print_output("MD5 integrity of written file matches archive\n"); if (!HAS_MD5) diff --git a/rzip.c b/rzip.c index 58d2f31d..63514508 100644 --- a/rzip.c +++ b/rzip.c @@ -150,19 +150,19 @@ static void remap_low_sb(rzip_control *control) round_to_page(&new_offset); print_maxverbose("Sliding main buffer to offset %lld\n", new_offset); if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal("Failed to munmap in remap_low_sb\n"); + fatal(control, "Failed to munmap in remap_low_sb\n"); if (new_offset + sb.size_low > sb.orig_size) sb.size_low = sb.orig_size - new_offset; sb.offset_low = new_offset; sb.buf_low = (uchar *)mmap(sb.buf_low, sb.size_low, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_low); if (unlikely(sb.buf_low == MAP_FAILED)) - fatal("Failed to re mmap in remap_low_sb\n"); + fatal(control, "Failed to re mmap in remap_low_sb\n"); } static inline void remap_high_sb(rzip_control *control, i64 p) { if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal("Failed to munmap in remap_high_sb\n"); + fatal(control, "Failed to munmap in remap_high_sb\n"); sb.size_high = sb.high_length; /* In case we shrunk it when we hit the end of the file */ sb.offset_high = p; /* Make sure offset is rounded to page size of total offset */ @@ -171,7 +171,7 @@ static inline void remap_high_sb(rzip_control *control, i64 p) sb.size_high = sb.orig_size - sb.offset_high; sb.buf_high = (uchar *)mmap(sb.buf_high, sb.size_high, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_high); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal("Failed to re mmap in remap_high_sb\n"); + fatal(control, "Failed to re mmap in remap_high_sb\n"); } /* We use a "sliding mmap" to effectively read more than we can fit into the @@ -199,14 +199,14 @@ static uchar *get_sb(rzip_control *control, i64 p) static inline void put_u8(rzip_control *control, void *ss, uchar b) { if (unlikely(write_stream(control, ss, 0, &b, 1))) - fatal("Failed to put_u8\n"); + fatal(control, "Failed to put_u8\n"); } static inline void put_u32(rzip_control *control, void *ss, uint32_t s) { s = htole32(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, 4))) - fatal("Failed to put_u32\n"); + fatal(control, "Failed to put_u32\n"); } /* Put a variable length of bytes dependant on how big the chunk is */ @@ -214,7 +214,7 @@ static inline void put_vchars(rzip_control *control, void *ss, i64 s, int length { s = htole64(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, length))) - fatal("Failed to put_vchars\n"); + fatal(control, "Failed to put_vchars\n"); } static void put_header(rzip_control *control, void *ss, uchar head, i64 len) @@ -280,7 +280,7 @@ static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, put_header(control, st->ss, 0, len); if (unlikely(len && write_sbstream(control, st->ss, 1, last, len))) - fatal("Failed to write_stream in put_literal\n"); + fatal(control, "Failed to write_stream in put_literal\n"); last += len; } while (p > last); } @@ -544,7 +544,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct } if (unlikely(!st->hash_table)) - fatal("Failed to allocate hash table in hash_search\n"); + fatal(control, "Failed to allocate hash table in hash_search\n"); st->minimum_tag_mask = tag_mask; st->tag_clean_ptr = 0; @@ -621,7 +621,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal("Failed to malloc ckbuf in hash_search\n"); + fatal(control, "Failed to malloc ckbuf in hash_search\n"); for (i = 0; i < n; i++) memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1); st->cksum = CrcUpdate(st->cksum, ckbuf, n); @@ -643,7 +643,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal("Failed to malloc ckbuf in hash_search\n"); + fatal(control, "Failed to malloc ckbuf in hash_search\n"); for (i = 0; i < n; i++) memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1); st->cksum = CrcUpdate(st->cksum, ckbuf, n); @@ -688,7 +688,7 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) ret = MIN(len, one_g); ret = read(0, offset_buf, (size_t)ret); if (unlikely(ret < 0)) - fatal("Failed to read in mmap_stdin\n"); + fatal(control, "Failed to read in mmap_stdin\n"); total += ret; if (ret == 0) { /* Should be EOF */ @@ -703,7 +703,7 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) st->chunk_size = 0; } if (unlikely(buf == MAP_FAILED)) - fatal("Failed to remap to smaller buf in mmap_stdin\n"); + fatal(control, "Failed to remap to smaller buf in mmap_stdin\n"); control->eof = st->stdin_eof = 1; break; } @@ -723,7 +723,7 @@ static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int sb.high_length += control->page_size - (sb.high_length % control->page_size); sb.buf_high = (uchar *)mmap(NULL, sb.high_length, PROT_READ, MAP_SHARED, fd_in, offset); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal("Unable to mmap buf_high in init_sliding_mmap\n"); + fatal(control, "Unable to mmap buf_high in init_sliding_mmap\n"); sb.size_high = sb.high_length; sb.offset_high = 0; } @@ -743,21 +743,21 @@ static void rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, st->ss = open_stream_out(control, fd_out, NUM_STREAMS, st->chunk_size, st->chunk_bytes); if (unlikely(!st->ss)) - fatal("Failed to open streams in rzip_chunk\n"); + fatal(control, "Failed to open streams in rzip_chunk\n"); print_verbose("Beginning rzip pre-processing phase\n"); hash_search(control, st, pct_base, pct_multiple); /* unmap buffer before closing and reallocating streams */ if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal("Failed to munmap in rzip_chunk\n"); + fatal(control, "Failed to munmap in rzip_chunk\n"); if (!STDIN) { if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal("Failed to munmap in rzip_chunk\n"); + fatal(control, "Failed to munmap in rzip_chunk\n"); } if (unlikely(close_stream_out(control, st->ss))) - fatal("Failed to flush/close streams in rzip_chunk\n"); + fatal(control, "Failed to flush/close streams in rzip_chunk\n"); } /* compress a whole file chunks at a time */ @@ -784,15 +784,15 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) st = calloc(sizeof(*st), 1); if (unlikely(!st)) - fatal("Failed to allocate control state in rzip_fd\n"); + fatal(control, "Failed to allocate control state in rzip_fd\n"); if (LZO_COMPRESS) { if (unlikely(lzo_init() != LZO_E_OK)) - fatal("lzo_init() failed\n"); + fatal(control, "lzo_init() failed\n"); } if (unlikely(fstat(fd_in, &s))) - fatal("Failed to stat fd_in in rzip_fd\n"); + fatal(control, "Failed to stat fd_in in rzip_fd\n"); if (!STDIN) { len = control->st_size = s.st_size; @@ -805,13 +805,13 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) * compressed file, based on the compressed file being as large as the * uncompressed file. */ if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal("Failed to fstatvfs in compress_file\n"); + fatal(control, "Failed to fstatvfs in compress_file\n"); free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < control->st_size) { if (FORCE_REPLACE) print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); else - failure("Possibly inadequate free space to compress file, use -f to override.\n"); + failure(control, "Possibly inadequate free space to compress file, use -f to override.\n"); } } @@ -879,11 +879,11 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) /* Better to shrink the window to the largest size that works than fail */ if (sb.buf_low == MAP_FAILED) { if (unlikely(errno != ENOMEM)) - fatal("Failed to mmap %s\n", control->infile); + fatal(control, "Failed to mmap %s\n", control->infile); st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); if (unlikely(!st->mmap_size)) - fatal("Unable to mmap any ram\n"); + fatal(control, "Unable to mmap any ram\n"); goto retry; } st->chunk_size = st->mmap_size; @@ -893,11 +893,11 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) sb.buf_low = (uchar *)mmap(sb.buf_low, st->mmap_size, PROT_READ, MAP_SHARED, fd_in, offset); if (sb.buf_low == MAP_FAILED) { if (unlikely(errno != ENOMEM)) - fatal("Failed to mmap %s\n", control->infile); + fatal(control, "Failed to mmap %s\n", control->infile); st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); if (unlikely(!st->mmap_size)) - fatal("Unable to mmap any ram\n"); + fatal(control, "Unable to mmap any ram\n"); goto retry; } if (st->mmap_size < st->chunk_size) @@ -976,7 +976,7 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) last_chunk = st->chunk_size; len -= st->chunk_size; if (unlikely(len > 0 && control->eof)) - failure("Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"); + failure(control, "Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"); } close_streamout_threads(control); @@ -996,7 +996,7 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) if (ENCRYPT) lrz_encrypt(control, md5_resblock, MD5_DIGEST_SIZE, control->salt_pass); if (unlikely(write_1g(control, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to write md5 in rzip_fd\n"); + fatal(control, "Failed to write md5 in rzip_fd\n"); } if (TMP_OUTBUF) @@ -1026,3 +1026,11 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) free(st); } + +void rzip_control_free(rzip_control *control) +{ + if (!control) return; + + free(control->tmpdir); + free(control); +} diff --git a/rzip.h b/rzip.h index 9c10a656..29b2e17f 100644 --- a/rzip.h +++ b/rzip.h @@ -22,5 +22,6 @@ #include "lrzip_private.h" void rzip_fd(rzip_control *control, int fd_in, int fd_out); +void rzip_control_free(rzip_control *control); #endif diff --git a/stream.c b/stream.c index 9a6f923f..7f83c628 100644 --- a/stream.c +++ b/stream.c @@ -65,8 +65,8 @@ #ifdef __APPLE__ -# define fmemopen fake_fmemopen -# define open_memstream fake_open_memstream +# define fmemopen(s, len, modes) fake_fmemopen(control, (s), (len), (modes)) +# define open_memstream(bufloc, sizeloc) fake_open_memstream(control, (bufloc), (sizeloc)) # define memstream_update_buffer fake_open_memstream_update_buffer # define mremap fake_mremap #else @@ -105,53 +105,53 @@ static pthread_mutex_t output_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t output_cond = PTHREAD_COND_INITIALIZER; static pthread_t *threads; -static void init_mutex(pthread_mutex_t *mutex) +static void init_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_init(mutex, NULL))) - fatal("pthread_mutex_init failed"); + fatal(control, "pthread_mutex_init failed"); } -static void unlock_mutex(pthread_mutex_t *mutex) +static void unlock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_unlock(mutex))) - fatal("pthread_mutex_unlock failed"); + fatal(control, "pthread_mutex_unlock failed"); } -static void lock_mutex(pthread_mutex_t *mutex) +static void lock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_lock(mutex))) - fatal("pthread_mutex_lock failed"); + fatal(control, "pthread_mutex_lock failed"); } -static void cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) +static void cond_wait(rzip_control *control, pthread_cond_t *cond, pthread_mutex_t *mutex) { if (unlikely(pthread_cond_wait(cond, mutex))) - fatal("pthread_cond_wait failed"); + fatal(control, "pthread_cond_wait failed"); } -static void cond_broadcast(pthread_cond_t *cond) +static void cond_broadcast(rzip_control *control, pthread_cond_t *cond) { if (unlikely(pthread_cond_broadcast(cond))) - fatal("pthread_cond_broadcast failed"); + fatal(control, "pthread_cond_broadcast failed"); } -void create_pthread(pthread_t *thread, pthread_attr_t * attr, +void create_pthread(rzip_control *control, pthread_t *thread, pthread_attr_t * attr, void * (*start_routine)(void *), void *arg) { if (unlikely(pthread_create(thread, attr, start_routine, arg))) - fatal("pthread_create"); + fatal(control, "pthread_create"); } -void detach_pthread(pthread_t *thread) +void detach_pthread(rzip_control *control, pthread_t *thread) { if (unlikely(pthread_detach(*thread))) - fatal("pthread_detach"); + fatal(control, "pthread_detach"); } -void join_pthread(pthread_t th, void **thread_return) +void join_pthread(rzip_control *control, pthread_t th, void **thread_return) { if (pthread_join(th, thread_return)) - fatal("pthread_join"); + fatal(control, "pthread_join"); } /* just to keep things clean, declare function here @@ -159,12 +159,12 @@ void join_pthread(pthread_t th, void **thread_return) */ static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len); -static inline FILE *fake_fmemopen(void *buf, size_t buflen, char *mode) +static inline FILE *fake_fmemopen(rzip_control *control, void *buf, size_t buflen, char *mode) { FILE *in; if (unlikely(strcmp(mode, "r"))) - failure("fake_fmemopen only supports mode \"r\"."); + failure(control, "fake_fmemopen only supports mode \"r\"."); in = tmpfile(); if (unlikely(!in)) return NULL; @@ -174,12 +174,12 @@ static inline FILE *fake_fmemopen(void *buf, size_t buflen, char *mode) return in; } -static inline FILE *fake_open_memstream(char **buf, size_t *length) +static inline FILE *fake_open_memstream(rzip_control *control, char **buf, size_t *length) { FILE *out; if (unlikely(buf == NULL || length == NULL)) - failure("NULL parameter to fake_open_memstream"); + failure(control, "NULL parameter to fake_open_memstream"); out = tmpfile(); if (unlikely(!out)) return NULL; @@ -239,7 +239,7 @@ static int zpaq_compress_buf(rzip_control *control, struct compress_thread *cthr (int)(SHOW_PROGRESS), thread); if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal("Failed to memstream_update_buffer in zpaq_compress_buf"); + fatal(control, "Failed to memstream_update_buffer in zpaq_compress_buf"); fclose(in); fclose(out); @@ -501,7 +501,7 @@ static int zpaq_decompress_buf(rzip_control *control, struct uncomp_thread *ucth zpipe_decompress(in, out, control->msgout, ucthread->u_len, (int)(SHOW_PROGRESS), thread); if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal("Failed to memstream_update_buffer in zpaq_decompress_buf"); + fatal(control, "Failed to memstream_update_buffer in zpaq_decompress_buf"); fclose(in); fclose(out); @@ -715,7 +715,7 @@ static void read_fdin(struct rzip_control *control, i64 len) for (i = 0; i < len; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure("Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", + failure(control, "Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", len, i); control->tmp_inbuf[control->in_ofs + i] = (char)tmpchar; } @@ -747,7 +747,7 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) if (TMP_OUTBUF && fd == control->fd_out) { if (unlikely(control->out_ofs + len > control->out_maxlen)) - failure("Trying to read beyond out_ofs in tmpoutbuf\n"); + failure(control, "Trying to read beyond out_ofs in tmpoutbuf\n"); memcpy(buf, control->tmp_outbuf + control->out_ofs, len); control->out_ofs += len; return len; @@ -896,7 +896,7 @@ static i64 get_seek(rzip_control *control, int fd) return control->out_relofs + control->out_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal("Failed to lseek in get_seek\n"); + fatal(control, "Failed to lseek in get_seek\n"); return ret; } @@ -908,7 +908,7 @@ static i64 get_readseek(rzip_control *control, int fd) return control->in_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal("Failed to lseek in get_seek\n"); + fatal(control, "Failed to lseek in get_seek\n"); return ret; } @@ -926,14 +926,14 @@ void prepare_streamout_threads(rzip_control *control) control->threads = 1; threads = calloc(sizeof(pthread_t), control->threads); if (unlikely(!threads)) - fatal("Unable to calloc threads in prepare_streamout_threads\n"); + fatal(control, "Unable to calloc threads in prepare_streamout_threads\n"); cthread = calloc(sizeof(struct compress_thread), control->threads); if (unlikely(!cthread)) - fatal("Unable to calloc cthread in prepare_streamout_threads\n"); + fatal(control, "Unable to calloc cthread in prepare_streamout_threads\n"); for (i = 0; i < control->threads; i++) - init_mutex(&cthread[i].mutex); + init_mutex(control, &cthread[i].mutex); } @@ -944,7 +944,7 @@ void close_streamout_threads(rzip_control *control) /* Wait for the threads in the correct order in case they end up * serialised */ for (i = 0; i < control->threads; i++) { - lock_mutex(&cthread[close_thread].mutex); + lock_mutex(control, &cthread[close_thread].mutex); if (++close_thread == control->threads) close_thread = 0; } @@ -1045,7 +1045,7 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li for (i = 0; i < n; i++) { sinfo->s[i].buf = calloc(sinfo->bufsize , 1); if (unlikely(!sinfo->s[i].buf)) - fatal("Unable to malloc buffer of size %lld in open_stream_out\n", sinfo->bufsize); + fatal(control, "Unable to malloc buffer of size %lld in open_stream_out\n", sinfo->bufsize); } return (void *)sinfo; @@ -1095,7 +1095,7 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) ucthread = calloc(sizeof(struct uncomp_thread), total_threads); if (unlikely(!ucthread)) - fatal("Unable to calloc cthread in open_stream_in\n"); + fatal(control, "Unable to calloc cthread in open_stream_in\n"); sinfo->num_streams = n; sinfo->fd = f; @@ -1227,22 +1227,22 @@ static void rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, cur_ofs = get_seek(control, sinfo->fd) - sinfo->initial_pos; head = malloc(25 + SALT_LEN); if (unlikely(!head)) - fatal("Failed to malloc head in rewrite_encrypted\n"); + fatal(control, "Failed to malloc head in rewrite_encrypted\n"); buf = head + SALT_LEN; - get_rand(head, SALT_LEN); + get_rand(control, head, SALT_LEN); if (unlikely(seekto(control, sinfo, ofs - SALT_LEN))) - failure("Failed to seekto buf ofs in rewrite_encrypted\n"); + failure(control, "Failed to seekto buf ofs in rewrite_encrypted\n"); if (unlikely(write_buf(control, head, SALT_LEN))) - failure("Failed to write_buf head in rewrite_encrypted\n"); + failure(control, "Failed to write_buf head in rewrite_encrypted\n"); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - failure("Failed to read_buf buf in rewrite_encrypted\n"); + failure(control, "Failed to read_buf buf in rewrite_encrypted\n"); lrz_encrypt(control, buf, 25, head); if (unlikely(seekto(control, sinfo, ofs))) - failure("Failed to seek back to ofs in rewrite_encrypted\n"); + failure(control, "Failed to seek back to ofs in rewrite_encrypted\n"); if (unlikely(write_buf(control, buf, 25))) - failure("Failed to write_buf encrypted buf in rewrite_encrypted\n"); + failure(control, "Failed to write_buf encrypted buf in rewrite_encrypted\n"); free(head); seekto(control, sinfo, cur_ofs); } @@ -1287,7 +1287,7 @@ static void *compthread(void *data) ret = gzip_compress_buf(control, cti); else if (ZPAQ_COMPRESS) ret = zpaq_compress_buf(control, cti, i); - else failure("Dunno wtf compression to use!\n"); + else failure(control, "Dunno wtf compression to use!\n"); } padded_len = cti->c_len; @@ -1298,21 +1298,21 @@ static void *compthread(void *data) padded_len = MIN_SIZE; cti->s_buf = realloc(cti->s_buf, MIN_SIZE); if (unlikely(!cti->s_buf)) - fatal("Failed to realloc s_buf in compthread\n"); - get_rand(cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len); + fatal(control, "Failed to realloc s_buf in compthread\n"); + get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len); } /* If compression fails for whatever reason multithreaded, then wait * for the previous thread to finish, serialising the work to decrease * the memory requirements, increasing the chance of success */ if (unlikely(ret && waited)) - failure("Failed to compress in compthread\n"); + failure(control, "Failed to compress in compthread\n"); if (!waited) { - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); while (output_thread != i) - cond_wait(&output_cond, &output_lock); - unlock_mutex(&output_lock); + cond_wait(control, &output_cond, &output_lock); + unlock_mutex(control, &output_lock); waited = 1; } if (unlikely(ret)) { @@ -1352,7 +1352,7 @@ static void *compthread(void *data) * later */ if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal("Failed to write_buf blank salt in compthread %d\n", i); + fatal(control, "Failed to write_buf blank salt in compthread %d\n", i); ctis->cur_pos += SALT_LEN; } ctis->s[j].last_head = ctis->cur_pos + 1 + (write_len * 2); @@ -1365,23 +1365,23 @@ static void *compthread(void *data) } if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) - fatal("Failed to seekto in compthread %d\n", i); + fatal(control, "Failed to seekto in compthread %d\n", i); if (unlikely(write_val(control, ctis->cur_pos, write_len))) - fatal("Failed to write_val cur_pos in compthread %d\n", i); + fatal(control, "Failed to write_val cur_pos in compthread %d\n", i); if (ENCRYPT) rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17); ctis->s[cti->streamno].last_head = ctis->cur_pos + 1 + (write_len * 2) + (ENCRYPT ? SALT_LEN : 0); if (unlikely(seekto(control, ctis, ctis->cur_pos))) - fatal("Failed to seekto cur_pos in compthread %d\n", i); + fatal(control, "Failed to seekto cur_pos in compthread %d\n", i); print_maxverbose("Thread %ld writing %lld compressed bytes from stream %d\n", i, padded_len, cti->streamno); if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal("Failed to write_buf header salt in compthread %d\n", i); + fatal(control, "Failed to write_buf header salt in compthread %d\n", i); ctis->cur_pos += SALT_LEN; ctis->s[cti->streamno].last_headofs = ctis->cur_pos; } @@ -1390,30 +1390,30 @@ static void *compthread(void *data) write_val(control, cti->c_len, write_len) || write_val(control, cti->s_len, write_len) || write_val(control, 0, write_len))) { - fatal("Failed write in compthread %d\n", i); + fatal(control, "Failed write in compthread %d\n", i); } ctis->cur_pos += 1 + (write_len * 3); if (ENCRYPT) { - get_rand(cti->salt, SALT_LEN); + get_rand(control, cti->salt, SALT_LEN); if (unlikely(write_buf(control, cti->salt, SALT_LEN))) - fatal("Failed to write_buf block salt in compthread %d\n", i); + fatal(control, "Failed to write_buf block salt in compthread %d\n", i); lrz_encrypt(control, cti->s_buf, padded_len, cti->salt); ctis->cur_pos += SALT_LEN; } if (unlikely(write_buf(control, cti->s_buf, padded_len))) - fatal("Failed to write_buf s_buf in compthread %d\n", i); + fatal(control, "Failed to write_buf s_buf in compthread %d\n", i); ctis->cur_pos += padded_len; free(cti->s_buf); - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); if (++output_thread == control->threads) output_thread = 0; - cond_broadcast(&output_cond); - unlock_mutex(&output_lock); + cond_broadcast(control, &output_cond); + unlock_mutex(control, &output_lock); - unlock_mutex(&cti->mutex); + unlock_mutex(control, &cti->mutex); return 0; } @@ -1424,7 +1424,7 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s stream_thread_struct *s; /* Make sure this thread doesn't already exist */ - lock_mutex(&cthread[i].mutex); + lock_mutex(control, &cthread[i].mutex); cthread[i].sinfo = sinfo; cthread[i].streamno = streamno; @@ -1436,18 +1436,18 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s s = malloc(sizeof(stream_thread_struct)); if (unlikely(!s)) - fatal("Unable to malloc in clear_buffer"); + fatal(control, "Unable to malloc in clear_buffer"); s->i = i; s->control = control; - create_pthread(&threads[i], NULL, compthread, s); - detach_pthread(&threads[i]); + create_pthread(control, &threads[i], NULL, compthread, s); + detach_pthread(control, &threads[i]); if (newbuf) { /* The stream buffer has been given to the thread, allocate a * new one. */ sinfo->s[streamno].buf = malloc(sinfo->bufsize); if (unlikely(!sinfo->s[streamno].buf)) - fatal("Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize); + fatal(control, "Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize); sinfo->s[streamno].buflen = 0; } @@ -1494,7 +1494,7 @@ static void *ucompthread(void *data) ret = zpaq_decompress_buf(control, uci, i); break; default: - failure("Dunno wtf decompression type to use!\n"); + failure(control, "Dunno wtf decompression type to use!\n"); break; } } @@ -1503,15 +1503,15 @@ static void *ucompthread(void *data) * parallel */ if (unlikely(ret)) { if (unlikely(waited)) - failure("Failed to decompress in ucompthread\n"); + failure(control, "Failed to decompress in ucompthread\n"); print_maxverbose("Unable to decompress in parallel, waiting for previous thread to complete before trying again\n"); /* We do not strictly need to wait for this, so it's used when * decompression fails due to inadequate memory to try again * serialised. */ - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); while (output_thread != i) - cond_wait(&output_cond, &output_lock); - unlock_mutex(&output_lock); + cond_wait(control, &output_cond, &output_lock); + unlock_mutex(control, &output_lock); waited = 1; goto retry; } @@ -1536,7 +1536,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str goto out; fill_another: if (unlikely(ucthread[s->uthread_no].busy)) - failure("Trying to start a busy thread, this shouldn't happen!\n"); + failure(control, "Trying to start a busy thread, this shouldn't happen!\n"); if (unlikely(read_seekto(control, sinfo, s->last_head))) return -1; @@ -1589,7 +1589,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str s_buf = malloc(MAX(u_len, MIN_SIZE)); if (unlikely(u_len && !s_buf)) - fatal("Unable to malloc buffer of size %lld in fill_buffer\n", u_len); + fatal(control, "Unable to malloc buffer of size %lld in fill_buffer\n", u_len); sinfo->ram_alloced += u_len; if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len))) @@ -1612,10 +1612,10 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str st = malloc(sizeof(stream_thread_struct)); if (unlikely(!st)) - fatal("Unable to malloc in fill_buffer"); + fatal(control, "Unable to malloc in fill_buffer"); st->i = s->uthread_no; st->control = control; - create_pthread(&threads[s->uthread_no], NULL, ucompthread, st); + create_pthread(control, &threads[s->uthread_no], NULL, ucompthread, st); if (++s->uthread_no == s->base_thread + s->total_threads) s->uthread_no = s->base_thread; @@ -1630,13 +1630,13 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str sinfo->ram_alloced < control->maxram) goto fill_another; out: - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); output_thread = s->unext_thread; - cond_broadcast(&output_cond); - unlock_mutex(&output_lock); + cond_broadcast(control, &output_cond); + unlock_mutex(control, &output_lock); /* join_pthread here will make it wait till the data is ready */ - join_pthread(threads[s->unext_thread], NULL); + join_pthread(control, threads[s->unext_thread], NULL); ucthread[s->unext_thread].busy = 0; print_maxverbose("Taking decompressed data from thread %ld\n", s->unext_thread); @@ -1722,8 +1722,8 @@ int close_stream_out(rzip_control *control, void *ss) int close_thread = output_thread; for (i = 0; i < control->threads; i++) { - lock_mutex(&cthread[close_thread].mutex); - unlock_mutex(&cthread[close_thread].mutex); + lock_mutex(control, &cthread[close_thread].mutex); + unlock_mutex(control, &cthread[close_thread].mutex); if (++close_thread == control->threads) close_thread = 0; } @@ -1779,14 +1779,14 @@ static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len) return 1; wrkmem = (lzo_bytep) malloc(LZO1X_1_MEM_COMPRESS); if (unlikely(wrkmem == NULL)) - fatal("Unable to allocate wrkmem in lzo_compresses\n"); + fatal(control, "Unable to allocate wrkmem in lzo_compresses\n"); in_len = MIN(test_len, buftest_size); dlen = STREAM_BUFSIZE + STREAM_BUFSIZE / 16 + 64 + 3; c_buf = malloc(dlen); if (unlikely(!c_buf)) - fatal("Unable to allocate c_buf in lzo_compresses\n"); + fatal(control, "Unable to allocate c_buf in lzo_compresses\n"); /* Test progressively larger blocks at a time and as soon as anything compressible is found, jump out as a success */ diff --git a/util.c b/util.c index a92fa7ed..3ba7f959 100644 --- a/util.c +++ b/util.c @@ -57,40 +57,34 @@ #include "sha4.h" #include "aes.h" -static const char *infile = NULL; -static char delete_infile = 0; -static const char *outfile = NULL; -static char delete_outfile = 0; -static FILE *outputfile = NULL; - -void register_infile(const char *name, char delete) +void register_infile(rzip_control *control, const char *name, char delete) { - infile = name; - delete_infile = delete; + control->util_infile = name; + control->delete_infile = delete; } -void register_outfile(const char *name, char delete) +void register_outfile(rzip_control *control, const char *name, char delete) { - outfile = name; - delete_outfile = delete; + control->util_outfile = name; + control->delete_outfile = delete; } -void register_outputfile(FILE *f) +void register_outputfile(rzip_control *control, FILE *f) { - outputfile = f; + control->outputfile = f; } -void unlink_files(void) +void unlink_files(rzip_control *control) { /* Delete temporary files generated for testing or faking stdio */ - if (outfile && delete_outfile) - unlink(outfile); + if (control->util_outfile && control->delete_outfile) + unlink(control->util_outfile); - if (infile && delete_infile) - unlink(infile); + if (control->util_infile && control->delete_infile) + unlink(control->util_infile); } -static void fatal_exit(void) +static void fatal_exit(rzip_control *control) { struct termios termios_p; @@ -99,14 +93,14 @@ static void fatal_exit(void) termios_p.c_lflag |= ECHO; tcsetattr(fileno(stdin), 0, &termios_p); - unlink_files(); - fprintf(outputfile, "Fatal error - exiting\n"); - fflush(outputfile); + unlink_files(control); + fprintf(control->outputfile, "Fatal error - exiting\n"); + fflush(control->outputfile); abort(); } /* Failure when there is likely to be a meaningful error in perror */ -void fatal(const char *format, ...) +void fatal(const rzip_control *control, const char *format, ...) { va_list ap; @@ -117,10 +111,10 @@ void fatal(const char *format, ...) } perror(NULL); - fatal_exit(); + fatal_exit((rzip_control*)control); } -void failure(const char *format, ...) +void failure(const rzip_control *control, const char *format, ...) { va_list ap; @@ -130,7 +124,7 @@ void failure(const char *format, ...) va_end(ap); } - fatal_exit(); + fatal_exit((rzip_control*)control); } void setup_overhead(rzip_control *control) @@ -177,7 +171,7 @@ void round_to_page(i64 *size) *size = PAGE_SIZE; } -void get_rand(uchar *buf, int len) +void get_rand(rzip_control *control, uchar *buf, int len) { int fd, i; @@ -187,9 +181,9 @@ void get_rand(uchar *buf, int len) buf[i] = (uchar)random(); } else { if (unlikely(read(fd, buf, len) != len)) - fatal("Failed to read fd in get_rand\n"); + fatal(control, "Failed to read fd in get_rand\n"); if (unlikely(close(fd))) - fatal("Failed to close fd in get_rand\n"); + fatal(control, "Failed to close fd in get_rand\n"); } } @@ -243,7 +237,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa if (encrypt == LRZ_ENCRYPT) { print_maxverbose("Encrypting data \n"); if (unlikely(aes_setkey_enc(&aes_ctx, key, 128))) - failure("Failed to aes_setkey_enc in lrz_crypt\n"); + failure(control, "Failed to aes_setkey_enc in lrz_crypt\n"); aes_crypt_cbc(&aes_ctx, AES_ENCRYPT, N, iv, buf, buf); if (M) { @@ -256,7 +250,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa } } else { if (unlikely(aes_setkey_dec(&aes_ctx, key, 128))) - failure("Failed to aes_setkey_dec in lrz_crypt\n"); + failure(control, "Failed to aes_setkey_dec in lrz_crypt\n"); print_maxverbose("Decrypting data \n"); if (M) { aes_crypt_cbc(&aes_ctx, AES_DECRYPT, N - CBC_LEN, diff --git a/util.h b/util.h index d70a7a96..fe2167fd 100644 --- a/util.h +++ b/util.h @@ -21,16 +21,16 @@ #include "lrzip_private.h" -void register_infile(const char *name, char delete); -void register_outfile(const char *name, char delete); -void unlink_files(void); -void register_outputfile(FILE *f); -void fatal(const char *format, ...); -void failure(const char *format, ...); +void register_infile(rzip_control *control, const char *name, char delete); +void register_outfile(rzip_control *control, const char *name, char delete); +void unlink_files(rzip_control *control); +void register_outputfile(rzip_control *control, FILE *f); +void fatal(const rzip_control *control, const char *format, ...); +void failure(const rzip_control *control, const char *format, ...); void setup_overhead(rzip_control *control); void setup_ram(rzip_control *control); void round_to_page(i64 *size); -void get_rand(uchar *buf, int len); +void get_rand(rzip_control *control, uchar *buf, int len); void lrz_stretch(rzip_control *control); void lrz_stretch2(rzip_control *control); void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); From 08d2294e5ec8c6dfd551a0227f9878d0620ed394 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 03:41:22 -0400 Subject: [PATCH 08/30] add password callback hook --- lrzip.c | 42 +++++++++++++++++++++++------------------- lrzip_private.h | 2 ++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/lrzip.c b/lrzip.c index 2de89203..6edea410 100644 --- a/lrzip.c +++ b/lrzip.c @@ -538,34 +538,38 @@ static void get_hash(rzip_control *control, int make_hash) mlock(control->salt_pass, PASS_LEN); mlock(control->hash, HASH_LEN); - /* Disable stdin echo to screen */ - tcgetattr(fileno(stdin), &termios_p); - termios_p.c_lflag &= ~ECHO; - tcsetattr(fileno(stdin), 0, &termios_p); + if (control->pass_cb) { + control->pass_cb(control->pass_data, passphrase, PASS_LEN); + if (!passphrase[0]) fatal(control, "Supplied password was null!"); + } else { + /* Disable stdin echo to screen */ + tcgetattr(fileno(stdin), &termios_p); + termios_p.c_lflag &= ~ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); retry_pass: - print_output("Enter passphrase: "); - control->salt_pass_len = get_pass(control, passphrase) + SALT_LEN; - print_output("\n"); - if (make_hash) { - print_output("Re-enter passphrase: "); - get_pass(control, testphrase); + print_output("Enter passphrase: "); + control->salt_pass_len = get_pass(control, passphrase) + SALT_LEN; print_output("\n"); - if (strcmp(passphrase, testphrase)) { - print_output("Passwords do not match. Try again.\n"); - goto retry_pass; + if (make_hash) { + print_output("Re-enter passphrase: "); + get_pass(control, testphrase); + print_output("\n"); + if (strcmp(passphrase, testphrase)) { + print_output("Passwords do not match. Try again.\n"); + goto retry_pass; + } } + termios_p.c_lflag |= ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); + memset(testphrase, 0, PASS_LEN); } - termios_p.c_lflag |= ECHO; - tcsetattr(fileno(stdin), 0, &termios_p); - memset(testphrase, 0, PASS_LEN); - munlock(testphrase, PASS_LEN); - free(testphrase); - memcpy(control->salt_pass, control->salt, SALT_LEN); memcpy(control->salt_pass + SALT_LEN, passphrase, PASS_LEN - SALT_LEN); lrz_stretch(control); memset(passphrase, 0, PASS_LEN); munlock(passphrase, PASS_LEN); + munlock(testphrase, PASS_LEN); + free(testphrase); free(passphrase); } diff --git a/lrzip_private.h b/lrzip_private.h index f0803234..e96cc99e 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -311,6 +311,8 @@ struct rzip_control { int fd_hist; i64 encloops; i64 secs; + void (*pass_cb)(void *, char *, size_t); /* callback to get password in lib */ + void *pass_data; uchar salt[SALT_LEN]; uchar *salt_pass; int salt_pass_len; From 133b201867983eda6697e32f0a5e210b1b498f99 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Thu, 11 Aug 2011 05:44:16 -0400 Subject: [PATCH 09/30] move more code out of main.c, allocate outfile suffix --- lrzip.c | 2 +- lrzip_private.h | 2 +- main.c | 166 ------------------------------------------------ rzip.c | 1 + util.c | 165 +++++++++++++++++++++++++++++++++++++++++++++++ util.h | 1 + 6 files changed, 169 insertions(+), 168 deletions(-) diff --git a/lrzip.c b/lrzip.c index 6edea410..991b8706 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1157,7 +1157,7 @@ void initialize_control(rzip_control *control) control->msgout = stderr; register_outputfile(control, control->msgout); control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; - control->suffix = ".lrz"; + control->suffix = strdup(".lrz"); control->compression_level = 7; control->ramsize = get_ram(control); /* for testing single CPU */ diff --git a/lrzip_private.h b/lrzip_private.h index e96cc99e..1b48aaec 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -289,7 +289,7 @@ struct rzip_control { i64 in_len; i64 in_maxlen; FILE *msgout; //stream for output messages - const char *suffix; + char *suffix; uchar compression_level; i64 overhead; // compressor overhead i64 usable_ram; // the most ram we'll try to use on one activity diff --git a/main.c b/main.c index 9aae29f4..d0e8f48e 100644 --- a/main.c +++ b/main.c @@ -26,9 +26,6 @@ #ifdef HAVE_UNISTD_H # include #endif -#ifdef HAVE_CTYPE_H -# include -#endif #ifdef HAVE_SYS_TIME_H # include #endif @@ -61,10 +58,6 @@ /* needed for CRC routines */ #include "lzma/C/7zCrc.h" -/* Macros for testing parameters */ -#define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) -#define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) - static rzip_control controlstaticvariablehaha, *control; static void usage(void) @@ -192,165 +185,6 @@ static void show_summary(void) } } -static void read_config(rzip_control *control) -{ - /* check for lrzip.conf in ., $HOME/.lrzip and /etc/lrzip */ - char *HOME, *homeconf; - char *parametervalue; - char *parameter; - char *line; - FILE *fp; - - line = malloc(255); - homeconf = malloc(255); - if (line == NULL || homeconf == NULL) - fatal(control, "Fatal Memory Error in read_config"); - - fp = fopen("lrzip.conf", "r"); - if (fp) - fprintf(control->msgout, "Using configuration file ./lrzip.conf\n"); - if (fp == NULL) { - fp = fopen("/etc/lrzip/lrzip.conf", "r"); - if (fp) - fprintf(control->msgout, "Using configuration file /etc/lrzip/lrzip.conf\n"); - } - if (fp == NULL) { - HOME=getenv("HOME"); - if (HOME) { - strcpy(homeconf, HOME); - strcat(homeconf,"/.lrzip/lrzip.conf"); - fp = fopen(homeconf, "r"); - if (fp) - fprintf(control->msgout, "Using configuration file %s\n", homeconf); - } - } - if (fp == NULL) - goto out; - - /* if we get here, we have a file. read until no more. */ - - while ((fgets(line, 255, fp)) != NULL) { - if (strlen(line)) - line[strlen(line) - 1] = '\0'; - parameter = strtok(line, " ="); - if (parameter == NULL) - continue; - /* skip if whitespace or # */ - if (isspace(*parameter)) - continue; - if (*parameter == '#') - continue; - - parametervalue = strtok(NULL, " ="); - if (parametervalue == NULL) - continue; - - /* have valid parameter line, now assign to control */ - - if (isparameter(parameter, "window")) - control->window = atoi(parametervalue); - else if (isparameter(parameter, "unlimited")) { - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_UNLIMITED; - } else if (isparameter(parameter, "compressionlevel")) { - control->compression_level = atoi(parametervalue); - if ( control->compression_level < 1 || control->compression_level > 9 ) - failure(control, "CONF.FILE error. Compression Level must between 1 and 9"); - } else if (isparameter(parameter, "compressionmethod")) { - /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ - if (control->flags & FLAG_NOT_LZMA) - failure(control, "CONF.FILE error. Can only specify one compression method"); - if (isparameter(parametervalue, "bzip2")) - control->flags |= FLAG_BZIP2_COMPRESS; - else if (isparameter(parametervalue, "gzip")) - control->flags |= FLAG_ZLIB_COMPRESS; - else if (isparameter(parametervalue, "lzo")) - control->flags |= FLAG_LZO_COMPRESS; - else if (isparameter(parametervalue, "rzip")) - control->flags |= FLAG_NO_COMPRESS; - else if (isparameter(parametervalue, "zpaq")) - control->flags |= FLAG_ZPAQ_COMPRESS; - else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ - failure(control, "CONF.FILE error. Invalid compression method %s specified\n",parametervalue); - } else if (isparameter(parameter, "lzotest")) { - /* default is yes */ - if (isparameter(parametervalue, "no")) - control->flags &= ~FLAG_THRESHOLD; - } else if (isparameter(parameter, "hashcheck")) { - if (isparameter(parametervalue, "yes")) { - control->flags |= FLAG_CHECK; - control->flags |= FLAG_HASH; - } - } else if (isparameter(parameter, "showhash")) { - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_HASH; - } else if (isparameter(parameter, "outputdirectory")) { - control->outdir = malloc(strlen(parametervalue) + 2); - if (!control->outdir) - fatal(control, "Fatal Memory Error in read_config"); - strcpy(control->outdir, parametervalue); - if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) - strcat(control->outdir, "/"); - } else if (isparameter(parameter,"verbosity")) { - if (control->flags & FLAG_VERBOSE) - failure(control, "CONF.FILE error. Verbosity already defined."); - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_VERBOSITY; - else if (isparameter(parametervalue,"max")) - control->flags |= FLAG_VERBOSITY_MAX; - else /* oops, unrecognized value */ - print_err("lrzip.conf: Unrecognized verbosity value %s. Ignored.\n", parametervalue); - } else if (isparameter(parameter, "showprogress")) { - /* Yes by default */ - if (isparameter(parametervalue, "NO")) - control->flags &= ~FLAG_SHOW_PROGRESS; - } else if (isparameter(parameter,"nice")) { - control->nice_val = atoi(parametervalue); - if (control->nice_val < -20 || control->nice_val > 19) - failure(control, "CONF.FILE error. Nice must be between -20 and 19"); - } else if (isparameter(parameter, "keepbroken")) { - if (isparameter(parametervalue, "yes" )) - control->flags |= FLAG_KEEP_BROKEN; - } else if (iscaseparameter(parameter, "DELETEFILES")) { - /* delete files must be case sensitive */ - if (iscaseparameter(parametervalue, "YES")) - control->flags &= ~FLAG_KEEP_FILES; - } else if (iscaseparameter(parameter, "REPLACEFILE")) { - /* replace lrzip file must be case sensitive */ - if (iscaseparameter(parametervalue, "YES")) - control->flags |= FLAG_FORCE_REPLACE; - } else if (isparameter(parameter, "tmpdir")) { - control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); - if (!control->tmpdir) - fatal(control, "Fatal Memory Error in read_config"); - strcpy(control->tmpdir, parametervalue); - if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) - strcat(control->tmpdir, "/"); - } else if (isparameter(parameter, "encrypt")) { - if (isparameter(parameter, "YES")) - control->flags |= FLAG_ENCRYPT; - } else - /* oops, we have an invalid parameter, display */ - print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ - parameter, parametervalue); - } - - if (unlikely(fclose(fp))) - fatal(control, "Failed to fclose fp in read_config\n"); -out: - /* clean up */ - free(line); - free(homeconf); - -/* fprintf(stderr, "\nWindow = %d \ - \nCompression Level = %d \ - \nThreshold = %1.2f \ - \nOutput Directory = %s \ - \nFlags = %d\n", control->window,control->compression_level, control->threshold, control->outdir, control->flags); -*/ -} - - int main(int argc, char *argv[]) { struct timeval start_time, end_time; diff --git a/rzip.c b/rzip.c index 63514508..c0d4feb6 100644 --- a/rzip.c +++ b/rzip.c @@ -1032,5 +1032,6 @@ void rzip_control_free(rzip_control *control) if (!control) return; free(control->tmpdir); + free(control->suffix); free(control); } diff --git a/util.c b/util.c index 3ba7f959..153d2045 100644 --- a/util.c +++ b/util.c @@ -56,6 +56,13 @@ #include "util.h" #include "sha4.h" #include "aes.h" +#ifdef HAVE_CTYPE_H +# include +#endif + +/* Macros for testing parameters */ +#define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) +#define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) void register_infile(rzip_control *control, const char *name, char delete) { @@ -187,6 +194,164 @@ void get_rand(rzip_control *control, uchar *buf, int len) } } +void read_config(rzip_control *control) +{ + /* check for lrzip.conf in ., $HOME/.lrzip and /etc/lrzip */ + char *HOME, *homeconf; + char *parametervalue; + char *parameter; + char *line; + FILE *fp; + + line = malloc(255); + homeconf = malloc(255); + if (line == NULL || homeconf == NULL) + fatal(control, "Fatal Memory Error in read_config"); + + fp = fopen("lrzip.conf", "r"); + if (fp) + fprintf(control->msgout, "Using configuration file ./lrzip.conf\n"); + if (fp == NULL) { + fp = fopen("/etc/lrzip/lrzip.conf", "r"); + if (fp) + fprintf(control->msgout, "Using configuration file /etc/lrzip/lrzip.conf\n"); + } + if (fp == NULL) { + HOME=getenv("HOME"); + if (HOME) { + strcpy(homeconf, HOME); + strcat(homeconf,"/.lrzip/lrzip.conf"); + fp = fopen(homeconf, "r"); + if (fp) + fprintf(control->msgout, "Using configuration file %s\n", homeconf); + } + } + if (fp == NULL) + goto out; + + /* if we get here, we have a file. read until no more. */ + + while ((fgets(line, 255, fp)) != NULL) { + if (strlen(line)) + line[strlen(line) - 1] = '\0'; + parameter = strtok(line, " ="); + if (parameter == NULL) + continue; + /* skip if whitespace or # */ + if (isspace(*parameter)) + continue; + if (*parameter == '#') + continue; + + parametervalue = strtok(NULL, " ="); + if (parametervalue == NULL) + continue; + + /* have valid parameter line, now assign to control */ + + if (isparameter(parameter, "window")) + control->window = atoi(parametervalue); + else if (isparameter(parameter, "unlimited")) { + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_UNLIMITED; + } else if (isparameter(parameter, "compressionlevel")) { + control->compression_level = atoi(parametervalue); + if ( control->compression_level < 1 || control->compression_level > 9 ) + failure(control, "CONF.FILE error. Compression Level must between 1 and 9"); + } else if (isparameter(parameter, "compressionmethod")) { + /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ + if (control->flags & FLAG_NOT_LZMA) + failure(control, "CONF.FILE error. Can only specify one compression method"); + if (isparameter(parametervalue, "bzip2")) + control->flags |= FLAG_BZIP2_COMPRESS; + else if (isparameter(parametervalue, "gzip")) + control->flags |= FLAG_ZLIB_COMPRESS; + else if (isparameter(parametervalue, "lzo")) + control->flags |= FLAG_LZO_COMPRESS; + else if (isparameter(parametervalue, "rzip")) + control->flags |= FLAG_NO_COMPRESS; + else if (isparameter(parametervalue, "zpaq")) + control->flags |= FLAG_ZPAQ_COMPRESS; + else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ + failure(control, "CONF.FILE error. Invalid compression method %s specified\n",parametervalue); + } else if (isparameter(parameter, "lzotest")) { + /* default is yes */ + if (isparameter(parametervalue, "no")) + control->flags &= ~FLAG_THRESHOLD; + } else if (isparameter(parameter, "hashcheck")) { + if (isparameter(parametervalue, "yes")) { + control->flags |= FLAG_CHECK; + control->flags |= FLAG_HASH; + } + } else if (isparameter(parameter, "showhash")) { + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_HASH; + } else if (isparameter(parameter, "outputdirectory")) { + control->outdir = malloc(strlen(parametervalue) + 2); + if (!control->outdir) + fatal(control, "Fatal Memory Error in read_config"); + strcpy(control->outdir, parametervalue); + if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) + strcat(control->outdir, "/"); + } else if (isparameter(parameter,"verbosity")) { + if (control->flags & FLAG_VERBOSE) + failure(control, "CONF.FILE error. Verbosity already defined."); + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_VERBOSITY; + else if (isparameter(parametervalue,"max")) + control->flags |= FLAG_VERBOSITY_MAX; + else /* oops, unrecognized value */ + print_err("lrzip.conf: Unrecognized verbosity value %s. Ignored.\n", parametervalue); + } else if (isparameter(parameter, "showprogress")) { + /* Yes by default */ + if (isparameter(parametervalue, "NO")) + control->flags &= ~FLAG_SHOW_PROGRESS; + } else if (isparameter(parameter,"nice")) { + control->nice_val = atoi(parametervalue); + if (control->nice_val < -20 || control->nice_val > 19) + failure(control, "CONF.FILE error. Nice must be between -20 and 19"); + } else if (isparameter(parameter, "keepbroken")) { + if (isparameter(parametervalue, "yes" )) + control->flags |= FLAG_KEEP_BROKEN; + } else if (iscaseparameter(parameter, "DELETEFILES")) { + /* delete files must be case sensitive */ + if (iscaseparameter(parametervalue, "YES")) + control->flags &= ~FLAG_KEEP_FILES; + } else if (iscaseparameter(parameter, "REPLACEFILE")) { + /* replace lrzip file must be case sensitive */ + if (iscaseparameter(parametervalue, "YES")) + control->flags |= FLAG_FORCE_REPLACE; + } else if (isparameter(parameter, "tmpdir")) { + control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); + if (!control->tmpdir) + fatal(control, "Fatal Memory Error in read_config"); + strcpy(control->tmpdir, parametervalue); + if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) + strcat(control->tmpdir, "/"); + } else if (isparameter(parameter, "encrypt")) { + if (isparameter(parameter, "YES")) + control->flags |= FLAG_ENCRYPT; + } else + /* oops, we have an invalid parameter, display */ + print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ + parameter, parametervalue); + } + + if (unlikely(fclose(fp))) + fatal(control, "Failed to fclose fp in read_config\n"); +out: + /* clean up */ + free(line); + free(homeconf); + +/* fprintf(stderr, "\nWindow = %d \ + \nCompression Level = %d \ + \nThreshold = %1.2f \ + \nOutput Directory = %s \ + \nFlags = %d\n", control->window,control->compression_level, control->threshold, control->outdir, control->flags); +*/ +} + static void xor128 (void *pa, const void *pb) { i64 *a = pa; diff --git a/util.h b/util.h index fe2167fd..fed28f2c 100644 --- a/util.h +++ b/util.h @@ -31,6 +31,7 @@ void setup_overhead(rzip_control *control); void setup_ram(rzip_control *control); void round_to_page(i64 *size); void get_rand(rzip_control *control, uchar *buf, int len); +void read_config(rzip_control *control); void lrz_stretch(rzip_control *control); void lrz_stretch2(rzip_control *control); void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); From 4721863c4f1781d028de6b60d5d9176d38b8c293 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Fri, 12 Aug 2011 02:42:57 -0400 Subject: [PATCH 10/30] v0.00000000000000000000001 of liblrzip --- Makefile.am | 27 ++++- compress_demo.c | 42 ++++++++ decompress_demo.c | 42 ++++++++ liblrzip.c | 240 +++++++++++++++++++++++++++++++++++++++++++++ liblrzip_private.h | 20 ++++ 5 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 compress_demo.c create mode 100644 decompress_demo.c create mode 100644 liblrzip.c create mode 100644 liblrzip_private.h diff --git a/Makefile.am b/Makefile.am index 122ee05d..a71422c9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,20 +23,20 @@ m4/ltversion.m4 SUBDIRS = lzma man doc -AM_CFLAGS = -I lzma/C -DNDEBUG +AM_CFLAGS = -I. -I lzma/C -DNDEBUG +AM_CXXFLAGS = $(AM_CFLAGS) lrztardir = @bindir@ lrztar_SCRIPTS = lrztar -bin_PROGRAMS = lrzip -lrzip_SOURCES = \ +noinst_LTLIBRARIES = libtmplrzip.la +libtmplrzip_la_SOURCES = \ zpipe.cpp \ zpipe.h \ lrzip_private.h \ liblrzip.h \ lrzip.c \ lrzip.h \ - main.c \ rzip.h \ rzip.c \ runzip.c \ @@ -51,12 +51,29 @@ lrzip_SOURCES = \ aes.h \ sha4.c \ sha4.h +libtmplrzip_la_LIBADD = lzma/C/liblzma.la + + +lib_LTLIBRARIES = liblrzip.la +liblrzip_la_SOURCES = \ + liblrzip.c +nodist_EXTRA_liblrzip_la_SOURCES = dummy.cxx +liblrzip_la_LIBADD = libtmplrzip.la -lrzip_LDADD = lzma/C/liblzma.la +bin_PROGRAMS = lrzip +lrzip_SOURCES = \ + main.c +nodist_EXTRA_lrzip_SOURCES = dummyy.cxx + +lrzip_LDADD = libtmplrzip.la if STATIC lrzip_LDFLAGS = -all-static endif +noinst_PROGRAMS = decompress_demo +decompress_demo_SOURCES = decompress_demo.c +decompress_demo_LDADD = liblrzip.la + dist_doc_DATA = \ AUTHORS \ BUGS \ diff --git a/compress_demo.c b/compress_demo.c new file mode 100644 index 00000000..5f3bf4c6 --- /dev/null +++ b/compress_demo.c @@ -0,0 +1,42 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#undef NDEBUG +#include +#include +#include +#include +#include + +static const char *suffix_me(const char *file) +{ + const char *p; + static char buf[4096]; + + p = strrchr(file, '.'); + if (p && (strlen(p + 1) < 4)) + strncat(buf, file, p - file); + else + strcat(buf, file); + return &buf[0]; +} + +int main(int argc, char *argv[]) +{ + Lrzip *lr; + if ((argc != 2) && (argc != 3)) { + fprintf(stderr, "Usage: %s file [file [file [... [outfile]]]]\n", argv[0]); + exit(1); + } + lr = lrzip_new(LRZIP_MODE_DECOMPRESS); + assert(lr); + lrzip_config_env(lr); + assert(lrzip_file_add(lr, argv[1])); + if (argc == 2) { + lrzip_outfile_set(lr, suffix_me(argv[1])); + } + else + lrzip_outfile_set(lr, argv[2]); + assert(lrzip_run(lr)); + return 0; +} diff --git a/decompress_demo.c b/decompress_demo.c new file mode 100644 index 00000000..3e036260 --- /dev/null +++ b/decompress_demo.c @@ -0,0 +1,42 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#undef NDEBUG +#include +#include +#include +#include +#include + +static const char *suffix_me(const char *file) +{ + const char *p; + static char buf[4096]; + + p = strrchr(file, '.'); + if (p && (strlen(p + 1) < 4)) + strncat(buf, file, p - file); + else + strcat(buf, file); + return &buf[0]; +} + +int main(int argc, char *argv[]) +{ + Lrzip *lr; + if ((argc != 2) && (argc != 3)) { + fprintf(stderr, "Usage: %s file [outfile]\n", argv[0]); + exit(1); + } + lr = lrzip_new(LRZIP_MODE_DECOMPRESS); + assert(lr); + lrzip_config_env(lr); + assert(lrzip_file_add(lr, argv[1])); + if (argc == 2) { + lrzip_outfile_set(lr, suffix_me(argv[1])); + } + else + lrzip_outfile_set(lr, argv[2]); + assert(lrzip_run(lr)); + return 0; +} diff --git a/liblrzip.c b/liblrzip.c new file mode 100644 index 00000000..ac01c98f --- /dev/null +++ b/liblrzip.c @@ -0,0 +1,240 @@ +#include + +#ifdef HAVE_SYS_TIME_H +# include +#endif +#ifdef HAVE_SYS_RESOURCE_H +# include +#endif +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif + +/* needed for CRC routines */ +#include "lzma/C/7zCrc.h" +#include "util.h" +#include "lrzip.h" +#include "rzip.h" + +bool lrzip_init(void) +{ + /* generate crc table */ + CrcGenerateTable(); + return true; +} + +void lrzip_config_env(Lrzip *lr) +{ + const char *eptr; + /* Get Preloaded Defaults from lrzip.conf + * Look in ., $HOME/.lrzip/, /etc/lrzip. + * If LRZIP=NOCONFIG is set, then ignore config + */ + eptr = getenv("LRZIP"); + if (!eptr) + read_config(lr->control); + else if (!strstr(eptr,"NOCONFIG")) + read_config(lr->control); +} + +void lrzip_free(Lrzip *lr) +{ + if (!lr) return; + rzip_control_free(lr->control); + for (; lr->infiles && *lr->infiles; lr->infiles++) + free(*lr->infiles); + free(lr->infiles); + free(lr->outfile); + free(lr); +} + +Lrzip *lrzip_new(Lrzip_Mode mode) +{ + Lrzip *lr; + + lr = calloc(1, sizeof(Lrzip)); + if (unlikely(!lr)) return NULL; + lr->control = calloc(1, sizeof(rzip_control)); + if (unlikely(!lr->control)) goto error; + if (unlikely(!initialize_control(lr->control))) goto error; + lr->mode = mode; + lr->control->library_mode = 1; +#define MODE_CHECK(X) \ + case LRZIP_MODE_##X: \ + lr->control->flags |= FLAG_##X; \ + break + + switch (mode) { + MODE_CHECK(DECOMPRESS); + MODE_CHECK(NO_COMPRESS); + MODE_CHECK(LZO_COMPRESS); + MODE_CHECK(BZIP2_COMPRESS); + MODE_CHECK(ZLIB_COMPRESS); + MODE_CHECK(ZPAQ_COMPRESS); +#undef MODE_CHECK + default: + goto error; + } + setup_overhead(lr->control); + return lr; +error: + lrzip_free(lr); + return NULL; +} + +Lrzip_Mode lrzip_mode_get(Lrzip *lr) +{ + if (!lr) return LRZIP_MODE_NONE; + return lr->mode; +} + +char **lrzip_files_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->infiles; +} + +bool lrzip_file_add(Lrzip *lr, const char *file) +{ + if ((!lr) || (!file) || (!file[0])) return false; + + if (!lr->infile_buckets) { + /* no files added */ + lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(char*)); + lr->infile_buckets++; + } else if (lr->infile_idx == INFILE_BUCKET_SIZE * lr->infile_buckets + 1) { + /* all buckets full, create new bucket */ + char **tmp; + + tmp = realloc(lr->infiles, ++lr->infile_buckets * INFILE_BUCKET_SIZE + 1); + if (!tmp) return false; + lr->infiles = tmp; + } + + lr->infiles[lr->infile_idx++] = strdup(file); + return true; +} + +bool lrzip_file_del(Lrzip *lr, const char *file) +{ + size_t x; + + if ((!lr) || (!file) || (!file[0])) return false; + if (!lr->infile_buckets) return true; + + for (x = 0; x <= lr->infile_idx + 1; x++) { + if (!lr->infiles[x]) return true; /* not found */ + if (strcmp(lr->infiles[x], file)) continue; /* not a match */ + free(lr->infiles[x]); + break; + } + /* update index */ + for (; x < lr->infile_idx; x++) + lr->infiles[x] = lr->infiles[x] + 1; + lr->infile_idx--; + return true; +} + +void lrzip_outfile_set(Lrzip *lr, const char *file) +{ + if ((!lr) || (!file) || (!file[0])) return; + if (lr->outfile && (!strcmp(lr->outfile, file))) return; + free(lr->outfile); + lr->outfile = strdup(file); + lr->control->outname = lr->outfile; +} + +const char *lrzip_outfile_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->outfile; +} + +bool lrzip_run(Lrzip *lr) +{ + struct timeval start_time, end_time; + rzip_control *control; + double seconds,total_time; // for timers + size_t i; + int hours,minutes; + + if (!lr) return false; + control = lr->control; + /* One extra iteration for the case of no parameters means we will default to stdin/out */ + for (i = 0; i < lr->infile_idx; i++) { + lr->control->infile = lr->infiles[i]; + if (lr->control->infile) { + if ((strcmp(lr->control->infile, "-") == 0)) + lr->control->flags |= FLAG_STDIN; + else { + struct stat infile_stat; + + stat(lr->control->infile, &infile_stat); + if (unlikely(S_ISDIR(infile_stat.st_mode))) { + print_err("lrzip only works directly on FILES.\n" + "Use lrztar or pipe through tar for compressing directories.\n"); + return false; + } + } + } + + if (lr->control->outname && (strcmp(lr->control->outname, "-") == 0)) { + lr->control->flags |= FLAG_STDOUT; + lr->control->msgout = stderr; + register_outputfile(lr->control, lr->control->msgout); + } + + /* If no output filename is specified, and we're using stdin, + * use stdout */ + if (!lr->control->outname && STDIN) { + lr->control->flags |= FLAG_STDOUT; + lr->control->msgout = stderr; + register_outputfile(lr->control, lr->control->msgout); + } + + if (!STDOUT) { + lr->control->msgout = stdout; + register_outputfile(lr->control, lr->control->msgout); + } + + if (CHECK_FILE) { + if (!DECOMPRESS) { + print_err("Can only check file written on decompression.\n"); + lr->control->flags &= ~FLAG_CHECK; + } else if (STDOUT) { + print_err("Can't check file written when writing to stdout. Checking disabled.\n"); + lr->control->flags &= ~FLAG_CHECK; + } + } + + setup_ram(lr->control); + + gettimeofday(&start_time, NULL); + + if (unlikely(ENCRYPT && (!lr->control->pass_cb))) { + print_err("No password callback set!\n"); + return false; + } + + if (DECOMPRESS || TEST_ONLY) + decompress_file(lr->control); + else if (INFO) + get_fileinfo(lr->control); + else + compress_file(lr->control); + + /* compute total time */ + gettimeofday(&end_time, NULL); + total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) - + (start_time.tv_sec + (double)start_time.tv_usec / 1000000); + hours = (int)total_time / 3600; + minutes = (int)(total_time / 60) % 60; + seconds = total_time - hours * 3600 - minutes * 60; + if (!INFO) + print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds); + } + return true; +} diff --git a/liblrzip_private.h b/liblrzip_private.h new file mode 100644 index 00000000..4ba061d5 --- /dev/null +++ b/liblrzip_private.h @@ -0,0 +1,20 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include +#include + +#define INFILE_BUCKET_SIZE 10 + +struct Lrzip +{ + Lrzip_Mode mode; + rzip_control *control; + + char *outfile; + + /* bucket allocation is used here to avoid frequent calls to realloc */ + char **infiles; + size_t infile_idx; + size_t infile_buckets; +}; From d7495cee61c68406adb9962e7937e11f51f0a4ce Mon Sep 17 00:00:00 2001 From: discomfitor Date: Fri, 12 Aug 2011 02:43:17 -0400 Subject: [PATCH 11/30] add alloca check --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 723c4287..d537df7e 100644 --- a/configure.ac +++ b/configure.ac @@ -52,6 +52,7 @@ AC_PROG_INSTALL AC_PROG_LN_S AC_SUBST(SHELL) AC_SYS_LARGEFILE +AC_FUNC_ALLOCA AC_CHECK_PROG([HAVE_POD2MAN], [pod2man], [yes]) AS_IF([test "$HAVE_POD2MAN" != "yes"], From 6dd20d735b8b56dabbe9c6fc5f66b2557c5eb140 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Fri, 12 Aug 2011 02:43:42 -0400 Subject: [PATCH 12/30] giant clusterfuck commit: *add and use alloca/strdupa in some spots *implement return values on almost every function *free() is now a macro which also nulls the freed pointer *when in 'library mode', fatal_exit() will not be called *fatal()/failure() are now macros which add line number and file for logging *all occurrences of fatal/failure in files other than main.c are now wrapped to use a logging callback (if specified) and then return *functions now clean up resources on returning with an error --- liblrzip.h | 29 ++++ lrzip.c | 345 +++++++++++++++++++++++++++--------------------- lrzip.h | 26 ++-- lrzip_private.h | 32 +++++ main.c | 42 +++--- runzip.c | 137 ++++++++++++------- rzip.c | 252 +++++++++++++++++++++++------------ rzip.h | 2 +- stream.c | 268 +++++++++++++++++++++++-------------- stream.h | 10 +- util.c | 89 ++++--------- util.h | 70 ++++++++-- 12 files changed, 808 insertions(+), 494 deletions(-) diff --git a/liblrzip.h b/liblrzip.h index d683b957..5ab2927a 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -20,4 +20,33 @@ #ifndef LIBLRZIP_H #define LIBLRZIP_H +#include + +typedef struct Lrzip Lrzip; + +typedef enum { + LRZIP_MODE_NONE, + LRZIP_MODE_DECOMPRESS, + LRZIP_MODE_NO_COMPRESS, + LRZIP_MODE_LZO_COMPRESS, + LRZIP_MODE_BZIP2_COMPRESS, + LRZIP_MODE_ZLIB_COMPRESS, + LRZIP_MODE_ZPAQ_COMPRESS +} Lrzip_Mode; + +typedef void (*Lrzip_Password_Cb)(void *, char **, size_t); + +bool lrzip_init(void); +void lrzip_config_env(Lrzip *lr); +void lrzip_free(Lrzip *lr); +Lrzip *lrzip_new(Lrzip_Mode mode); +Lrzip_Mode lrzip_mode_get(Lrzip *lr); +char **lrzip_files_get(Lrzip *lr); +bool lrzip_file_add(Lrzip *lr, const char *file); +bool lrzip_file_del(Lrzip *lr, const char *file); +void lrzip_outfile_set(Lrzip *lr, const char *file); +const char *lrzip_outfile_get(Lrzip *lr); +bool lrzip_run(Lrzip *lr); + #endif + diff --git a/lrzip.c b/lrzip.c index 991b8706..15673e1a 100644 --- a/lrzip.c +++ b/lrzip.c @@ -56,6 +56,8 @@ #define MAGIC_LEN (24) +static void release_hashes(rzip_control *control); + static i64 fdout_seekto(rzip_control *control, i64 pos) { if (TMP_OUTBUF) { @@ -100,14 +102,16 @@ inline i64 get_ram(rzip_control *control) /* Workaround for uclibc which doesn't properly support sysconf */ if(!(meminfo = fopen("/proc/meminfo", "r"))) - fatal(control, "fopen\n"); + fatal_return(("fopen\n"), -1); while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { - if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) - fatal(control, "Failed to fgets in get_ram\n"); + if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) { + fclose(meminfo); + fatal_return(("Failed to fgets in get_ram\n"), -1); + } } if (fclose(meminfo) == -1) - fatal(control, "fclose"); + fatal_return(("fclose"), -1); ramsize *= 1000; return ramsize; @@ -130,7 +134,7 @@ i64 nloops(i64 seconds, uchar *b1, uchar *b2) } -void write_magic(rzip_control *control) +bool write_magic(rzip_control *control) { char magic[MAGIC_LEN]; @@ -168,11 +172,12 @@ void write_magic(rzip_control *control) magic[22] = 1; if (unlikely(fdout_seekto(control, 0))) - fatal(control, "Failed to seek to BOF to write Magic Header\n"); + fatal_return(("Failed to seek to BOF to write Magic Header\n"), false); if (unlikely(put_fdout(control, magic, MAGIC_LEN) != MAGIC_LEN)) - fatal(control, "Failed to write magic header\n"); + fatal_return(("Failed to write magic header\n"), false); control->magic_written = 1; + return true; } static inline i64 enc_loops(uchar b1, uchar b2) @@ -180,14 +185,14 @@ static inline i64 enc_loops(uchar b1, uchar b2) return (i64)b2 << (i64)b1; } -static void get_magic(rzip_control *control, char *magic) +static bool get_magic(rzip_control *control, char *magic) { int encrypted, md5, i; i64 expected_size; uint32_t v; if (unlikely(strncmp(magic, "LRZI", 4))) - failure(control, "Not an lrzip file\n"); + failure_return(("Not an lrzip file\n"), false); memcpy(&control->major_version, &magic[4], 1); memcpy(&control->minor_version, &magic[5], 1); @@ -230,7 +235,7 @@ static void get_magic(rzip_control *control, char *magic) if (encrypted == 1) control->flags |= FLAG_ENCRYPT; else - failure(control, "Unknown encryption\n"); + failure_return(("Unknown encryption\n"), false); /* In encrypted files, the size field is used to store the salt * instead and the size is unknown, just like a STDOUT chunked * file */ @@ -242,34 +247,37 @@ static void get_magic(rzip_control *control, char *magic) print_output("Asked to decrypt a non-encrypted archive. Bypassing decryption.\n"); control->flags &= ~FLAG_ENCRYPT; } + return true; } -void read_magic(rzip_control *control, int fd_in, i64 *expected_size) +bool read_magic(rzip_control *control, int fd_in, i64 *expected_size) { char magic[MAGIC_LEN]; memset(magic, 0, sizeof(magic)); /* Initially read only st_size; + return true; } /* preserve ownership and permissions where possible */ -void preserve_perms(rzip_control *control, int fd_in, int fd_out) +bool preserve_perms(rzip_control *control, int fd_in, int fd_out) { struct stat st; if (unlikely(fstat(fd_in, &st))) - fatal(control, "Failed to fstat input file\n"); + fatal_return(("Failed to fstat input file\n"), false); if (unlikely(fchmod(fd_out, (st.st_mode & 0666)))) print_err("Warning, unable to set permissions on %s\n", control->outfile); - /* chown fail is not fatal */ + /* chown fail is not fatal_return(( */ if (unlikely(fchown(fd_out, st.st_uid, st.st_gid))) print_err("Warning, unable to set owner on %s\n", control->outfile); + return true; } /* Open a temporary outputfile to emulate stdout */ @@ -282,24 +290,24 @@ int open_tmpoutfile(rzip_control *control) if (control->tmpdir) { control->outfile = realloc(NULL, strlen(control->tmpdir) + 16); if (unlikely(!control->outfile)) - fatal(control, "Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), -1); strcpy(control->outfile, control->tmpdir); strcat(control->outfile, "lrzipout.XXXXXX"); } else { control->outfile = realloc(NULL, 16); if (unlikely(!control->outfile)) - fatal(control, "Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), -1); strcpy(control->outfile, "lrzipout.XXXXXX"); } fd_out = mkstemp(control->outfile); if (unlikely(fd_out == -1)) - fatal(control, "Failed to create out tmpfile: %s\n", control->outfile); + fatal_return(("Failed to create out tmpfile: %s\n", control->outfile), -1); register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); return fd_out; } -static void fwrite_stdout(rzip_control *control, void *buf, i64 len) +static bool fwrite_stdout(rzip_control *control, void *buf, i64 len) { uchar *offset_buf = buf; ssize_t ret; @@ -313,15 +321,16 @@ static void fwrite_stdout(rzip_control *control, void *buf, i64 len) ret = len; ret = fwrite(offset_buf, 1, ret, stdout); if (unlikely(ret <= 0)) - fatal(control, "Failed to fwrite in fwrite_stdout\n"); + fatal_return(("Failed to fwrite in fwrite_stdout\n"), false); len -= ret; offset_buf += ret; total += ret; } fflush(stdout); + return true; } -void write_fdout(rzip_control *control, void *buf, i64 len) +bool write_fdout(rzip_control *control, void *buf, i64 len) { uchar *offset_buf = buf; ssize_t ret; @@ -330,27 +339,30 @@ void write_fdout(rzip_control *control, void *buf, i64 len) ret = MIN(len, one_g); ret = write(control->fd_out, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal(control, "Failed to write to fd_out in write_fdout\n"); + fatal_return(("Failed to write to fd_out in write_fdout\n"), false);; len -= ret; offset_buf += ret; } + return true; } -void flush_tmpoutbuf(rzip_control *control) +bool flush_tmpoutbuf(rzip_control *control) { if (!TEST_ONLY) { print_maxverbose("Dumping buffer to physical file.\n"); - if (STDOUT) - fwrite_stdout(control, control->tmp_outbuf, control->out_len); - else - write_fdout(control, control->tmp_outbuf, control->out_len); + if (STDOUT) { + if (unlikely(!fwrite_stdout(control, control->tmp_outbuf, control->out_len))) return false; + } else { + if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) return false; + } } control->out_relofs += control->out_len; control->out_ofs = control->out_len = 0; + return true; } /* Dump temporary outputfile to perform stdout */ -void dump_tmpoutfile(rzip_control *control, int fd_out) +bool dump_tmpoutfile(rzip_control *control, int fd_out) { FILE *tmpoutfp; int tmpchar; @@ -359,7 +371,7 @@ void dump_tmpoutfile(rzip_control *control, int fd_out) fsync(fd_out); tmpoutfp = fdopen(fd_out, "r"); if (unlikely(tmpoutfp == NULL)) - fatal(control, "Failed to fdopen out tmpfile\n"); + fatal_return(("Failed to fdopen out tmpfile\n"), false); rewind(tmpoutfp); if (!TEST_ONLY) { @@ -371,12 +383,13 @@ void dump_tmpoutfile(rzip_control *control, int fd_out) } if (unlikely(ftruncate(fd_out, 0))) - fatal(control, "Failed to ftruncate fd_out in dump_tmpoutfile\n"); + fatal_return(("Failed to ftruncate fd_out in dump_tmpoutfile\n"), false); + return true; } /* Used if we're unable to read STDIN into the temporary buffer, shunts data * to temporary file */ -void write_fdin(rzip_control *control) +bool write_fdin(rzip_control *control) { uchar *offset_buf = control->tmp_inbuf; i64 len = control->in_len; @@ -386,10 +399,11 @@ void write_fdin(rzip_control *control) ret = MIN(len, one_g); ret = write(control->fd_in, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal(control, "Failed to write to fd_in in write_fdin\n"); + fatal_return(("Failed to write to fd_in in write_fdin\n"), false); len -= ret; offset_buf += ret; } + return true; } /* Open a temporary inputfile to perform stdin decompression */ @@ -400,28 +414,31 @@ int open_tmpinfile(rzip_control *control) if (control->tmpdir) { control->infile = malloc(strlen(control->tmpdir) + 15); if (unlikely(!control->infile)) - fatal(control, "Failed to allocate infile name\n"); + fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, control->tmpdir); strcat(control->infile, "lrzipin.XXXXXX"); } else { control->infile = malloc(15); if (unlikely(!control->infile)) - fatal(control, "Failed to allocate infile name\n"); + fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, "lrzipin.XXXXXX"); } fd_in = mkstemp(control->infile); if (unlikely(fd_in == -1)) - fatal(control, "Failed to create in tmpfile: %s\n", control->infile); + fatal_return(("Failed to create in tmpfile: %s\n", control->infile), -1); register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); /* Unlink temporary file immediately to minimise chance of files left - * lying around in cases of failure. */ - if (unlikely(unlink(control->infile))) - fatal(control, "Failed to unlink tmpfile: %s\n", control->infile); + * lying around in cases of failure_return((. */ + if (unlikely(unlink(control->infile))) { + fatal("Failed to unlink tmpfile: %s\n", control->infile); + close(fd_in); + return -1; + } return fd_in; } -static void read_tmpinmagic(rzip_control *control) +static bool read_tmpinmagic(rzip_control *control) { char magic[MAGIC_LEN]; int i, tmpchar; @@ -430,14 +447,14 @@ static void read_tmpinmagic(rzip_control *control) for (i = 0; i < 24; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure(control, "Reached end of file on STDIN prematurely on v05 magic read\n"); + failure_return(("Reached end of file on STDIN prematurely on v05 magic read\n"), false); magic[i] = (char)tmpchar; } - get_magic(control, magic); + return get_magic(control, magic); } /* Read data from stdin into temporary inputfile */ -void read_tmpinfile(rzip_control *control, int fd_in) +bool read_tmpinfile(rzip_control *control, int fd_in) { FILE *tmpinfp; int tmpchar; @@ -446,18 +463,19 @@ void read_tmpinfile(rzip_control *control, int fd_in) fprintf(control->msgout, "Copying from stdin.\n"); tmpinfp = fdopen(fd_in, "w+"); if (unlikely(tmpinfp == NULL)) - fatal(control, "Failed to fdopen in tmpfile\n"); + fatal_return(("Failed to fdopen in tmpfile\n"), false); while ((tmpchar = getchar()) != EOF) fputc(tmpchar, tmpinfp); fflush(tmpinfp); rewind(tmpinfp); + return true; } /* To perform STDOUT, we allocate a proportion of ram that is then used as * a pseudo-temporary file */ -static void open_tmpoutbuf(rzip_control *control) +static bool open_tmpoutbuf(rzip_control *control) { control->flags |= FLAG_TMP_OUTBUF; control->out_maxlen = control->maxram; @@ -465,9 +483,10 @@ static void open_tmpoutbuf(rzip_control *control) * fall back to a real temporary file */ control->tmp_outbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_outbuf)) - fatal(control, "Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); + fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false); if (!DECOMPRESS && !TEST_ONLY) - control->out_ofs = control->out_len = MAGIC_LEN; + control->out_ofs = control->out_len = MAGIC_LEN;\ + return true; } void close_tmpoutbuf(rzip_control *control) @@ -476,13 +495,14 @@ void close_tmpoutbuf(rzip_control *control) free(control->tmp_outbuf); } -static void open_tmpinbuf(rzip_control *control) +static bool open_tmpinbuf(rzip_control *control) { control->flags |= FLAG_TMP_INBUF; control->in_maxlen = control->maxram; control->tmp_inbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_inbuf)) - fatal(control, "Failed to malloc tmp_inbuf in open_tmpinbuf\n"); + fatal_return(("Failed to malloc tmp_inbuf in open_tmpinbuf\n"), false); + return true; } inline void clear_tmpinbuf(rzip_control *control) @@ -490,12 +510,13 @@ inline void clear_tmpinbuf(rzip_control *control) control->in_len = control->in_ofs = 0; } -void clear_tmpinfile(rzip_control *control) +bool clear_tmpinfile(rzip_control *control) { if (unlikely(lseek(control->fd_in, 0, SEEK_SET))) - fatal(control, "Failed to lseek on fd_in in clear_tmpinfile\n"); + fatal_return(("Failed to lseek on fd_in in clear_tmpinfile\n"), false); if (unlikely(ftruncate(control->fd_in, 0))) - fatal(control, "Failed to truncate fd_in in clear_tmpinfile\n"); + fatal_return(("Failed to truncate fd_in in clear_tmpinfile\n"), false); + return true; } void close_tmpinbuf(rzip_control *control) @@ -510,7 +531,7 @@ static int get_pass(rzip_control *control, char *s) memset(s, 0, PASS_LEN - SALT_LEN); if (unlikely(fgets(s, PASS_LEN - SALT_LEN, stdin) == NULL)) - failure(control, "Failed to retrieve passphrase\n"); + failure_return(("Failed to retrieve passphrase\n"), -1); len = strlen(s); if (len > 0 && ('\r' == s[len - 1] || '\n' == s[len - 1])) s[len - 1] = '\0'; @@ -518,11 +539,11 @@ static int get_pass(rzip_control *control, char *s) s[len - 2] = '\0'; len = strlen(s); if (unlikely(0 == len)) - failure(control, "Empty passphrase\n"); + failure_return(("Empty passphrase\n"), -1); return len; } -static void get_hash(rzip_control *control, int make_hash) +static bool get_hash(rzip_control *control, int make_hash) { char *passphrase, *testphrase; struct termios termios_p; @@ -531,8 +552,12 @@ static void get_hash(rzip_control *control, int make_hash) testphrase = calloc(PASS_LEN, 1); control->salt_pass = calloc(PASS_LEN, 1); control->hash = calloc(HASH_LEN, 1); - if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) - fatal(control, "Failed to calloc encrypt buffers in compress_file\n"); + if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) { + fatal("Failed to calloc encrypt buffers in compress_file\n"); + free(testphrase); + free(passphrase); + return false; + } mlock(passphrase, PASS_LEN); mlock(testphrase, PASS_LEN); mlock(control->salt_pass, PASS_LEN); @@ -540,7 +565,15 @@ static void get_hash(rzip_control *control, int make_hash) if (control->pass_cb) { control->pass_cb(control->pass_data, passphrase, PASS_LEN); - if (!passphrase[0]) fatal(control, "Supplied password was null!"); + if (!passphrase[0]) { + fatal("Supplied password was null!"); + munlock(passphrase, PASS_LEN); + munlock(testphrase, PASS_LEN); + free(testphrase); + free(passphrase); + release_hashes(control); + return false; + } } else { /* Disable stdin echo to screen */ tcgetattr(fileno(stdin), &termios_p); @@ -571,6 +604,7 @@ static void get_hash(rzip_control *control, int make_hash) munlock(testphrase, PASS_LEN); free(testphrase); free(passphrase); + return true; } static void release_hashes(rzip_control *control) @@ -586,7 +620,7 @@ static void release_hashes(rzip_control *control) /* decompress one file from the command line */ -void decompress_file(rzip_control *control) +bool decompress_file(rzip_control *control) { char *tmp, *tmpoutfile, *infilecopy = NULL; int fd_in, fd_out = -1, fd_hist = -1; @@ -602,15 +636,11 @@ void decompress_file(rzip_control *control) /* make sure infile has an extension. If not, add it * because manipulations may be made to input filename, set local ptr */ - infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); - if (unlikely(infilecopy == NULL)) - fatal(control, "Failed to allocate memory for infile suffix\n"); - else { - strcpy(infilecopy, control->infile); - strcat(infilecopy, control->suffix); - } + infilecopy = alloca(strlen(control->infile) + strlen(control->suffix) + 1); + strcpy(infilecopy, control->infile); + strcat(infilecopy, control->suffix); } else - infilecopy = strdup(control->infile); + infilecopy = strdupa(control->infile); /* regardless, infilecopy has the input filename */ } @@ -624,9 +654,9 @@ void decompress_file(rzip_control *control) * infilecopy, then remove suffix. */ if (control->outdir && (tmp = strrchr(infilecopy, '/'))) - tmpoutfile = strdup(tmp + 1); + tmpoutfile = strdupa(tmp + 1); else - tmpoutfile = strdup(infilecopy); + tmpoutfile = strdupa(infilecopy); /* remove suffix to make outfile name */ if ((tmp = strrchr(tmpoutfile, '.')) && !strcmp(tmp, control->suffix)) @@ -634,14 +664,13 @@ void decompress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpoutfile) + 1); if (unlikely(!control->outfile)) - fatal(control, "Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), false); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); strcat(control->outfile, tmpoutfile); } else strcpy(control->outfile, tmpoutfile); - free(tmpoutfile); } if (!STDOUT) @@ -650,15 +679,16 @@ void decompress_file(rzip_control *control) if (STDIN) { fd_in = open_tmpinfile(control); + if (unlikely(fd_in == -1)) return false; read_tmpinmagic(control); if (ENCRYPT) - failure(control, "Cannot decompress encrypted file from STDIN\n"); + failure_return(("Cannot decompress encrypted file from STDIN\n"), false); expected_size = control->st_size; - open_tmpinbuf(control); + if (unlikely(!open_tmpinbuf(control))) return false; } else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) { - fatal(control, "Failed to open %s\n", infilecopy); + fatal_return(("Failed to open %s\n", infilecopy), false); } } control->fd_in = fd_in; @@ -667,50 +697,50 @@ void decompress_file(rzip_control *control) fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal(control, "Failed to unlink an existing file: %s\n", control->outfile); + fatal_return(("Failed to unlink an existing file: %s\n", control->outfile), false); fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal(control, "Failed to create %s\n", control->outfile); + fatal_return(("Failed to create %s\n", control->outfile), false); } fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal(control, "Failed to open history file %s\n", control->outfile); + fatal_return(("Failed to open history file %s\n", control->outfile), false); /* Can't copy permissions from STDIN */ if (!STDIN) - preserve_perms(control, fd_in, fd_out); + if (unlikely(!preserve_perms(control, fd_in, fd_out))) return false; } else { fd_out = open_tmpoutfile(control); if (unlikely(fd_out == -1)) - fatal(control, "Failed to create %s\n", control->outfile); + fatal_return(("Failed to create %s\n", control->outfile), false); fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal(control, "Failed to open history file %s\n", control->outfile); + fatal_return(("Failed to open history file %s\n", control->outfile), false); /* Unlink temporary file as soon as possible */ if (unlikely(unlink(control->outfile))) - fatal(control, "Failed to unlink tmpfile: %s\n", control->outfile); + fatal_return(("Failed to unlink tmpfile: %s\n", control->outfile), false); } - open_tmpoutbuf(control); + if (unlikely(!open_tmpoutbuf(control))) return false; if (!STDIN) - read_magic(control, fd_in, &expected_size); + if (unlikely(!read_magic(control, fd_in, &expected_size))) return false; if (!STDOUT) { /* Check if there's enough free space on the device chosen to fit the * decompressed file. */ if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal(control, "Failed to fstatvfs in decompress_file\n"); + fatal_return(("Failed to fstatvfs in decompress_file\n"), false); free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < expected_size) { if (FORCE_REPLACE) print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n"); else - failure(control, "Inadequate free space to decompress file, use -f to override.\n"); + failure_return(("Inadequate free space to decompress file, use -f to override.\n"), false); } } control->fd_out = fd_out; @@ -725,16 +755,16 @@ void decompress_file(rzip_control *control) print_verbose("being used for integrity testing.\n"); if (ENCRYPT) - get_hash(control, 0); + if (unlikely(!get_hash(control, 0))) return false; print_progress("Decompressing...\n"); - runzip_fd(control, fd_in, fd_out, fd_hist, expected_size); + if (unlikely(runzip_fd(control, fd_in, fd_out, fd_hist, expected_size) < 0)) return false; if (STDOUT && !TMP_OUTBUF) - dump_tmpoutfile(control, fd_out); + if (unlikely(!dump_tmpoutfile(control, fd_out))) return false; - /* if we get here, no fatal errors during decompression */ + /* if we get here, no fatal_return(( errors during decompression */ print_progress("\r"); if (!(STDOUT | TEST_ONLY)) print_output("Output filename is: %s: ", control->outfile); @@ -746,38 +776,38 @@ void decompress_file(rzip_control *control) print_progress("[OK] \n"); if (unlikely(close(fd_hist) || close(fd_out))) - fatal(control, "Failed to close files\n"); + fatal_return(("Failed to close files\n"), false); close(fd_in); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal(control, "Failed to unlink %s\n", infilecopy); + fatal_return(("Failed to unlink %s\n", infilecopy), false); } if (ENCRYPT) release_hashes(control); free(control->outfile); - free(infilecopy); + return true; } -void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, +bool get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head, int chunk_bytes) { if (unlikely(read(fd_in, ctype, 1) != 1)) - fatal(control, "Failed to read in get_header_info\n"); + fatal_return(("Failed to read in get_header_info\n"), false); *c_len = *u_len = *last_head = 0; if (control->major_version == 0 && control->minor_version < 4) { u32 c_len32, u_len32, last_head32; if (unlikely(read(fd_in, &c_len32, 4) != 4)) - fatal(control, "Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, &u_len32, 4) != 4)) - fatal(control, "Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, &last_head32, 4) != 4)) - fatal(control, "Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); c_len32 = le32toh(c_len32); u_len32 = le32toh(u_len32); last_head32 = le32toh(last_head32); @@ -792,15 +822,16 @@ void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, else read_len = chunk_bytes; if (unlikely(read(fd_in, c_len, read_len) != read_len)) - fatal(control, "Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, u_len, read_len) != read_len)) - fatal(control, "Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, last_head, read_len) != read_len)) - fatal(control, "Failed to read_i64 in get_header_info"); + fatal_return(("Failed to read_i64 in get_header_info"), false); *c_len = le64toh(*c_len); *u_len = le64toh(*u_len); *last_head = le64toh(*last_head); } + return true; } static double percentage(i64 num, i64 den) @@ -817,7 +848,7 @@ static double percentage(i64 num, i64 den) return d_num / d_den; } -void get_fileinfo(rzip_control *control) +bool get_fileinfo(rzip_control *control) { i64 u_len, c_len, last_head, utotal = 0, ctotal = 0, ofs = 25, stream_head[2]; i64 expected_size, infile_size, chunk_size = 0, chunk_total = 0; @@ -835,15 +866,11 @@ void get_fileinfo(rzip_control *control) stat(control->infile, &fdin_stat); if (!S_ISREG(fdin_stat.st_mode) && (tmp = strrchr(control->infile, '.')) && strcmp(tmp,control->suffix)) { - infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); - if (unlikely(infilecopy == NULL)) - fatal(control, "Failed to allocate memory for infile suffix\n"); - else { - strcpy(infilecopy, control->infile); - strcat(infilecopy, control->suffix); - } + infilecopy = alloca(strlen(control->infile) + strlen(control->suffix) + 1); + strcpy(infilecopy, control->infile); + strcat(infilecopy, control->suffix); } else - infilecopy = strdup(control->infile); + infilecopy = strdupa(control->infile); } if (STDIN) @@ -851,30 +878,31 @@ void get_fileinfo(rzip_control *control) else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) - fatal(control, "Failed to open %s\n", infilecopy); + fatal_return(("Failed to open %s\n", infilecopy), false); } /* Get file size */ if (unlikely(fstat(fd_in, &st))) - fatal(control, "bad magic file descriptor!?\n"); + fatal_goto(("bad magic file descriptor!?\n"), error); infile_size = st.st_size; /* Get decompressed size */ - read_magic(control, fd_in, &expected_size); + if (unlikely(!read_magic(control, fd_in, &expected_size))) goto error; if (ENCRYPT) { print_output("Encrypted lrzip archive. No further information available\n"); + if (!STDIN) close(fd_in); goto out; } if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal(control, "Failed to read chunk_byte in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error); if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal(control, "Failed to read eof in get_fileinfo\n"); + fatal_goto(("Failed to read eof in get_fileinfo\n"), error); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal(control, "Failed to read chunk_size in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error); chunk_size = le64toh(chunk_size); } } @@ -911,8 +939,8 @@ void get_fileinfo(rzip_control *control) int block = 1; if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET)) == -1) - fatal(control, "Failed to seek to header data in get_fileinfo\n"); - get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte); + fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error); + if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte))) return false; print_verbose("Stream: %d\n", stream); print_maxverbose("Offset: %lld\n", ofs); @@ -921,13 +949,13 @@ void get_fileinfo(rzip_control *control) i64 head_off; if (unlikely(last_head + ofs > infile_size)) - failure(control, "Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error); if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1) - fatal(control, "Failed to seek to header data in get_fileinfo\n"); - get_header_info(control, fd_in, &ctype, &c_len, &u_len, - &last_head, chunk_byte); + fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error); + if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len, + &last_head, chunk_byte))) return false; if (unlikely(last_head < 0 || c_len < 0 || u_len < 0)) - failure(control, "Entry negative, likely corrupted archive.\n"); + failure_goto(("Entry negative, likely corrupted archive.\n"), error); print_verbose("%d\t", block); if (ctype == CTYPE_NONE) print_verbose("none"); @@ -954,20 +982,20 @@ void get_fileinfo(rzip_control *control) } if (unlikely((ofs = lseek(fd_in, c_len, SEEK_CUR)) == -1)) - fatal(control, "Failed to lseek c_len in get_fileinfo\n"); + fatal_goto(("Failed to lseek c_len in get_fileinfo\n"), error); if (ofs >= infile_size - (HAS_MD5 ? MD5_DIGEST_SIZE : 0)) goto done; /* Chunk byte entry */ if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal(control, "Failed to read chunk_byte in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error); ofs++; if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal(control, "Failed to read eof in get_fileinfo\n"); + fatal_goto(("Failed to read eof in get_fileinfo\n"), error); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal(control, "Failed to read chunk_size in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error); chunk_size = le64toh(chunk_size); ofs += 1 + chunk_byte; header_length = 1 + (chunk_byte * 3); @@ -976,7 +1004,7 @@ void get_fileinfo(rzip_control *control) goto next_chunk; done: if (unlikely(ofs > infile_size)) - failure(control, "Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error); if (chunk_total > expected_size) expected_size = chunk_total; print_verbose("Rzip compression: %.1f%% %lld / %lld\n", @@ -1018,9 +1046,9 @@ void get_fileinfo(rzip_control *control) print_output("MD5 used for integrity testing\n"); if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1) - fatal(control, "Failed to seek to md5 data in runzip_fd\n"); + fatal_goto(("Failed to seek to md5 data in runzip_fd\n"), error); if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal(control, "Failed to read md5 data in runzip_fd\n"); + fatal_goto(("Failed to read md5 data in runzip_fd\n"), error); print_output("MD5: "); for (i = 0; i < MD5_DIGEST_SIZE; i++) print_output("%02x", md5_stored[i] & 0xFF); @@ -1028,40 +1056,43 @@ void get_fileinfo(rzip_control *control) } else print_output("CRC32 used for integrity testing\n"); if (unlikely(close(fd_in))) - fatal(control, "Failed to close fd_in in get_fileinfo\n"); + fatal_return(("Failed to close fd_in in get_fileinfo\n"), false); out: free(control->outfile); - free(infilecopy); + return true; +error: + if (!STDIN) close(fd_in); + return false; } /* compress one file from the command line */ -void compress_file(rzip_control *control) +bool compress_file(rzip_control *control) { const char *tmp, *tmpinfile; /* we're just using this as a proxy for control->infile. * Spares a compiler warning */ - int fd_in, fd_out = -1; + int fd_in = -1, fd_out = -1; char header[MAGIC_LEN]; if (MD5_RELIABLE) control->flags |= FLAG_MD5; if (ENCRYPT) - get_hash(control, 1); + if (unlikely(!get_hash(control, 1))) return false; memset(header, 0, sizeof(header)); if (!STDIN) { /* is extension at end of infile? */ if ((tmp = strrchr(control->infile, '.')) && !strcmp(tmp, control->suffix)) { print_err("%s: already has %s suffix. Skipping...\n", control->infile, control->suffix); - return; + return false; } fd_in = open(control->infile, O_RDONLY); if (unlikely(fd_in == -1)) - fatal(control, "Failed to open %s\n", control->infile); + fatal_return(("Failed to open %s\n", control->infile), false); } else fd_in = 0; @@ -1073,7 +1104,7 @@ void compress_file(rzip_control *control) else if ((tmp=strrchr(control->outname, '.')) && strcmp(tmp, control->suffix)) { control->outfile = malloc(strlen(control->outname) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal(control, "Failed to allocate outfile name\n"); + fatal_goto(("Failed to allocate outfile name\n"), error); strcpy(control->outfile, control->outname); strcat(control->outfile, control->suffix); print_output("Suffix added to %s.\nFull pathname is: %s\n", control->outname, control->outfile); @@ -1091,7 +1122,7 @@ void compress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpinfile) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal(control, "Failed to allocate outfile name\n"); + fatal_goto(("Failed to allocate outfile name\n"), error); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); @@ -1105,50 +1136,58 @@ void compress_file(rzip_control *control) fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal(control, "Failed to unlink an existing file: %s\n", control->outfile); + fatal_goto(("Failed to unlink an existing file: %s\n", control->outfile), error); fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal(control, "Failed to create %s\n", control->outfile); + fatal_goto(("Failed to create %s\n", control->outfile), error); } control->fd_out = fd_out; if (!STDIN) - preserve_perms(control, fd_in, fd_out); + if (unlikely(!preserve_perms(control, fd_in, fd_out))) goto error; } else - open_tmpoutbuf(control); + if (unlikely(!open_tmpoutbuf(control))) goto error; /* Write zeroes to header at beginning of file */ if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header))) - fatal(control, "Cannot write file header\n"); + fatal_goto(("Cannot write file header\n"), error); rzip_fd(control, fd_in, fd_out); /* Wwrite magic at end b/c lzma does not tell us properties until it is done */ if (!STDOUT) - write_magic(control); + if (unlikely(!write_magic(control))) goto error; if (ENCRYPT) release_hashes(control); - if (unlikely(close(fd_in))) - fatal(control, "Failed to close fd_in\n"); + if (unlikely(close(fd_in))) { + fatal("Failed to close fd_in\n"); + fd_in = -1; + goto error; + } if (unlikely(!STDOUT && close(fd_out))) - fatal(control, "Failed to close fd_out\n"); + fatal_return(("Failed to close fd_out\n"), false); if (TMP_OUTBUF) close_tmpoutbuf(control); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal(control, "Failed to unlink %s\n", control->infile); + fatal_return(("Failed to unlink %s\n", control->infile), false); } free(control->outfile); + return true; +error: + if (STDIN && (fd_in > 0)) close(fd_in); + if ((!STDOUT) && (fd_out > 0)) close(fd_out); + return false; } -void initialize_control(rzip_control *control) +bool initialize_control(rzip_control *control) { struct timeval tv; char *eptr; /* for environment */ @@ -1160,6 +1199,7 @@ void initialize_control(rzip_control *control) control->suffix = strdup(".lrz"); control->compression_level = 7; control->ramsize = get_ram(control); + if (unlikely(control->ramsize == -1)) return false; /* for testing single CPU */ control->threads = PROCESSORS; /* get CPUs for LZMA */ control->page_size = PAGE_SIZE; @@ -1169,10 +1209,10 @@ void initialize_control(rzip_control *control) * The next 2 bytes encode how many times to hash the password. * The last 9 bytes are random data, making 16 bytes of salt */ if (unlikely(gettimeofday(&tv, NULL))) - fatal(control, "Failed to gettimeofday in main\n"); + fatal_return(("Failed to gettimeofday in main\n"), false); control->secs = tv.tv_sec; control->encloops = nloops(control->secs, control->salt, control->salt + 1); - get_rand(control, control->salt + 2, 6); + if (unlikely(!get_rand(control, control->salt + 2, 6))) return false; /* Get Temp Dir */ eptr = getenv("TMP"); @@ -1180,10 +1220,11 @@ void initialize_control(rzip_control *control) size_t len = strlen(eptr); control->tmpdir = malloc(len+2); if (control->tmpdir == NULL) - fatal(control, "Failed to allocate for tmpdir\n"); + fatal_return(("Failed to allocate for tmpdir\n"), false); strcpy(control->tmpdir, eptr); if (eptr[len - 2] != '/') eptr[len - 2] = '/'; /* need a trailing slash */ eptr[len - 1] = 0; } + return true; } diff --git a/lrzip.h b/lrzip.h index 9ff6649f..a2b1e0f9 100644 --- a/lrzip.h +++ b/lrzip.h @@ -23,23 +23,23 @@ inline i64 get_ram(rzip_control *control); i64 nloops(i64 seconds, uchar *b1, uchar *b2); -void write_magic(rzip_control *control); -void read_magic(rzip_control *control, int fd_in, i64 *expected_size); +bool write_magic(rzip_control *control); +bool read_magic(rzip_control *control, int fd_in, i64 *expected_size); void preserve_perms(rzip_control *control, int fd_in, int fd_out); int open_tmpoutfile(rzip_control *control); -void dump_tmpoutfile(rzip_control *control, int fd_out); +bool dump_tmpoutfile(rzip_control *control, int fd_out); int open_tmpinfile(rzip_control *control); -void read_tmpinfile(rzip_control *control, int fd_in); -void decompress_file(rzip_control *control); -void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head); -void get_fileinfo(rzip_control *control); -void compress_file(rzip_control *control); -void write_fdout(rzip_control *control, void *buf, i64 len); -void write_fdin(rzip_control *control); -void flush_tmpoutbuf(rzip_control *control); +bool read_tmpinfile(rzip_control *control, int fd_in); +bool decompress_file(rzip_control *control); +bool get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head); +bool get_fileinfo(rzip_control *control); +bool compress_file(rzip_control *control); +bool write_fdout(rzip_control *control, void *buf, i64 len); +bool write_fdin(rzip_control *control); +bool flush_tmpoutbuf(rzip_control *control); void close_tmpoutbuf(rzip_control *control); void clear_tmpinbuf(rzip_control *control); -inline void clear_tmpinfile(rzip_control *control); +inline bool clear_tmpinfile(rzip_control *control); void close_tmpinbuf(rzip_control *control); -void initialize_control(rzip_control *control); +bool initialize_control(rzip_control *control); #endif diff --git a/lrzip_private.h b/lrzip_private.h index 1b48aaec..3b8b98ed 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -26,6 +26,7 @@ #include #include #include +#include #ifdef HAVE_STRING_H # include #endif @@ -34,6 +35,34 @@ # include #endif +#ifdef HAVE_ALLOCA_H +# include +#elif defined __GNUC__ +# define alloca __builtin_alloca +#elif defined _AIX +# define alloca __alloca +#elif defined _MSC_VER +# include +# define alloca _alloca +#else +# include +# ifdef __cplusplus +extern "C" +# endif +void *alloca (size_t); +#endif + +#define free(X) do { free((X)); (X) = NULL; } while (0) + +#ifndef strdupa +# define strdupa(str) strcpy(alloca(strlen(str) + 1), str) +#endif + +#ifndef strndupa +# define strndupa(str, len) strncpy(alloca(len + 1), str, len) +#endif + + #ifndef uchar #define uchar unsigned char #endif @@ -326,6 +355,9 @@ struct rzip_control { const char *util_outfile; char delete_outfile; FILE *outputfile; + char library_mode : 1; + void (*log_cb)(void *, unsigned int, const char *, const char *, va_list); + void *log_data; }; struct stream { diff --git a/main.c b/main.c index d0e8f48e..122cc8be 100644 --- a/main.c +++ b/main.c @@ -221,7 +221,7 @@ int main(int argc, char *argv[]) switch (c) { case 'b': if (control->flags & FLAG_NOT_LZMA) - failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); + failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_BZIP2_COMPRESS; break; case 'c': @@ -242,7 +242,7 @@ int main(int argc, char *argv[]) break; case 'g': if (control->flags & FLAG_NOT_LZMA) - failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); + failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_ZLIB_COMPRESS; break; case 'h': @@ -260,40 +260,40 @@ int main(int argc, char *argv[]) break; case 'l': if (control->flags & FLAG_NOT_LZMA) - failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); + failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_LZO_COMPRESS; break; case 'L': control->compression_level = atoi(optarg); if (control->compression_level < 1 || control->compression_level > 9) - failure(control, "Invalid compression level (must be 1-9)\n"); + failure("Invalid compression level (must be 1-9)\n"); break; case 'n': if (control->flags & FLAG_NOT_LZMA) - failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); + failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_NO_COMPRESS; break; case 'N': control->nice_val = atoi(optarg); if (control->nice_val < -20 || control->nice_val > 19) - failure(control, "Invalid nice value (must be -20..19)\n"); + failure("Invalid nice value (must be -20..19)\n"); break; case 'o': if (control->outdir) - failure(control, "Cannot have -o and -O together\n"); + failure("Cannot have -o and -O together\n"); if (unlikely(STDOUT)) - failure(control, "Cannot specify an output filename when outputting to stdout\n"); + failure("Cannot specify an output filename when outputting to stdout\n"); control->outname = optarg; control->suffix = ""; break; case 'O': if (control->outname) /* can't mix -o and -O */ - failure(control, "Cannot have options -o and -O together\n"); + failure("Cannot have options -o and -O together\n"); if (unlikely(STDOUT)) - failure(control, "Cannot specify an output directory when outputting to stdout\n"); + failure("Cannot specify an output directory when outputting to stdout\n"); control->outdir = malloc(strlen(optarg) + 2); if (control->outdir == NULL) - fatal(control, "Failed to allocate for outdir\n"); + fatal("Failed to allocate for outdir\n"); strcpy(control->outdir,optarg); if (strcmp(optarg+strlen(optarg) - 1, "/")) /* need a trailing slash */ strcat(control->outdir, "/"); @@ -301,23 +301,23 @@ int main(int argc, char *argv[]) case 'p': control->threads = atoi(optarg); if (control->threads < 1) - failure(control, "Must have at least one thread\n"); + failure("Must have at least one thread\n"); break; case 'q': control->flags &= ~FLAG_SHOW_PROGRESS; break; case 'S': if (control->outname) - failure(control, "Specified output filename already, can't specify an extension.\n"); + failure("Specified output filename already, can't specify an extension.\n"); if (unlikely(STDOUT)) - failure(control, "Cannot specify a filename suffix when outputting to stdout\n"); + failure("Cannot specify a filename suffix when outputting to stdout\n"); control->suffix = optarg; break; case 't': if (control->outname) - failure(control, "Cannot specify an output file name when just testing.\n"); + failure("Cannot specify an output file name when just testing.\n"); if (!KEEP_FILES) - failure(control, "Doubt that you want to delete a file when just testing.\n"); + failure("Doubt that you want to delete a file when just testing.\n"); control->flags |= FLAG_TEST_ONLY; break; case 'T': @@ -344,7 +344,7 @@ int main(int argc, char *argv[]) break; case 'z': if (control->flags & FLAG_NOT_LZMA) - failure(control, "Can only use one of -l, -b, -g, -z or -n\n"); + failure("Can only use one of -l, -b, -g, -z or -n\n"); control->flags |= FLAG_ZPAQ_COMPRESS; break; } @@ -354,7 +354,7 @@ int main(int argc, char *argv[]) argv += optind; if (control->outname && argc > 1) - failure(control, "Cannot specify output filename with more than 1 file\n"); + failure("Cannot specify output filename with more than 1 file\n"); if (VERBOSE && !SHOW_PROGRESS) { print_err("Cannot have -v and -q options. -v wins.\n"); @@ -400,13 +400,13 @@ int main(int argc, char *argv[]) stat(control->infile, &infile_stat); if (unlikely(S_ISDIR(infile_stat.st_mode))) - failure(control, "lrzip only works directly on FILES.\n" + failure("lrzip only works directly on FILES.\n" "Use lrztar or pipe through tar for compressing directories.\n"); } } if (INFO && STDIN) - failure(control, "Will not get file info from STDIN\n"); + failure("Will not get file info from STDIN\n"); if (control->outname && (strcmp(control->outname, "-") == 0)) { control->flags |= FLAG_STDOUT; @@ -460,7 +460,7 @@ int main(int argc, char *argv[]) gettimeofday(&start_time, NULL); if (unlikely(STDIN && ENCRYPT)) - failure(control, "Unable to work from STDIN while reading password\n"); + failure("Unable to work from STDIN while reading password\n"); if (DECOMPRESS || TEST_ONLY) decompress_file(control); diff --git a/runzip.c b/runzip.c index e882c53f..a6cd3589 100644 --- a/runzip.c +++ b/runzip.c @@ -49,21 +49,25 @@ /* needed for CRC routines */ #include "lzma/C/7zCrc.h" -static inline uchar read_u8(rzip_control *control, void *ss, int stream) +static inline uchar read_u8(rzip_control *control, void *ss, int stream, bool *err) { uchar b; - if (unlikely(read_stream(control, ss, stream, &b, 1) != 1)) - fatal(control, "Stream read u8 failed\n"); + if (unlikely(read_stream(control, ss, stream, &b, 1) != 1)) { + *err = true; + fatal_return(("Stream read u8 failed\n"), 0); + } return b; } -static inline u32 read_u32(rzip_control *control, void *ss, int stream) +static inline u32 read_u32(rzip_control *control, void *ss, int stream, bool *err) { u32 ret; - if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4)) - fatal(control, "Stream read u32 failed\n"); + if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4)) { + *err = true; + fatal_return(("Stream read u32 failed\n"), 0); + } ret = le32toh(ret); return ret; } @@ -74,7 +78,7 @@ static inline i64 read_vchars(rzip_control *control, void *ss, int stream, int l i64 s = 0; if (unlikely(read_stream(control, ss, stream, (uchar *)&s, length) != length)) - fatal(control, "Stream read of %d bytes failed\n", length); + fatal_return(("Stream read of %d bytes failed\n", length), -1); s = le64toh(s); return s; } @@ -128,7 +132,7 @@ static i64 seekto_fdinend(rzip_control *control) while ((tmpchar = getchar()) != EOF) { control->tmp_inbuf[control->in_len++] = (char)tmpchar; if (unlikely(control->in_len > control->in_maxlen)) - failure(control, "Trying to read greater than max_len\n"); + failure_return(("Trying to read greater than max_len\n"), -1); } control->in_ofs = control->in_len; return control->in_ofs; @@ -137,11 +141,13 @@ static i64 seekto_fdinend(rzip_control *control) static i64 read_header(rzip_control *control, void *ss, uchar *head) { int chunk_bytes = 2; + bool err = false; /* All chunks were unnecessarily encoded 8 bytes wide version 0.4x */ if (control->major_version == 0 && control->minor_version == 4) chunk_bytes = 8; - *head = read_u8(control, ss, 0); + *head = read_u8(control, ss, 0, &err); + if (err) return -1; return read_vchars(control, ss, 0, chunk_bytes); } @@ -151,18 +157,22 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, uint32 *cksum uchar *buf; if (unlikely(len < 0)) - failure(control, "len %lld is negative in unzip_literal!\n",len); + failure_return(("len %lld is negative in unzip_literal!\n",len), -1); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal(control, "Failed to malloc literal buffer of size %lld\n", len); + fatal_return(("Failed to malloc literal buffer of size %lld\n", len), -1); stream_read = read_stream(control, ss, 1, buf, len); - if (unlikely(stream_read == -1 )) - fatal(control, "Failed to read_stream in unzip_literal\n"); + if (unlikely(stream_read == -1 )) { + free(buf); + fatal_return(("Failed to read_stream in unzip_literal\n"), -1); + } - if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) - fatal(control, "Failed to write literal buffer of size %lld\n", stream_read); + if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) { + free(buf); + fatal_return(("Failed to write literal buffer of size %lld\n", stream_read), -1); + } if (!HAS_MD5) *cksum = CrcUpdate(*cksum, buf, stream_read); @@ -191,32 +201,36 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, uchar *buf, *off_buf; if (unlikely(len < 0)) - failure(control, "len %lld is negative in unzip_match!\n",len); + failure_return(("len %lld is negative in unzip_match!\n",len), -1); total = 0; cur_pos = seekcur_fdout(control); if (unlikely(cur_pos == -1)) - fatal(control, "Seek failed on out file in unzip_match.\n"); + fatal_return(("Seek failed on out file in unzip_match.\n"), -1); /* Note the offset is in a different format v0.40+ */ offset = read_vchars(control, ss, 0, chunk_bytes); + if (unlikely(offset == -1)) return -1; if (unlikely(seekto_fdhist(control, cur_pos - offset) == -1)) - fatal(control, "Seek failed by %d from %d on history file in unzip_match\n", - offset, cur_pos); + fatal_return(("Seek failed by %d from %d on history file in unzip_match\n", + offset, cur_pos), -1); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal(control, "Failed to malloc match buffer of size %lld\n", len); + fatal_return(("Failed to malloc match buffer of size %lld\n", len), -1); off_buf = buf; while (len) { n = MIN(len, offset); - if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal(control, "Failed to read %d bytes in unzip_match\n", n); - - if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal(control, "Failed to write %d bytes in unzip_match\n", n); + if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) { + free(buf); + fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1); + } + if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) { + free(buf); + fatal_return(("Failed to write %d bytes in unzip_match\n", n), -1); + } if (!HAS_MD5) *cksum = CrcUpdate(*cksum, off_buf, n); @@ -233,7 +247,7 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, return total; } -/* decompress a section of an open file. Call fatal(control, ) on error +/* decompress a section of an open file. Call fatal_return(() on error return the number of bytes that have been retrieved */ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 tally) @@ -245,6 +259,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 struct stat st; uchar head; void *ss; + bool err = false; /* for display of progress */ unsigned long divisor[] = {1,1024,1048576,1073741824U}; @@ -273,9 +288,9 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 else { /* Read in the stored chunk byte width from the file */ if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1)) - fatal(control, "Failed to read chunk_bytes size in runzip_chunk\n"); + fatal_return(("Failed to read chunk_bytes size in runzip_chunk\n"), -1); if (unlikely(chunk_bytes < 1 || chunk_bytes > 8)) - failure(control, "chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes); + failure_return(("chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes), -1); } if (!tally && expected_size) print_maxverbose("Expected size: %lld\n", expected_size); @@ -283,23 +298,35 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 ofs = seekcur_fdin(control); if (unlikely(ofs == -1)) - fatal(control, "Failed to seek input file in runzip_fd\n"); + fatal_return(("Failed to seek input file in runzip_fd\n"), -1); if (fstat(fd_in, &st) || st.st_size - ofs == 0) return 0; ss = open_stream_in(control, fd_in, NUM_STREAMS, chunk_bytes); if (unlikely(!ss)) - failure(control, "Failed to open_stream_in in runzip_chunk\n"); + failure_return(("Failed to open_stream_in in runzip_chunk\n"), -1); while ((len = read_header(control, ss, &head)) || head) { + i64 u; + if (unlikely(len == -1)) return -1; switch (head) { case 0: - total += unzip_literal(control, ss, len, &cksum); + u = unzip_literal(control, ss, len, &cksum); + if (unlikely(u == -1)) { + close_stream_in(ss); + return -1; + } + total += u; break; default: - total += unzip_match(control, ss, len, &cksum, chunk_bytes); + u = unzip_match(control, ss, len, &cksum, chunk_bytes); + if (unlikely(u == -1)) { + close_stream_in(ss); + return -1; + } + total += u; break; } if (expected_size) { @@ -314,19 +341,25 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 } if (!HAS_MD5) { - good_cksum = read_u32(control, ss, 0); - if (unlikely(good_cksum != cksum)) - failure(control, "Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum); + good_cksum = read_u32(control, ss, 0, &err); + if (unlikely(err)) { + close_stream_in(ss); + return -1; + } + if (unlikely(good_cksum != cksum)) { + close_stream_in(ss); + failure_return(("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum), -1); + } print_maxverbose("Checksum for block: 0x%08x\n", cksum); } if (unlikely(close_stream_in(ss))) - fatal(control, "Failed to close stream!\n"); + fatal_return(("Failed to close stream!\n"), -1); return total; } -/* Decompress an open file. Call fatal(control, ) on error +/* Decompress an open file. Call fatal_return(() on error return the number of bytes that have been retrieved */ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size) @@ -334,22 +367,24 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp uchar md5_resblock[MD5_DIGEST_SIZE]; uchar md5_stored[MD5_DIGEST_SIZE]; struct timeval start,end; - i64 total = 0; + i64 total = 0, u; if (!NO_MD5) md5_init_ctx (&control->ctx); gettimeofday(&start,NULL); do { - total += runzip_chunk(control, fd_in, expected_size, total); + u = runzip_chunk(control, fd_in, expected_size, total); + if (unlikely(u == -1)) return -1; + total += u; if (TMP_OUTBUF) - flush_tmpoutbuf(control); + { if (unlikely(!flush_tmpoutbuf(control))) return -1; } else if (STDOUT) - dump_tmpoutfile(control, fd_out); + { if (unlikely(!dump_tmpoutfile(control, fd_out))) return -1; } if (TMP_INBUF) clear_tmpinbuf(control); else if (STDIN) - clear_tmpinfile(control); + if (unlikely(!clear_tmpinfile(control))) return -1; } while (total < expected_size || (!expected_size && !control->eof)); gettimeofday(&end,NULL); @@ -365,14 +400,14 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp i64 fdinend = seekto_fdinend(control); if (unlikely(fdinend == -1)) - failure(control, "Failed to seekto_fdinend in rzip_fd\n"); + failure_return(("Failed to seekto_fdinend in rzip_fd\n"), -1); if (unlikely(seekto_fdin(control, fdinend - MD5_DIGEST_SIZE) == -1)) - failure(control, "Failed to seekto_fdin in rzip_fd\n"); + failure_return(("Failed to seekto_fdin in rzip_fd\n"), -1); if (unlikely(read_1g(control, fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal(control, "Failed to read md5 data in runzip_fd\n"); + fatal_return(("Failed to read md5 data in runzip_fd\n"), -1); if (ENCRYPT) - lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass); + if (unlikely(!lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass))) return -1; for (i = 0; i < MD5_DIGEST_SIZE; i++) if (md5_stored[i] != md5_resblock[i]) { print_output("MD5 CHECK FAILED.\nStored:"); @@ -381,7 +416,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_resblock[j] & 0xFF); - failure(control, "\n"); + failure_return(("\n"), -1); } } @@ -400,11 +435,11 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp close_tmpoutbuf(control); memcpy(md5_stored, md5_resblock, MD5_DIGEST_SIZE); if (unlikely(seekto_fdhist(control, 0) == -1)) - fatal(control, "Failed to seekto_fdhist in runzip_fd\n"); + fatal_return(("Failed to seekto_fdhist in runzip_fd\n"), -1); if (unlikely((md5_fstream = fdopen(fd_hist, "r")) == NULL)) - fatal(control, "Failed to fdopen fd_hist in runzip_fd\n"); + fatal_return(("Failed to fdopen fd_hist in runzip_fd\n"), -1); if (unlikely(md5_stream(md5_fstream, md5_resblock))) - fatal(control, "Failed to md5_stream in runzip_fd\n"); + fatal_return(("Failed to md5_stream in runzip_fd\n"), -1); /* We don't close the file here as it's closed in main */ for (i = 0; i < MD5_DIGEST_SIZE; i++) if (md5_stored[i] != md5_resblock[i]) { @@ -414,7 +449,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_resblock[j] & 0xFF); - failure(control, "\n"); + failure_return(("\n"), -1); } print_output("MD5 integrity of written file matches archive\n"); if (!HAS_MD5) diff --git a/rzip.c b/rzip.c index c0d4feb6..a85728e5 100644 --- a/rzip.c +++ b/rzip.c @@ -142,7 +142,7 @@ struct sliding_buffer { int fd; /* The fd of the mmap */ } sb; /* Sliding buffer */ -static void remap_low_sb(rzip_control *control) +static bool remap_low_sb(rzip_control *control) { i64 new_offset; @@ -150,19 +150,20 @@ static void remap_low_sb(rzip_control *control) round_to_page(&new_offset); print_maxverbose("Sliding main buffer to offset %lld\n", new_offset); if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal(control, "Failed to munmap in remap_low_sb\n"); + fatal_return(("Failed to munmap in remap_low_sb\n"), false); if (new_offset + sb.size_low > sb.orig_size) sb.size_low = sb.orig_size - new_offset; sb.offset_low = new_offset; sb.buf_low = (uchar *)mmap(sb.buf_low, sb.size_low, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_low); if (unlikely(sb.buf_low == MAP_FAILED)) - fatal(control, "Failed to re mmap in remap_low_sb\n"); + fatal_return(("Failed to re mmap in remap_low_sb\n"), false); + return true; } -static inline void remap_high_sb(rzip_control *control, i64 p) +static inline bool remap_high_sb(rzip_control *control, i64 p) { if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal(control, "Failed to munmap in remap_high_sb\n"); + fatal_return(("Failed to munmap in remap_high_sb\n"), false); sb.size_high = sb.high_length; /* In case we shrunk it when we hit the end of the file */ sb.offset_high = p; /* Make sure offset is rounded to page size of total offset */ @@ -171,7 +172,8 @@ static inline void remap_high_sb(rzip_control *control, i64 p) sb.size_high = sb.orig_size - sb.offset_high; sb.buf_high = (uchar *)mmap(sb.buf_high, sb.size_high, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_high); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal(control, "Failed to re mmap in remap_high_sb\n"); + fatal_return(("Failed to re mmap in remap_high_sb\n"), false); + return true; } /* We use a "sliding mmap" to effectively read more than we can fit into the @@ -185,45 +187,49 @@ static uchar *get_sb(rzip_control *control, i64 p) i64 low_end = sb.offset_low + sb.size_low; if (unlikely(sb.offset_search > low_end)) - remap_low_sb(control); + if (unlikely(!remap_low_sb(control))) return NULL; if (p >= sb.offset_low && p < low_end) return (sb.buf_low + p - sb.offset_low); if (p >= sb.offset_high && p < (sb.offset_high + sb.size_high)) return (sb.buf_high + (p - sb.offset_high)); /* p is not within the low or high buffer range */ - remap_high_sb(control, p); + if (unlikely(!remap_high_sb(control, p))) return NULL; return (sb.buf_high + (p - sb.offset_high)); } /* All put_u8/u32/vchars go to stream 0 */ -static inline void put_u8(rzip_control *control, void *ss, uchar b) +static inline bool put_u8(rzip_control *control, void *ss, uchar b) { if (unlikely(write_stream(control, ss, 0, &b, 1))) - fatal(control, "Failed to put_u8\n"); + fatal_return(("Failed to put_u8\n"), false); + return true; } -static inline void put_u32(rzip_control *control, void *ss, uint32_t s) +static inline bool put_u32(rzip_control *control, void *ss, uint32_t s) { s = htole32(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, 4))) - fatal(control, "Failed to put_u32\n"); + fatal_return(("Failed to put_u32\n"), false); + return true; } /* Put a variable length of bytes dependant on how big the chunk is */ -static inline void put_vchars(rzip_control *control, void *ss, i64 s, int length) +static inline bool put_vchars(rzip_control *control, void *ss, i64 s, int length) { s = htole64(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, length))) - fatal(control, "Failed to put_vchars\n"); + fatal_return(("Failed to put_vchars\n"), false); + return true; } -static void put_header(rzip_control *control, void *ss, uchar head, i64 len) +static bool put_header(rzip_control *control, void *ss, uchar head, i64 len) { - put_u8(control, ss, head); - put_vchars(control, ss, len, 2); + if (unlikely(!put_u8(control, ss, head))) return false; + if (unlikely(!put_vchars(control, ss, len, 2))) return false; + return true; } -static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 offset, i64 len) +static bool put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 offset, i64 len) { do { i64 ofs; @@ -232,8 +238,8 @@ static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 o n = 0xFFFF; ofs = (p - offset); - put_header(control, st->ss, 1, n); - put_vchars(control, st->ss, ofs, st->chunk_bytes); + if (unlikely(!put_header(control, st->ss, 1, n))) return false; + if (unlikely(!put_vchars(control, st->ss, ofs, st->chunk_bytes))) return false; st->stats.matches++; st->stats.match_bytes += n; @@ -241,6 +247,7 @@ static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 o p += n; offset += n; } while (len); + return true; } /* write some data to a stream mmap encoded. Return -1 on failure */ @@ -254,20 +261,23 @@ static int write_sbstream(rzip_control *control, void *ss, int stream, i64 p, i6 n = MIN(sinfo->bufsize - sinfo->s[stream].buflen, len); for (i = 0; i < n; i++) { - memcpy(sinfo->s[stream].buf + sinfo->s[stream].buflen + i, - get_sb(control, p + i), 1); + uchar *u; + + u = get_sb(control, p + i); + if (unlikely(!u)) return -1; + memcpy(sinfo->s[stream].buf + sinfo->s[stream].buflen + i, u, 1); } sinfo->s[stream].buflen += n; p += n; len -= n; if (sinfo->s[stream].buflen == sinfo->bufsize) - flush_buffer(control, sinfo, stream); + if (unlikely(!flush_buffer(control, sinfo, stream))) return -1; } return 0; } -static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, i64 p) +static bool put_literal(rzip_control *control, struct rzip_state *st, i64 last, i64 p) { do { i64 len = p - last; @@ -277,12 +287,13 @@ static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, st->stats.literals++; st->stats.literal_bytes += len; - put_header(control, st->ss, 0, len); + if (unlikely(!put_header(control, st->ss, 0, len))) return false; if (unlikely(len && write_sbstream(control, st->ss, 1, last, len))) - fatal(control, "Failed to write_stream in put_literal\n"); + fatal_return(("Failed to write_stream in put_literal\n"), false); last += len; } while (p > last); + return true; } /* Could give false positive on offset 0. Who cares. */ @@ -404,8 +415,14 @@ static tag clean_one_from_hash(rzip_control *control, struct rzip_state *st) static inline tag next_tag(rzip_control *control, struct rzip_state *st, i64 p, tag t) { - t ^= st->hash_index[*get_sb(control, p - 1)]; - t ^= st->hash_index[*get_sb(control, p + MINIMUM_MATCH - 1)]; + uchar *u; + + u = get_sb(control, p - 1); + if (unlikely(!u)) return -1; + t ^= st->hash_index[*u]; + u = get_sb(control, p + MINIMUM_MATCH - 1); + if (unlikely(!u)) return -1; + t ^= st->hash_index[*u]; return t; } @@ -413,9 +430,13 @@ static inline tag full_tag(rzip_control *control, struct rzip_state *st, i64 p) { tag ret = 0; int i; + uchar *u; - for (i = 0; i < MINIMUM_MATCH; i++) - ret ^= st->hash_index[*get_sb(control, p + i)]; + for (i = 0; i < MINIMUM_MATCH; i++) { + u = get_sb(control, p + i); + if (unlikely(!u)) return -1; + ret ^= st->hash_index[*u]; + } return ret; } @@ -515,7 +536,7 @@ static void show_distrib(rzip_control *control, struct rzip_state *st) primary * 100.0 / (total ? : 1)); } -static void hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple) +static bool hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple) { int lastpct = 0, last_chunkpct = 0; i64 cksum_limit = 0, p, end; @@ -544,7 +565,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct } if (unlikely(!st->hash_table)) - fatal(control, "Failed to allocate hash table in hash_search\n"); + fatal_return(("Failed to allocate hash table in hash_search\n"), false); st->minimum_tag_mask = tag_mask; st->tag_clean_ptr = 0; @@ -558,8 +579,10 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct current.p = p; current.ofs = 0; - if (likely(end > 0)) + if (likely(end > 0)) { t = full_tag(control, st, p); + if (unlikely(t == -1)) return false; + } while (p < end) { i64 reverse, mlen, offset = 0; @@ -567,6 +590,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct p++; sb.offset_search = p; t = next_tag(control, st, p, t); + if (unlikely(t == -1)) return false; /* Don't look for a match if there are no tags with this number of bits in the hash table. */ @@ -593,12 +617,13 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct if ((current.len >= GREAT_MATCH || p >= current.p + MINIMUM_MATCH) && current.len >= MINIMUM_MATCH) { if (st->last_match < current.p) - put_literal(control, st, st->last_match, current.p); - put_match(control, st, current.p, current.ofs, current.len); + if (unlikely(!put_literal(control, st, st->last_match, current.p))) return false; + if (unlikely(!put_match(control, st, current.p, current.ofs, current.len))) return false; st->last_match = current.p + current.len; current.p = p = st->last_match; current.len = 0; t = full_tag(control, st, p); + if (unlikely(t == -1)) return false; } if (unlikely(p % 128 == 0)) { @@ -621,7 +646,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal(control, "Failed to malloc ckbuf in hash_search\n"); + fatal_return(("Failed to malloc ckbuf in hash_search\n"), false); for (i = 0; i < n; i++) memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1); st->cksum = CrcUpdate(st->cksum, ckbuf, n); @@ -643,7 +668,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal(control, "Failed to malloc ckbuf in hash_search\n"); + fatal_return(("Failed to malloc ckbuf in hash_search\n"), false); for (i = 0; i < n; i++) memcpy(ckbuf + i, get_sb(control, cksum_limit + i), 1); st->cksum = CrcUpdate(st->cksum, ckbuf, n); @@ -653,8 +678,9 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct free(ckbuf); } - put_literal(control, st, 0, 0); - put_u32(control, st->ss, st->cksum); + if (unlikely(!put_literal(control, st, 0, 0))) return false; + if (unlikely(!put_u32(control, st->ss, st->cksum))) return false; + return true; } @@ -676,7 +702,7 @@ static inline void *fake_mremap(void *old_address, size_t old_size, size_t new_s * anonymous ram and reading stdin into it. It means the maximum ram * we can use will be less but we will already have determined this in * rzip_chunk */ -static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) +static bool mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) { i64 len = st->chunk_size; uchar *offset_buf = buf; @@ -688,7 +714,7 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) ret = MIN(len, one_g); ret = read(0, offset_buf, (size_t)ret); if (unlikely(ret < 0)) - fatal(control, "Failed to read in mmap_stdin\n"); + fatal_return(("Failed to read in mmap_stdin\n"), false); total += ret; if (ret == 0) { /* Should be EOF */ @@ -703,7 +729,7 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) st->chunk_size = 0; } if (unlikely(buf == MAP_FAILED)) - fatal(control, "Failed to remap to smaller buf in mmap_stdin\n"); + fatal_return(("Failed to remap to smaller buf in mmap_stdin\n"), false); control->eof = st->stdin_eof = 1; break; } @@ -711,9 +737,10 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) len -= ret; } control->st_size += total; + return true; } -static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in, i64 offset) +static bool init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in, i64 offset) { /* Initialise the high buffer */ if (!STDIN) { @@ -723,7 +750,7 @@ static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int sb.high_length += control->page_size - (sb.high_length % control->page_size); sb.buf_high = (uchar *)mmap(NULL, sb.high_length, PROT_READ, MAP_SHARED, fd_in, offset); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal(control, "Unable to mmap buf_high in init_sliding_mmap\n"); + fatal_return(("Unable to mmap buf_high in init_sliding_mmap\n"), false); sb.size_high = sb.high_length; sb.offset_high = 0; } @@ -732,36 +759,45 @@ static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int sb.size_low = st->mmap_size; sb.orig_size = st->chunk_size; sb.fd = fd_in; + return true; } /* compress a chunk of an open file. Assumes that the file is able to be mmap'd and is seekable */ -static void rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out, i64 offset, +static bool rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out, i64 offset, double pct_base, double pct_multiple) { - init_sliding_mmap(control, st, fd_in, offset); + if (unlikely(!init_sliding_mmap(control, st, fd_in, offset))) return false; st->ss = open_stream_out(control, fd_out, NUM_STREAMS, st->chunk_size, st->chunk_bytes); if (unlikely(!st->ss)) - fatal(control, "Failed to open streams in rzip_chunk\n"); + fatal_return(("Failed to open streams in rzip_chunk\n"), false); print_verbose("Beginning rzip pre-processing phase\n"); - hash_search(control, st, pct_base, pct_multiple); + if (unlikely(!hash_search(control, st, pct_base, pct_multiple))) { + close_stream_out(control, st->ss); + return false; + } /* unmap buffer before closing and reallocating streams */ - if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal(control, "Failed to munmap in rzip_chunk\n"); + if (unlikely(munmap(sb.buf_low, sb.size_low))) { + close_stream_out(control, st->ss); + fatal_return(("Failed to munmap in rzip_chunk\n"), false); + } if (!STDIN) { - if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal(control, "Failed to munmap in rzip_chunk\n"); + if (unlikely(munmap(sb.buf_high, sb.size_high))) { + close_stream_out(control, st->ss); + fatal_return(("Failed to munmap in rzip_chunk\n"), false); + } } if (unlikely(close_stream_out(control, st->ss))) - fatal(control, "Failed to flush/close streams in rzip_chunk\n"); + fatal_return(("Failed to flush/close streams in rzip_chunk\n"), false); + return true; } /* compress a whole file chunks at a time */ -void rzip_fd(rzip_control *control, int fd_in, int fd_out) +bool rzip_fd(rzip_control *control, int fd_in, int fd_out) { /* add timers for ETA estimates * Base it off the file size and number of iterations required @@ -784,15 +820,19 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) st = calloc(sizeof(*st), 1); if (unlikely(!st)) - fatal(control, "Failed to allocate control state in rzip_fd\n"); + fatal_return(("Failed to allocate control state in rzip_fd\n"), false); if (LZO_COMPRESS) { - if (unlikely(lzo_init() != LZO_E_OK)) - fatal(control, "lzo_init() failed\n"); + if (unlikely(lzo_init() != LZO_E_OK)) { + free(st); + fatal_return(("lzo_init() failed\n"), false); + } } - if (unlikely(fstat(fd_in, &s))) - fatal(control, "Failed to stat fd_in in rzip_fd\n"); + if (unlikely(fstat(fd_in, &s))) { + free(st); + fatal_return(("Failed to stat fd_in in rzip_fd\n"), false); + } if (!STDIN) { len = control->st_size = s.st_size; @@ -804,14 +844,18 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) /* Check if there's enough free space on the device chosen to fit the * compressed file, based on the compressed file being as large as the * uncompressed file. */ - if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal(control, "Failed to fstatvfs in compress_file\n"); + if (unlikely(fstatvfs(fd_out, &fbuf))) { + free(st); + fatal_return(("Failed to fstatvfs in compress_file\n"), false); + } free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < control->st_size) { if (FORCE_REPLACE) print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); - else - failure(control, "Possibly inadequate free space to compress file, use -f to override.\n"); + else { + free(st); + failure_return(("Possibly inadequate free space to compress file, use -f to override.\n"), false); + } } } @@ -855,7 +899,11 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) last.tv_sec = last.tv_usec = 0; gettimeofday(&start, NULL); - prepare_streamout_threads(control); + if (unlikely(!prepare_streamout_threads(control))) { + free(st->hash_table); + free(st); + return false; + } while (!pass || len > 0 || (STDIN && !st->stdin_eof)) { double pct_base, pct_multiple; @@ -878,26 +926,47 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) sb.buf_low = mmap(NULL, st->mmap_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); /* Better to shrink the window to the largest size that works than fail */ if (sb.buf_low == MAP_FAILED) { - if (unlikely(errno != ENOMEM)) - fatal(control, "Failed to mmap %s\n", control->infile); + if (unlikely(errno != ENOMEM)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Failed to mmap %s\n", control->infile), false); + } st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); - if (unlikely(!st->mmap_size)) - fatal(control, "Unable to mmap any ram\n"); + if (unlikely(!st->mmap_size)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Unable to mmap any ram\n"), false); + } goto retry; } st->chunk_size = st->mmap_size; - mmap_stdin(control, sb.buf_low, st); + if (unlikely(!mmap_stdin(control, sb.buf_low, st))) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + return false; + } } else { /* NOTE The buf is saved here for !STDIN mode */ sb.buf_low = (uchar *)mmap(sb.buf_low, st->mmap_size, PROT_READ, MAP_SHARED, fd_in, offset); if (sb.buf_low == MAP_FAILED) { - if (unlikely(errno != ENOMEM)) - fatal(control, "Failed to mmap %s\n", control->infile); + if (unlikely(errno != ENOMEM)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Failed to mmap %s\n", control->infile), false); + } st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); - if (unlikely(!st->mmap_size)) - fatal(control, "Unable to mmap any ram\n"); + if (unlikely(!st->mmap_size)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Unable to mmap any ram\n"), false); + } goto retry; } if (st->mmap_size < st->chunk_size) @@ -970,18 +1039,30 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) if (st->chunk_size == len) control->eof = 1; - rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple); + if (unlikely(!rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple))) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + return false; + } /* st->chunk_size may be shrunk in rzip_chunk */ last_chunk = st->chunk_size; len -= st->chunk_size; - if (unlikely(len > 0 && control->eof)) - failure(control, "Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"); + if (unlikely(len > 0 && control->eof)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + failure_return(("Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"), false); + } } - close_streamout_threads(control); if (likely(st->hash_table)) free(st->hash_table); + if (unlikely(!close_streamout_threads(control))) { + free(st); + return false; + } if (!NO_MD5) { /* Temporary workaround till someone fixes apple md5 */ @@ -994,13 +1075,21 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) } /* When encrypting data, we encrypt the MD5 value as well */ if (ENCRYPT) - lrz_encrypt(control, md5_resblock, MD5_DIGEST_SIZE, control->salt_pass); - if (unlikely(write_1g(control, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal(control, "Failed to write md5 in rzip_fd\n"); + if (unlikely(!lrz_encrypt(control, md5_resblock, MD5_DIGEST_SIZE, control->salt_pass))) { + free(st); + return false; + } + if (unlikely(write_1g(control, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) { + free(st); + fatal_return(("Failed to write md5 in rzip_fd\n"), false); + } } if (TMP_OUTBUF) - flush_tmpoutbuf(control); + if (unlikely(!flush_tmpoutbuf(control))) { + free(st); + return false; + } gettimeofday(¤t, NULL); if (STDIN) @@ -1025,6 +1114,7 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) 1.0 * s.st_size / s2.st_size, chunkmbs); free(st); + return true; } void rzip_control_free(rzip_control *control) diff --git a/rzip.h b/rzip.h index 29b2e17f..116198dd 100644 --- a/rzip.h +++ b/rzip.h @@ -21,7 +21,7 @@ #define RZIP_H #include "lrzip_private.h" -void rzip_fd(rzip_control *control, int fd_in, int fd_out); +bool rzip_fd(rzip_control *control, int fd_in, int fd_out); void rzip_control_free(rzip_control *control); #endif diff --git a/stream.c b/stream.c index 7f83c628..da797449 100644 --- a/stream.c +++ b/stream.c @@ -105,53 +105,61 @@ static pthread_mutex_t output_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t output_cond = PTHREAD_COND_INITIALIZER; static pthread_t *threads; -static void init_mutex(rzip_control *control, pthread_mutex_t *mutex) +static bool init_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_init(mutex, NULL))) - fatal(control, "pthread_mutex_init failed"); + fatal_return(("pthread_mutex_init failed"), false); + return true; } -static void unlock_mutex(rzip_control *control, pthread_mutex_t *mutex) +static bool unlock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_unlock(mutex))) - fatal(control, "pthread_mutex_unlock failed"); + fatal_return(("pthread_mutex_unlock failed"), false); + return true; } -static void lock_mutex(rzip_control *control, pthread_mutex_t *mutex) +static bool lock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_lock(mutex))) - fatal(control, "pthread_mutex_lock failed"); + fatal_return(("pthread_mutex_lock failed"), false); + return true; } -static void cond_wait(rzip_control *control, pthread_cond_t *cond, pthread_mutex_t *mutex) +static bool cond_wait(rzip_control *control, pthread_cond_t *cond, pthread_mutex_t *mutex) { if (unlikely(pthread_cond_wait(cond, mutex))) - fatal(control, "pthread_cond_wait failed"); + fatal_return(("pthread_cond_wait failed"), false); + return true; } -static void cond_broadcast(rzip_control *control, pthread_cond_t *cond) +static bool cond_broadcast(rzip_control *control, pthread_cond_t *cond) { if (unlikely(pthread_cond_broadcast(cond))) - fatal(control, "pthread_cond_broadcast failed"); + fatal_return(("pthread_cond_broadcast failed"), false); + return true; } -void create_pthread(rzip_control *control, pthread_t *thread, pthread_attr_t * attr, +bool create_pthread(rzip_control *control, pthread_t *thread, pthread_attr_t * attr, void * (*start_routine)(void *), void *arg) { if (unlikely(pthread_create(thread, attr, start_routine, arg))) - fatal(control, "pthread_create"); + fatal_return(("pthread_create"), false); + return true; } -void detach_pthread(rzip_control *control, pthread_t *thread) +bool detach_pthread(rzip_control *control, pthread_t *thread) { if (unlikely(pthread_detach(*thread))) - fatal(control, "pthread_detach"); + fatal_return(("pthread_detach"), false); + return true; } -void join_pthread(rzip_control *control, pthread_t th, void **thread_return) +bool join_pthread(rzip_control *control, pthread_t th, void **thread_return) { if (pthread_join(th, thread_return)) - fatal(control, "pthread_join"); + fatal_return(("pthread_join"), false); + return true; } /* just to keep things clean, declare function here @@ -164,12 +172,14 @@ static inline FILE *fake_fmemopen(rzip_control *control, void *buf, size_t bufle FILE *in; if (unlikely(strcmp(mode, "r"))) - failure(control, "fake_fmemopen only supports mode \"r\"."); + failure_return(("fake_fmemopen only supports mode \"r\"."), NULL); in = tmpfile(); if (unlikely(!in)) return NULL; - if (unlikely(fwrite(buf, buflen, 1, in) != 1)) + if (unlikely(fwrite(buf, buflen, 1, in) != 1)) { + fclose(in); return NULL; + } rewind(in); return in; } @@ -179,7 +189,7 @@ static inline FILE *fake_open_memstream(rzip_control *control, char **buf, size_ FILE *out; if (unlikely(buf == NULL || length == NULL)) - failure(control, "NULL parameter to fake_open_memstream"); + failure_return(("NULL parameter to fake_open_memstream"), NULL); out = tmpfile(); if (unlikely(!out)) return NULL; @@ -197,10 +207,14 @@ static inline int fake_open_memstream_update_buffer(FILE *fp, uchar **buf, size_ *buf = (uchar *)malloc(*length); if (unlikely(!*buf)) return -1; - if (unlikely(fread(*buf, *length, 1, fp) != 1)) + if (unlikely(fread(*buf, *length, 1, fp) != 1)) { + free(*buf); return -1; - if (unlikely(fseek(fp, original_pos, SEEK_SET))) + } + if (unlikely(fseek(fp, original_pos, SEEK_SET))) { + free(*buf); return -1; + } return 0; } @@ -238,8 +252,11 @@ static int zpaq_compress_buf(rzip_control *control, struct compress_thread *cthr zpipe_compress(in, out, control->msgout, cthread->s_len, (int)(SHOW_PROGRESS), thread); - if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal(control, "Failed to memstream_update_buffer in zpaq_compress_buf"); + if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) { + fclose(in); + fclose(out); + fatal_return(("Failed to memstream_update_buffer in zpaq_compress_buf"), -1); + } fclose(in); fclose(out); @@ -495,13 +512,17 @@ static int zpaq_decompress_buf(rzip_control *control, struct uncomp_thread *ucth out = open_memstream((char **)&c_buf, &dlen); if (unlikely(!out)) { print_err("Failed to open_memstream in zpaq_decompress_buf\n"); + fclose(in); return -1; } zpipe_decompress(in, out, control->msgout, ucthread->u_len, (int)(SHOW_PROGRESS), thread); - if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal(control, "Failed to memstream_update_buffer in zpaq_decompress_buf"); + if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) { + fclose(in); + fclose(out); + fatal_return(("Failed to memstream_update_buffer in zpaq_decompress_buf"), -1); + } fclose(in); fclose(out); @@ -673,9 +694,9 @@ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret) /* The data won't fit in a temporary output buffer so we have * to fall back to temporary files. */ print_verbose("Unable to decompress entirely in ram, will use physical files\n"); - write_fdout(control, control->tmp_outbuf, control->out_len); + if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) return -1; close_tmpoutbuf(control); - write_fdout(control, offset_buf, ret); + if (unlikely(!write_fdout(control, offset_buf, ret))) return -1; return ret; } memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret); @@ -707,7 +728,7 @@ ssize_t write_1g(rzip_control *control, void *buf, i64 len) return total; } -static void read_fdin(struct rzip_control *control, i64 len) +static bool read_fdin(struct rzip_control *control, i64 len) { int tmpchar; i64 i; @@ -715,11 +736,12 @@ static void read_fdin(struct rzip_control *control, i64 len) for (i = 0; i < len; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure(control, "Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", - len, i); + failure_return(("Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", + len, i), false); control->tmp_inbuf[control->in_ofs + i] = (char)tmpchar; } control->in_len = control->in_ofs + len; + return true; } /* Ditto for read */ @@ -733,13 +755,13 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) /* We're decompressing from STDIN */ if (unlikely(control->in_ofs + len > control->in_maxlen)) { /* We're unable to fit it all into the temp buffer */ - write_fdin(control); - read_tmpinfile(control, control->fd_in); + if (unlikely(!write_fdin(control))) return -1; + if (unlikely(!read_tmpinfile(control, control->fd_in))) return -1; close_tmpinbuf(control); goto read_fd; } if (control->in_ofs + len > control->in_len) - read_fdin(control, control->in_ofs + len - control->in_len); + if (unlikely(!read_fdin(control, control->in_ofs + len - control->in_len))) return false; memcpy(buf, control->tmp_inbuf + control->in_ofs, len); control->in_ofs += len; return len; @@ -747,7 +769,7 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) if (TMP_OUTBUF && fd == control->fd_out) { if (unlikely(control->out_ofs + len > control->out_maxlen)) - failure(control, "Trying to read beyond out_ofs in tmpoutbuf\n"); + failure_return(("Trying to read beyond out_ofs in tmpoutbuf\n"), -1); memcpy(buf, control->tmp_outbuf + control->out_ofs, len); control->out_ofs += len; return len; @@ -876,7 +898,7 @@ static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos if (TMP_INBUF) { if (spos > control->in_len) - read_fdin(control, spos - control->in_len); + if (unlikely(!read_fdin(control, spos - control->in_len))) return -1; control->in_ofs = spos; if (unlikely(spos < 0)) { print_err("Trying to seek to %lld outside tmp inbuf in read_seekto\n", spos); @@ -896,7 +918,7 @@ static i64 get_seek(rzip_control *control, int fd) return control->out_relofs + control->out_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal(control, "Failed to lseek in get_seek\n"); + fatal_return(("Failed to lseek in get_seek\n"), -1); return ret; } @@ -908,11 +930,11 @@ static i64 get_readseek(rzip_control *control, int fd) return control->in_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal(control, "Failed to lseek in get_seek\n"); + fatal_return(("Failed to lseek in get_seek\n"), -1); return ret; } -void prepare_streamout_threads(rzip_control *control) +bool prepare_streamout_threads(rzip_control *control) { int i; @@ -926,30 +948,47 @@ void prepare_streamout_threads(rzip_control *control) control->threads = 1; threads = calloc(sizeof(pthread_t), control->threads); if (unlikely(!threads)) - fatal(control, "Unable to calloc threads in prepare_streamout_threads\n"); + fatal_return(("Unable to calloc threads in prepare_streamout_threads\n"), false); cthread = calloc(sizeof(struct compress_thread), control->threads); - if (unlikely(!cthread)) - fatal(control, "Unable to calloc cthread in prepare_streamout_threads\n"); + if (unlikely(!cthread)) { + free(threads); + fatal_return(("Unable to calloc cthread in prepare_streamout_threads\n"), false); + } for (i = 0; i < control->threads; i++) - init_mutex(control, &cthread[i].mutex); + if (unlikely(!init_mutex(control, &cthread[i].mutex))) { + int x; + for (x = 0; x < i; x++) pthread_mutex_destroy(&cthread[x].mutex); + free(threads); + free(cthread); + return false; + } + return true; } -void close_streamout_threads(rzip_control *control) +bool close_streamout_threads(rzip_control *control) { int i, close_thread = output_thread; /* Wait for the threads in the correct order in case they end up * serialised */ for (i = 0; i < control->threads; i++) { - lock_mutex(control, &cthread[close_thread].mutex); + if (unlikely(!lock_mutex(control, &cthread[close_thread].mutex))) { + int x; + for (x = 0; x < i; x++) unlock_mutex(control, &cthread[close_thread].mutex); + free(cthread); + free(threads); + return false; + } + if (++close_thread == control->threads) close_thread = 0; } free(cthread); free(threads); + return true; } /* open a set of output streams, compressing with the given @@ -1044,8 +1083,12 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li for (i = 0; i < n; i++) { sinfo->s[i].buf = calloc(sinfo->bufsize , 1); - if (unlikely(!sinfo->s[i].buf)) - fatal(control, "Unable to malloc buffer of size %lld in open_stream_out\n", sinfo->bufsize); + if (unlikely(!sinfo->s[i].buf)) { + fatal("Unable to malloc buffer of size %lld in open_stream_out\n", sinfo->bufsize); + free(sinfo->s); + free(sinfo); + return NULL; + } } return (void *)sinfo; @@ -1054,7 +1097,7 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li /* The block headers are all encrypted so we read the data and salt associated * with them, decrypt the data, then return the decrypted version of the * values */ -static void decrypt_header(rzip_control *control, uchar *head, uchar *c_type, +static bool decrypt_header(rzip_control *control, uchar *head, uchar *c_type, i64 *c_len, i64 *u_len, i64 *last_head) { uchar *buf = head + SALT_LEN; @@ -1064,12 +1107,13 @@ static void decrypt_header(rzip_control *control, uchar *head, uchar *c_type, memcpy(buf + 9, u_len, 8); memcpy(buf + 17, last_head, 8); - lrz_decrypt(control, buf, 25, head); + if (unlikely(!lrz_decrypt(control, buf, 25, head))) return false; memcpy(c_type, buf, 1); memcpy(c_len, buf + 1, 8); memcpy(u_len, buf + 9, 8); memcpy(last_head, buf + 17, 8); + return true; } /* prepare a set of n streams for reading on file descriptor f */ @@ -1094,8 +1138,11 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) return NULL; ucthread = calloc(sizeof(struct uncomp_thread), total_threads); - if (unlikely(!ucthread)) - fatal(control, "Unable to calloc cthread in open_stream_in\n"); + if (unlikely(!ucthread)) { + free(sinfo); + free(threads); + fatal_return(("Unable to calloc cthread in open_stream_in\n"), NULL); + } sinfo->num_streams = n; sinfo->fd = f; @@ -1129,6 +1176,7 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) } } sinfo->initial_pos = get_readseek(control, f); + if (unlikely(sinfo->initial_pos == -1)) goto failed; for (i = 0; i < n; i++) { uchar c, enc_head[25 + SALT_LEN]; @@ -1176,7 +1224,7 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) header_length = 1 + (read_len * 3); } if (ENCRYPT) - decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head); + if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; v1 = le64toh(v1); v2 = le64toh(v2); @@ -1219,32 +1267,38 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) * by reading what has been written, encrypting it, and writing back over it. * This is very convoluted depending on whether a last_head value is written * to this block or not. See the callers of this function */ -static void rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, i64 ofs) +static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, i64 ofs) { uchar *buf, *head; i64 cur_ofs; cur_ofs = get_seek(control, sinfo->fd) - sinfo->initial_pos; + if (unlikely(cur_ofs == -1)) return false; head = malloc(25 + SALT_LEN); if (unlikely(!head)) - fatal(control, "Failed to malloc head in rewrite_encrypted\n"); + fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false); buf = head + SALT_LEN; - get_rand(control, head, SALT_LEN); + if (unlikely(!get_rand(control, head, SALT_LEN))) goto error; if (unlikely(seekto(control, sinfo, ofs - SALT_LEN))) - failure(control, "Failed to seekto buf ofs in rewrite_encrypted\n"); + failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, head, SALT_LEN))) - failure(control, "Failed to write_buf head in rewrite_encrypted\n"); + failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - failure(control, "Failed to read_buf buf in rewrite_encrypted\n"); + + failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error); - lrz_encrypt(control, buf, 25, head); + if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error; if (unlikely(seekto(control, sinfo, ofs))) - failure(control, "Failed to seek back to ofs in rewrite_encrypted\n"); + failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, buf, 25))) - failure(control, "Failed to write_buf encrypted buf in rewrite_encrypted\n"); + failure_goto(("Failed to write_buf encrypted buf in rewrite_encrypted\n"), error); free(head); seekto(control, sinfo, cur_ofs); + return true; +error: + free(head); + return false; } /* Enter with s_buf allocated,s_buf points to the compressed data after the @@ -1287,7 +1341,7 @@ static void *compthread(void *data) ret = gzip_compress_buf(control, cti); else if (ZPAQ_COMPRESS) ret = zpaq_compress_buf(control, cti, i); - else failure(control, "Dunno wtf compression to use!\n"); + else failure_goto(("Dunno wtf compression to use!\n"), error); } padded_len = cti->c_len; @@ -1298,15 +1352,15 @@ static void *compthread(void *data) padded_len = MIN_SIZE; cti->s_buf = realloc(cti->s_buf, MIN_SIZE); if (unlikely(!cti->s_buf)) - fatal(control, "Failed to realloc s_buf in compthread\n"); - get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len); + fatal_goto(("Failed to realloc s_buf in compthread\n"), error); + if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error; } /* If compression fails for whatever reason multithreaded, then wait * for the previous thread to finish, serialising the work to decrease * the memory requirements, increasing the chance of success */ if (unlikely(ret && waited)) - failure(control, "Failed to compress in compthread\n"); + failure_goto(("Failed to compress in compthread\n"), error); if (!waited) { lock_mutex(control, &output_lock); @@ -1332,7 +1386,7 @@ static void *compthread(void *data) if (TMP_OUTBUF) { if (!control->magic_written) write_magic(control); - flush_tmpoutbuf(control); + if (unlikely(!flush_tmpoutbuf(control))) goto error; } /* Write chunk bytes of this block */ @@ -1346,13 +1400,14 @@ static void *compthread(void *data) /* First chunk of this stream, write headers */ ctis->initial_pos = get_seek(control, ctis->fd); + if (unlikely(ctis->initial_pos == -1)) goto error; for (j = 0; j < ctis->num_streams; j++) { /* If encrypting, we leave SALT_LEN room to write in salt * later */ if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal(control, "Failed to write_buf blank salt in compthread %d\n", i); + fatal_goto(("Failed to write_buf blank salt in compthread %d\n", i), error); ctis->cur_pos += SALT_LEN; } ctis->s[j].last_head = ctis->cur_pos + 1 + (write_len * 2); @@ -1365,23 +1420,23 @@ static void *compthread(void *data) } if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) - fatal(control, "Failed to seekto in compthread %d\n", i); + fatal_goto(("Failed to seekto in compthread %d\n", i), error); if (unlikely(write_val(control, ctis->cur_pos, write_len))) - fatal(control, "Failed to write_val cur_pos in compthread %d\n", i); + fatal_goto(("Failed to write_val cur_pos in compthread %d\n", i), error); if (ENCRYPT) rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17); ctis->s[cti->streamno].last_head = ctis->cur_pos + 1 + (write_len * 2) + (ENCRYPT ? SALT_LEN : 0); if (unlikely(seekto(control, ctis, ctis->cur_pos))) - fatal(control, "Failed to seekto cur_pos in compthread %d\n", i); + fatal_goto(("Failed to seekto cur_pos in compthread %d\n", i), error); print_maxverbose("Thread %ld writing %lld compressed bytes from stream %d\n", i, padded_len, cti->streamno); if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal(control, "Failed to write_buf header salt in compthread %d\n", i); + fatal_goto(("Failed to write_buf header salt in compthread %d\n", i), error); ctis->cur_pos += SALT_LEN; ctis->s[cti->streamno].last_headofs = ctis->cur_pos; } @@ -1390,19 +1445,19 @@ static void *compthread(void *data) write_val(control, cti->c_len, write_len) || write_val(control, cti->s_len, write_len) || write_val(control, 0, write_len))) { - fatal(control, "Failed write in compthread %d\n", i); + fatal_goto(("Failed write in compthread %d\n", i), error); } ctis->cur_pos += 1 + (write_len * 3); if (ENCRYPT) { - get_rand(control, cti->salt, SALT_LEN); + if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error; if (unlikely(write_buf(control, cti->salt, SALT_LEN))) - fatal(control, "Failed to write_buf block salt in compthread %d\n", i); - lrz_encrypt(control, cti->s_buf, padded_len, cti->salt); + fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error); + if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; ctis->cur_pos += SALT_LEN; } if (unlikely(write_buf(control, cti->s_buf, padded_len))) - fatal(control, "Failed to write_buf s_buf in compthread %d\n", i); + fatal_goto(("Failed to write_buf s_buf in compthread %d\n", i), error); ctis->cur_pos += padded_len; free(cti->s_buf); @@ -1413,12 +1468,13 @@ static void *compthread(void *data) cond_broadcast(control, &output_cond); unlock_mutex(control, &output_lock); +error: unlock_mutex(control, &cti->mutex); - return 0; + return NULL; } -static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int streamno, int newbuf) +static bool clear_buffer(rzip_control *control, struct stream_info *sinfo, int streamno, int newbuf) { static long i = 0; stream_thread_struct *s; @@ -1435,30 +1491,39 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s i, cthread[i].s_len, streamno); s = malloc(sizeof(stream_thread_struct)); - if (unlikely(!s)) - fatal(control, "Unable to malloc in clear_buffer"); + if (unlikely(!s)) { + unlock_mutex(control, &cthread[i].mutex); + fatal_return(("Unable to malloc in clear_buffer"), false); + } s->i = i; s->control = control; - create_pthread(control, &threads[i], NULL, compthread, s); - detach_pthread(control, &threads[i]); + if (unlikely((!create_pthread(control, &threads[i], NULL, compthread, s)) || + (!detach_pthread(control, &threads[i])))) { + unlock_mutex(control, &cthread[i].mutex); + return false; + } + if (newbuf) { /* The stream buffer has been given to the thread, allocate a * new one. */ sinfo->s[streamno].buf = malloc(sinfo->bufsize); - if (unlikely(!sinfo->s[streamno].buf)) - fatal(control, "Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize); + if (unlikely(!sinfo->s[streamno].buf)) { + unlock_mutex(control, &cthread[i].mutex); + fatal_return(("Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize), false); + } sinfo->s[streamno].buflen = 0; } if (++i == control->threads) i = 0; + return true; } /* flush out any data in a stream buffer */ -void flush_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) +bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) { - clear_buffer(control, sinfo, streamno, 1); + return clear_buffer(control, sinfo, streamno, 1); } static void *ucompthread(void *data) @@ -1494,7 +1559,7 @@ static void *ucompthread(void *data) ret = zpaq_decompress_buf(control, uci, i); break; default: - failure(control, "Dunno wtf decompression type to use!\n"); + failure_return(("Dunno wtf decompression type to use!\n"), NULL); break; } } @@ -1503,7 +1568,7 @@ static void *ucompthread(void *data) * parallel */ if (unlikely(ret)) { if (unlikely(waited)) - failure(control, "Failed to decompress in ucompthread\n"); + failure_return(("Failed to decompress in ucompthread\n"), NULL); print_maxverbose("Unable to decompress in parallel, waiting for previous thread to complete before trying again\n"); /* We do not strictly need to wait for this, so it's used when * decompression fails due to inadequate memory to try again @@ -1536,7 +1601,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str goto out; fill_another: if (unlikely(ucthread[s->uthread_no].busy)) - failure(control, "Trying to start a busy thread, this shouldn't happen!\n"); + failure_return(("Trying to start a busy thread, this shouldn't happen!\n"), -1); if (unlikely(read_seekto(control, sinfo, s->last_head))) return -1; @@ -1576,7 +1641,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str } if (ENCRYPT) { - decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head); + if (unlikely(!decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head))) return -1; if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN))) return -1; } @@ -1589,14 +1654,14 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str s_buf = malloc(MAX(u_len, MIN_SIZE)); if (unlikely(u_len && !s_buf)) - fatal(control, "Unable to malloc buffer of size %lld in fill_buffer\n", u_len); + fatal_return(("Unable to malloc buffer of size %lld in fill_buffer\n", u_len), -1); sinfo->ram_alloced += u_len; if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len))) return -1; if (ENCRYPT) - lrz_decrypt(control, s_buf, padded_len, blocksalt); + if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt))) return -1; ucthread[s->uthread_no].s_buf = s_buf; ucthread[s->uthread_no].c_len = c_len; @@ -1612,10 +1677,13 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str st = malloc(sizeof(stream_thread_struct)); if (unlikely(!st)) - fatal(control, "Unable to malloc in fill_buffer"); + fatal_return(("Unable to malloc in fill_buffer"), -1); st->i = s->uthread_no; st->control = control; - create_pthread(control, &threads[s->uthread_no], NULL, ucompthread, st); + if (unlikely(!create_pthread(control, &threads[s->uthread_no], NULL, ucompthread, st))) { + free(st); + return -1; + } if (++s->uthread_no == s->base_thread + s->total_threads) s->uthread_no = s->base_thread; @@ -1636,7 +1704,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str unlock_mutex(control, &output_lock); /* join_pthread here will make it wait till the data is ready */ - join_pthread(control, threads[s->unext_thread], NULL); + if (unlikely(!join_pthread(control, threads[s->unext_thread], NULL))) return -1; ucthread[s->unext_thread].busy = 0; print_maxverbose("Taking decompressed data from thread %ld\n", s->unext_thread); @@ -1668,7 +1736,7 @@ int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 le /* Flush the buffer every sinfo->bufsize into one thread */ if (sinfo->s[streamno].buflen == sinfo->bufsize) - flush_buffer(control, sinfo, streamno); + if (unlikely(!flush_buffer(control, sinfo, streamno))) return -1; } return 0; } @@ -1712,7 +1780,7 @@ int close_stream_out(rzip_control *control, void *ss) for (i = 0; i < sinfo->num_streams; i++) { if (sinfo->s[i].buflen) - clear_buffer(control, sinfo, i, 0); + if (unlikely(!clear_buffer(control, sinfo, i, 0))) return -1; } if (ENCRYPT) { @@ -1779,14 +1847,16 @@ static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len) return 1; wrkmem = (lzo_bytep) malloc(LZO1X_1_MEM_COMPRESS); if (unlikely(wrkmem == NULL)) - fatal(control, "Unable to allocate wrkmem in lzo_compresses\n"); + fatal_return(("Unable to allocate wrkmem in lzo_compresses\n"), 0); in_len = MIN(test_len, buftest_size); dlen = STREAM_BUFSIZE + STREAM_BUFSIZE / 16 + 64 + 3; c_buf = malloc(dlen); - if (unlikely(!c_buf)) - fatal(control, "Unable to allocate c_buf in lzo_compresses\n"); + if (unlikely(!c_buf)) { + free(wrkmem); + fatal_return(("Unable to allocate c_buf in lzo_compresses\n"), 0); + } /* Test progressively larger blocks at a time and as soon as anything compressible is found, jump out as a success */ diff --git a/stream.h b/stream.h index e91a170f..69677020 100644 --- a/stream.h +++ b/stream.h @@ -23,16 +23,16 @@ #include "lrzip_private.h" #include -void create_pthread(pthread_t *thread, pthread_attr_t *attr, +bool create_pthread(pthread_t *thread, pthread_attr_t *attr, void * (*start_routine)(void *), void *arg); -void join_pthread(pthread_t th, void **thread_return); +bool join_pthread(pthread_t th, void **thread_return); ssize_t write_1g(rzip_control *control, void *buf, i64 len); ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len); -void prepare_streamout_threads(rzip_control *control); -void close_streamout_threads(rzip_control *control); +bool prepare_streamout_threads(rzip_control *control); +bool close_streamout_threads(rzip_control *control); void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes); void *open_stream_in(rzip_control *control, int f, int n, char cbytes); -void flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); +bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); int close_stream_out(rzip_control *control, void *ss); diff --git a/util.c b/util.c index 153d2045..5c75377c 100644 --- a/util.c +++ b/util.c @@ -91,7 +91,7 @@ void unlink_files(rzip_control *control) unlink(control->util_infile); } -static void fatal_exit(rzip_control *control) +void fatal_exit(rzip_control *control) { struct termios termios_p; @@ -103,35 +103,7 @@ static void fatal_exit(rzip_control *control) unlink_files(control); fprintf(control->outputfile, "Fatal error - exiting\n"); fflush(control->outputfile); - abort(); -} - -/* Failure when there is likely to be a meaningful error in perror */ -void fatal(const rzip_control *control, const char *format, ...) -{ - va_list ap; - - if (format) { - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); - } - - perror(NULL); - fatal_exit((rzip_control*)control); -} - -void failure(const rzip_control *control, const char *format, ...) -{ - va_list ap; - - if (format) { - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); - } - - fatal_exit((rzip_control*)control); + exit(1); } void setup_overhead(rzip_control *control) @@ -178,7 +150,7 @@ void round_to_page(i64 *size) *size = PAGE_SIZE; } -void get_rand(rzip_control *control, uchar *buf, int len) +bool get_rand(rzip_control *control, uchar *buf, int len) { int fd, i; @@ -188,26 +160,20 @@ void get_rand(rzip_control *control, uchar *buf, int len) buf[i] = (uchar)random(); } else { if (unlikely(read(fd, buf, len) != len)) - fatal(control, "Failed to read fd in get_rand\n"); + fatal_return(("Failed to read fd in get_rand\n"), false); if (unlikely(close(fd))) - fatal(control, "Failed to close fd in get_rand\n"); + fatal_return(("Failed to close fd in get_rand\n"), false); } -} - -void read_config(rzip_control *control) + return true; +}bool read_config(rzip_control *control) { /* check for lrzip.conf in ., $HOME/.lrzip and /etc/lrzip */ - char *HOME, *homeconf; + char *HOME, homeconf[255]; char *parametervalue; char *parameter; - char *line; + char line[255]; FILE *fp; - line = malloc(255); - homeconf = malloc(255); - if (line == NULL || homeconf == NULL) - fatal(control, "Fatal Memory Error in read_config"); - fp = fopen("lrzip.conf", "r"); if (fp) fprintf(control->msgout, "Using configuration file ./lrzip.conf\n"); @@ -219,15 +185,13 @@ void read_config(rzip_control *control) if (fp == NULL) { HOME=getenv("HOME"); if (HOME) { - strcpy(homeconf, HOME); - strcat(homeconf,"/.lrzip/lrzip.conf"); + snprintf(homeconf, sizeof(homeconf), "%s/.lrzip/lrzip.conf", HOME); fp = fopen(homeconf, "r"); if (fp) fprintf(control->msgout, "Using configuration file %s\n", homeconf); } } - if (fp == NULL) - goto out; + if (fp == NULL) return true; /* if we get here, we have a file. read until no more. */ @@ -257,11 +221,11 @@ void read_config(rzip_control *control) } else if (isparameter(parameter, "compressionlevel")) { control->compression_level = atoi(parametervalue); if ( control->compression_level < 1 || control->compression_level > 9 ) - failure(control, "CONF.FILE error. Compression Level must between 1 and 9"); + failure_return(("CONF.FILE error. Compression Level must between 1 and 9"), false); } else if (isparameter(parameter, "compressionmethod")) { /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ if (control->flags & FLAG_NOT_LZMA) - failure(control, "CONF.FILE error. Can only specify one compression method"); + failure_return(("CONF.FILE error. Can only specify one compression method"), false); if (isparameter(parametervalue, "bzip2")) control->flags |= FLAG_BZIP2_COMPRESS; else if (isparameter(parametervalue, "gzip")) @@ -273,7 +237,7 @@ void read_config(rzip_control *control) else if (isparameter(parametervalue, "zpaq")) control->flags |= FLAG_ZPAQ_COMPRESS; else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ - failure(control, "CONF.FILE error. Invalid compression method %s specified\n",parametervalue); + failure_return(("CONF.FILE error. Invalid compression method %s specified\n",parametervalue), false); } else if (isparameter(parameter, "lzotest")) { /* default is yes */ if (isparameter(parametervalue, "no")) @@ -289,13 +253,13 @@ void read_config(rzip_control *control) } else if (isparameter(parameter, "outputdirectory")) { control->outdir = malloc(strlen(parametervalue) + 2); if (!control->outdir) - fatal(control, "Fatal Memory Error in read_config"); + fatal_return(("Fatal Memory Error in read_config"), false); strcpy(control->outdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->outdir, "/"); } else if (isparameter(parameter,"verbosity")) { if (control->flags & FLAG_VERBOSE) - failure(control, "CONF.FILE error. Verbosity already defined."); + failure_return(("CONF.FILE error. Verbosity already defined."), false); if (isparameter(parametervalue, "yes")) control->flags |= FLAG_VERBOSITY; else if (isparameter(parametervalue,"max")) @@ -309,7 +273,7 @@ void read_config(rzip_control *control) } else if (isparameter(parameter,"nice")) { control->nice_val = atoi(parametervalue); if (control->nice_val < -20 || control->nice_val > 19) - failure(control, "CONF.FILE error. Nice must be between -20 and 19"); + failure_return(("CONF.FILE error. Nice must be between -20 and 19"), false); } else if (isparameter(parameter, "keepbroken")) { if (isparameter(parametervalue, "yes" )) control->flags |= FLAG_KEEP_BROKEN; @@ -324,7 +288,7 @@ void read_config(rzip_control *control) } else if (isparameter(parameter, "tmpdir")) { control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); if (!control->tmpdir) - fatal(control, "Fatal Memory Error in read_config"); + fatal_return(("Fatal Memory Error in read_config"), false); strcpy(control->tmpdir, parametervalue); if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) strcat(control->tmpdir, "/"); @@ -338,11 +302,7 @@ void read_config(rzip_control *control) } if (unlikely(fclose(fp))) - fatal(control, "Failed to fclose fp in read_config\n"); -out: - /* clean up */ - free(line); - free(homeconf); + fatal_return(("Failed to fclose fp in read_config\n"), false); /* fprintf(stderr, "\nWindow = %d \ \nCompression Level = %d \ @@ -350,6 +310,7 @@ void read_config(rzip_control *control) \nOutput Directory = %s \ \nFlags = %d\n", control->window,control->compression_level, control->threshold, control->outdir, control->flags); */ + return true; } static void xor128 (void *pa, const void *pb) @@ -380,7 +341,7 @@ static void lrz_keygen(const rzip_control *control, const uchar *salt, uchar *ke munlock(buf, sizeof(buf)); } -void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt) +bool lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt) { /* Encryption requires CBC_LEN blocks so we can use ciphertext * stealing to not have to pad the block */ @@ -388,6 +349,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa uchar tmp0[CBC_LEN], tmp1[CBC_LEN]; aes_context aes_ctx; i64 N, M; + bool ret = false; /* Generate unique key and IV for each block of data based on salt */ mlock(&aes_ctx, sizeof(aes_ctx)); @@ -402,7 +364,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa if (encrypt == LRZ_ENCRYPT) { print_maxverbose("Encrypting data \n"); if (unlikely(aes_setkey_enc(&aes_ctx, key, 128))) - failure(control, "Failed to aes_setkey_enc in lrz_crypt\n"); + failure_goto(("Failed to aes_setkey_enc in lrz_crypt\n"), error); aes_crypt_cbc(&aes_ctx, AES_ENCRYPT, N, iv, buf, buf); if (M) { @@ -415,7 +377,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa } } else { if (unlikely(aes_setkey_dec(&aes_ctx, key, 128))) - failure(control, "Failed to aes_setkey_dec in lrz_crypt\n"); + failure_goto(("Failed to aes_setkey_dec in lrz_crypt\n"), error); print_maxverbose("Decrypting data \n"); if (M) { aes_crypt_cbc(&aes_ctx, AES_DECRYPT, N - CBC_LEN, @@ -435,12 +397,15 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa iv, buf, buf); } + ret = true; +error: memset(&aes_ctx, 0, sizeof(aes_ctx)); memset(iv, 0, HASH_LEN); memset(key, 0, HASH_LEN); munlock(&aes_ctx, sizeof(aes_ctx)); munlock(iv, HASH_LEN); munlock(key, HASH_LEN); + return ret; } void lrz_stretch(rzip_control *control) diff --git a/util.h b/util.h index fed28f2c..55d60f9b 100644 --- a/util.h +++ b/util.h @@ -20,33 +20,85 @@ #define LRZIP_UTIL_H #include "lrzip_private.h" +#include void register_infile(rzip_control *control, const char *name, char delete); void register_outfile(rzip_control *control, const char *name, char delete); void unlink_files(rzip_control *control); void register_outputfile(rzip_control *control, FILE *f); -void fatal(const rzip_control *control, const char *format, ...); -void failure(const rzip_control *control, const char *format, ...); +void fatal_exit(rzip_control *control); +/* Failure when there is likely to be a meaningful error in perror */ +static inline void fatal(const rzip_control *control, unsigned int line, const char *file, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + if (!control->log_cb) { + vfprintf(stderr, format, ap); + perror(NULL); + } else + control->log_cb(control->log_data, line, file, format, ap); + va_end(ap); + if (!control->library_mode) + fatal_exit((rzip_control*)control); +} +#ifdef fatal +# undef fatal +#endif +#define fatal(stuff...) fatal(control, __LINE__, __FILE__, stuff) +#define fatal_return(stuff, ...) do { \ + fatal stuff; \ + return __VA_ARGS__; \ +} while (0) +#define fatal_goto(stuff, label) do { \ + fatal stuff; \ + goto label; \ +} while (0) +static inline void failure(const rzip_control *control, unsigned int line, const char *file, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + if (!control->log_cb) + vfprintf(stderr, format, ap); + else + control->log_cb(control->log_data, line, file, format, ap); + va_end(ap); + if (!control->library_mode) + fatal_exit((rzip_control*)control); +} +#ifdef failure +# undef failure +#endif +#define failure(stuff...) failure(control, __LINE__, __FILE__, stuff) +#define failure_return(stuff, ...) do { \ + failure stuff; \ + return __VA_ARGS__; \ +} while (0) +#define failure_goto(stuff, label) do { \ + failure stuff; \ + goto label; \ +} while (0) void setup_overhead(rzip_control *control); void setup_ram(rzip_control *control); void round_to_page(i64 *size); -void get_rand(rzip_control *control, uchar *buf, int len); -void read_config(rzip_control *control); +bool get_rand(rzip_control *control, uchar *buf, int len); +bool read_config(rzip_control *control); void lrz_stretch(rzip_control *control); void lrz_stretch2(rzip_control *control); -void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); +bool lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); #define LRZ_DECRYPT (0) #define LRZ_ENCRYPT (1) -static inline void lrz_encrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) +static inline bool lrz_encrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) { - lrz_crypt(control, buf, len, salt, LRZ_ENCRYPT); + return lrz_crypt(control, buf, len, salt, LRZ_ENCRYPT); } -static inline void lrz_decrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) +static inline bool lrz_decrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) { - lrz_crypt(control, buf, len, salt, LRZ_DECRYPT); + return lrz_crypt(control, buf, len, salt, LRZ_DECRYPT); } #endif From b6dc92a7afc3f99949c0483341378e7b72dabef5 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 02:20:20 -0400 Subject: [PATCH 13/30] test for file existence and delete files between tests --- regression_test.sh | 55 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/regression_test.sh b/regression_test.sh index 086905ec..54c6c38f 100755 --- a/regression_test.sh +++ b/regression_test.sh @@ -33,34 +33,37 @@ trap 'echo "ABORTING";end;exit' 1 2 15 echo testing compression from stdin ./lrzip -vvlfo lrztest.lrz < $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression from stdin end exit 1 fi +rm lrztest.lrz echo testing compression to stdout ./lrzip -vvlo - $infile > lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression to stdout end exit 1 fi +rm lrztest.lrz echo testing compression from stdin to stdout ./lrzip -vvl < $infile > lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression from stdin to stdout end exit 1 fi +rm lrztest.lrz echo testing standard compression ./lrzip -vvlfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing standard compression end exit 1 @@ -69,52 +72,57 @@ fi echo testing standard decompression ./lrzip -vvdo lrztest lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing standard decompression end exit 1 fi +rm lrztest echo testing standard decompression with file checking ./lrzip -vvdfco lrztest lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing standard decompression with file checking end exit 1 fi +rm lrztest echo testing decompression from stdin ./lrzip -vvfo lrztest -d < lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression from stdin end exit 1 fi +rm lrztest echo testing decompression to stdout ./lrzip -vvdo - lrztest.lrz > lrztest -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression to stdout end exit 1 fi +rm lrztest echo testing decompression from stdin to stdout ./lrzip -vvd < lrztest.lrz > lrztest -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression from stdin to stdout end exit 1 fi +rm lrztest echo testing testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing testing end exit 1 @@ -123,16 +131,17 @@ fi echo testing testing from stdin ./lrzip -vvt < lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing testing from stdin end exit 1 fi +rm lrztest.lrz echo testing rzip only compression ./lrzip -vvnfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing rzip only compression end exit 1 @@ -141,16 +150,17 @@ fi echo testing rzip only testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing rzip only testing end exit 1 fi +rm lrztest.lrz echo testing lzma compression ./lrzip -vvfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing lzma compression end exit 1 @@ -159,16 +169,17 @@ fi echo testing lzma testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing lzma testing end exit 1 fi +rm lrztest.lrz echo testing gzip compression ./lrzip -vvgfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing gzip compression end exit 1 @@ -177,16 +188,17 @@ fi echo testing gzip testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing gzip testing end exit 1 fi +rm lrztest.lrz echo testing bzip2 compression ./lrzip -vvbfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing bzip2 compression end exit 1 @@ -195,16 +207,17 @@ fi echo testing bzip2 testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing bzip2 testing end exit 1 fi +rm lrztest.lrz echo testing zpaq compression ./lrzip -vvzfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing zpaq compression end exit 1 @@ -213,7 +226,7 @@ fi echo testing zpaq testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing zpaq testing end exit 1 From 949c6d7a97af4196d4e0de1c5a368c8b6d25e39f Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 02:21:45 -0400 Subject: [PATCH 14/30] another semi-clusterfuck commit: *new logging macros *allow compression to and from any FILE, not just stdin/stdout *update main.c to work with the above --- lrzip.c | 13 +++---- lrzip_private.h | 93 +++++++++++++++++++++++++++++++++++-------------- main.c | 6 +++- runzip.c | 18 +++++----- rzip.c | 14 ++++---- stream.c | 7 ++-- util.c | 1 - util.h | 12 +++---- 8 files changed, 102 insertions(+), 62 deletions(-) diff --git a/lrzip.c b/lrzip.c index 15673e1a..0c3463eb 100644 --- a/lrzip.c +++ b/lrzip.c @@ -52,7 +52,6 @@ #include "runzip.h" #include "util.h" #include "stream.h" -#include "liblrzip.h" /* flag defines */ #define MAGIC_LEN (24) @@ -319,14 +318,14 @@ static bool fwrite_stdout(rzip_control *control, void *buf, i64 len) ret = one_g; else ret = len; - ret = fwrite(offset_buf, 1, ret, stdout); + ret = fwrite(offset_buf, 1, ret, control->outFILE); if (unlikely(ret <= 0)) fatal_return(("Failed to fwrite in fwrite_stdout\n"), false); len -= ret; offset_buf += ret; total += ret; } - fflush(stdout); + fflush(control->outFILE); return true; } @@ -375,10 +374,10 @@ bool dump_tmpoutfile(rzip_control *control, int fd_out) rewind(tmpoutfp); if (!TEST_ONLY) { - print_verbose("Dumping temporary file to stdout.\n"); + print_verbose("Dumping temporary file to control->outFILE.\n"); while ((tmpchar = fgetc(tmpoutfp)) != EOF) putchar(tmpchar); - fflush(stdout); + fflush(control->outFILE); rewind(tmpoutfp); } @@ -564,7 +563,7 @@ static bool get_hash(rzip_control *control, int make_hash) mlock(control->hash, HASH_LEN); if (control->pass_cb) { - control->pass_cb(control->pass_data, passphrase, PASS_LEN); + control->pass_cb(control->pass_data, passphrase, PASS_LEN - SALT_LEN); if (!passphrase[0]) { fatal("Supplied password was null!"); munlock(passphrase, PASS_LEN); @@ -574,6 +573,7 @@ static bool get_hash(rzip_control *control, int make_hash) release_hashes(control); return false; } + control->salt_pass_len = strlen(passphrase) + SALT_LEN; } else { /* Disable stdin echo to screen */ tcgetattr(fileno(stdin), &termios_p); @@ -1194,6 +1194,7 @@ bool initialize_control(rzip_control *control) memset(control, 0, sizeof(rzip_control)); control->msgout = stderr; + control->msgerr = stderr; register_outputfile(control, control->msgout); control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; control->suffix = strdup(".lrz"); diff --git a/lrzip_private.h b/lrzip_private.h index 3b8b98ed..3dcb9398 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -27,6 +27,7 @@ #include #include #include +#include #ifdef HAVE_STRING_H # include #endif @@ -52,6 +53,10 @@ extern "C" void *alloca (size_t); #endif +#ifndef MD5_DIGEST_SIZE +# define MD5_DIGEST_SIZE 16 +#endif + #define free(X) do { free((X)); (X) = NULL; } while (0) #ifndef strdupa @@ -262,31 +267,6 @@ typedef struct md5_ctx md5_ctx; #define TMP_INBUF (control->flags & FLAG_TMP_INBUF) #define ENCRYPT (control->flags & FLAG_ENCRYPT) -#define print_output(format, args...) do {\ - fprintf(control->msgout, format, ##args); \ - fflush(control->msgout); \ -} while (0) - -#define print_progress(format, args...) do {\ - if (SHOW_PROGRESS) \ - print_output(format, ##args); \ -} while (0) - -#define print_verbose(format, args...) do {\ - if (VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -#define print_maxverbose(format, args...) do {\ - if (MAX_VERBOSE) \ - print_output(format, ##args); \ -} while (0) - - -#define print_err(format, args...) do {\ - fprintf(stderr, format, ##args); \ -} while (0) - /* Structure to save state of computation between the single steps. */ struct md5_ctx @@ -303,8 +283,10 @@ struct md5_ctx struct rzip_control { char *infile; + FILE *inFILE; // if a FILE is being read from char *outname; char *outfile; + FILE *outFILE; // if a FILE is being written to char *outdir; char *tmpdir; // when stdin, stdout, or test used uchar *tmp_outbuf; // Temporary file storage for stdout @@ -318,6 +300,7 @@ struct rzip_control { i64 in_len; i64 in_maxlen; FILE *msgout; //stream for output messages + FILE *msgerr; //stream for output errors char *suffix; uchar compression_level; i64 overhead; // compressor overhead @@ -349,6 +332,7 @@ struct rzip_control { unsigned char eof; unsigned char magic_written; md5_ctx ctx; + uchar md5_resblock[MD5_DIGEST_SIZE]; i64 md5_read; // How far into the file the md5 has done so far const char *util_infile; char delete_infile; @@ -356,7 +340,8 @@ struct rzip_control { char delete_outfile; FILE *outputfile; char library_mode : 1; - void (*log_cb)(void *, unsigned int, const char *, const char *, va_list); + int log_level; + void (*log_cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *func, const char *format, va_list); void *log_data; }; @@ -387,4 +372,60 @@ struct stream_info { int chunks; char chunk_bytes; }; + +static inline void print_stuff(const rzip_control *control, int level, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + if (control->library_mode && control->log_cb && (control->log_level >= level)) { + va_start(ap, format); + control->log_cb(control->log_data, level, line, file, func, format, ap); + va_end(ap); + } else if (control->msgout) { + va_start(ap, format); + vfprintf(control->msgout, format, ap); + fflush(control->msgout); + va_end(ap); + } +} + +static inline void print_err(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + if (control->library_mode && control->log_cb && (control->log_level >= 0)) { + va_start(ap, format); + control->log_cb(control->log_data, 0, line, file, func, format, ap); + va_end(ap); + } else if (control->msgerr) { + va_start(ap, format); + vfprintf(control->msgerr, format, ap); + va_end(ap); + } +} + +#define print_stuff(level, format, args...) do {\ + print_stuff(control, level, __LINE__, __FILE__, __func__, format, ##args); \ +} while (0) + +#define print_output(format, args...) do {\ + print_stuff(1, format, ##args); \ +} while (0) + +#define print_progress(format, args...) do {\ + if (SHOW_PROGRESS) \ + print_stuff(2, format, ##args); \ +} while (0) + +#define print_verbose(format, args...) do {\ + if (VERBOSE) \ + print_stuff(3, format, ##args); \ +} while (0) + +#define print_maxverbose(format, args...) do {\ + if (MAX_VERBOSE) \ + print_stuff(4, format, ##args); \ +} while (0) + +#define print_err(format, args...) do {\ + print_err(control, __LINE__, __FILE__, __func__, format, ##args); \ +} while (0) #endif diff --git a/main.c b/main.c index 122cc8be..bae4bb61 100644 --- a/main.c +++ b/main.c @@ -49,7 +49,6 @@ # include #endif -#include "liblrzip.h" #include "rzip.h" #include "lrzip.h" #include "util.h" @@ -284,6 +283,7 @@ int main(int argc, char *argv[]) if (unlikely(STDOUT)) failure("Cannot specify an output filename when outputting to stdout\n"); control->outname = optarg; + free(control->suffix); control->suffix = ""; break; case 'O': @@ -311,6 +311,7 @@ int main(int argc, char *argv[]) failure("Specified output filename already, can't specify an extension.\n"); if (unlikely(STDOUT)) failure("Cannot specify a filename suffix when outputting to stdout\n"); + free(control->suffix); control->suffix = optarg; break; case 't': @@ -410,6 +411,7 @@ int main(int argc, char *argv[]) if (control->outname && (strcmp(control->outname, "-") == 0)) { control->flags |= FLAG_STDOUT; + control->outFILE = stdout; control->msgout = stderr; register_outputfile(control, control->msgout); } @@ -419,6 +421,7 @@ int main(int argc, char *argv[]) if (!control->outname && STDIN) { control->flags |= FLAG_STDOUT; control->msgout = stderr; + control->outFILE = stdout; register_outputfile(control, control->msgout); } @@ -426,6 +429,7 @@ int main(int argc, char *argv[]) control->msgout = stdout; register_outputfile(control, control->msgout); } + if (STDIN) control->inFILE = stdin; /* Implement signal handler only once flags are set */ handler.sa_handler = &sighandler; sigaction(SIGTERM, &handler, 0); diff --git a/runzip.c b/runzip.c index a6cd3589..0a2ee827 100644 --- a/runzip.c +++ b/runzip.c @@ -45,7 +45,6 @@ #include "stream.h" #include "util.h" #include "lrzip.h" -#include "liblrzip.h" /* needed for CRC routines */ #include "lzma/C/7zCrc.h" @@ -364,7 +363,6 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 */ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size) { - uchar md5_resblock[MD5_DIGEST_SIZE]; uchar md5_stored[MD5_DIGEST_SIZE]; struct timeval start,end; i64 total = 0, u; @@ -395,7 +393,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (!NO_MD5) { int i,j; - md5_finish_ctx (&control->ctx, md5_resblock); + md5_finish_ctx (&control->ctx, control->md5_resblock); if (HAS_MD5) { i64 fdinend = seekto_fdinend(control); @@ -409,13 +407,13 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (ENCRYPT) if (unlikely(!lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass))) return -1; for (i = 0; i < MD5_DIGEST_SIZE; i++) - if (md5_stored[i] != md5_resblock[i]) { + if (md5_stored[i] != control->md5_resblock[i]) { print_output("MD5 CHECK FAILED.\nStored:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_stored[j] & 0xFF); print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); + print_output("%02x", control->md5_resblock[j] & 0xFF); failure_return(("\n"), -1); } } @@ -423,7 +421,7 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (HASH_CHECK || MAX_VERBOSE) { print_output("MD5: "); for (i = 0; i < MD5_DIGEST_SIZE; i++) - print_output("%02x", md5_resblock[i] & 0xFF); + print_output("%02x", control->md5_resblock[i] & 0xFF); print_output("\n"); } @@ -433,22 +431,22 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (TMP_OUTBUF) close_tmpoutbuf(control); - memcpy(md5_stored, md5_resblock, MD5_DIGEST_SIZE); + memcpy(md5_stored, control->md5_resblock, MD5_DIGEST_SIZE); if (unlikely(seekto_fdhist(control, 0) == -1)) fatal_return(("Failed to seekto_fdhist in runzip_fd\n"), -1); if (unlikely((md5_fstream = fdopen(fd_hist, "r")) == NULL)) fatal_return(("Failed to fdopen fd_hist in runzip_fd\n"), -1); - if (unlikely(md5_stream(md5_fstream, md5_resblock))) + if (unlikely(md5_stream(md5_fstream, control->md5_resblock))) fatal_return(("Failed to md5_stream in runzip_fd\n"), -1); /* We don't close the file here as it's closed in main */ for (i = 0; i < MD5_DIGEST_SIZE; i++) - if (md5_stored[i] != md5_resblock[i]) { + if (md5_stored[i] != control->md5_resblock[i]) { print_output("MD5 CHECK FAILED.\nStored:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_stored[j] & 0xFF); print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); + print_output("%02x", control->md5_resblock[j] & 0xFF); failure_return(("\n"), -1); } print_output("MD5 integrity of written file matches archive\n"); diff --git a/rzip.c b/rzip.c index a85728e5..9a3a00e2 100644 --- a/rzip.c +++ b/rzip.c @@ -54,7 +54,6 @@ #include "md5.h" #include "stream.h" #include "util.h" -#include "liblrzip.h" #include "lrzip.h" /* needed for CRC routines */ #include "lzma/C/7zCrc.h" @@ -712,7 +711,7 @@ static bool mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) total = 0; while (len > 0) { ret = MIN(len, one_g); - ret = read(0, offset_buf, (size_t)ret); + ret = read(fileno(control->inFILE), offset_buf, (size_t)ret); if (unlikely(ret < 0)) fatal_return(("Failed to read in mmap_stdin\n"), false); total += ret; @@ -806,7 +805,6 @@ bool rzip_fd(rzip_control *control, int fd_in, int fd_out) * If file size < compression window, can't do */ struct timeval current, start, last; - uchar md5_resblock[MD5_DIGEST_SIZE]; i64 len = 0, last_chunk = 0; int pass = 0, passes, j; struct rzip_state *st; @@ -1066,20 +1064,20 @@ bool rzip_fd(rzip_control *control, int fd_in, int fd_out) if (!NO_MD5) { /* Temporary workaround till someone fixes apple md5 */ - md5_finish_ctx(&control->ctx, md5_resblock); + md5_finish_ctx(&control->ctx, control->md5_resblock); if (HASH_CHECK || MAX_VERBOSE) { print_output("MD5: "); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); + print_output("%02x", control->md5_resblock[j] & 0xFF); print_output("\n"); } /* When encrypting data, we encrypt the MD5 value as well */ if (ENCRYPT) - if (unlikely(!lrz_encrypt(control, md5_resblock, MD5_DIGEST_SIZE, control->salt_pass))) { + if (unlikely(!lrz_encrypt(control, control->md5_resblock, MD5_DIGEST_SIZE, control->salt_pass))) { free(st); return false; } - if (unlikely(write_1g(control, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) { + if (unlikely(write_1g(control, control->md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) { free(st); fatal_return(("Failed to write md5 in rzip_fd\n"), false); } @@ -1122,6 +1120,6 @@ void rzip_control_free(rzip_control *control) if (!control) return; free(control->tmpdir); - free(control->suffix); + if (control->suffix && control->suffix[0]) free(control->suffix); free(control); } diff --git a/stream.c b/stream.c index da797449..c946bfa3 100644 --- a/stream.c +++ b/stream.c @@ -60,7 +60,6 @@ #include "util.h" #include "zpipe.h" -#include "liblrzip.h" #include "lrzip.h" @@ -865,7 +864,7 @@ static inline int read_val(rzip_control *control, int f, i64 *v, int len) return ret; } -static int fd_seekto(struct stream_info *sinfo, i64 spos, i64 pos) +static int fd_seekto(rzip_control *control, struct stream_info *sinfo, i64 spos, i64 pos) { if (unlikely(lseek(sinfo->fd, spos, SEEK_SET) != spos)) { print_err("Failed to seek to %lld in stream\n", pos); @@ -889,7 +888,7 @@ static int seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) return 0; } - return fd_seekto(sinfo, spos, pos); + return fd_seekto(control, sinfo, spos, pos); } static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) @@ -907,7 +906,7 @@ static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos return 0; } - return fd_seekto(sinfo, spos, pos); + return fd_seekto(control, sinfo, spos, pos); } static i64 get_seek(rzip_control *control, int fd) diff --git a/util.c b/util.c index 5c75377c..58b1fa6a 100644 --- a/util.c +++ b/util.c @@ -52,7 +52,6 @@ #include #include #include "lrzip_private.h" -#include "liblrzip.h" #include "util.h" #include "sha4.h" #include "aes.h" diff --git a/util.h b/util.h index 55d60f9b..91932405 100644 --- a/util.h +++ b/util.h @@ -28,7 +28,7 @@ void unlink_files(rzip_control *control); void register_outputfile(rzip_control *control, FILE *f); void fatal_exit(rzip_control *control); /* Failure when there is likely to be a meaningful error in perror */ -static inline void fatal(const rzip_control *control, unsigned int line, const char *file, const char *format, ...) +static inline void fatal(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) { va_list ap; @@ -37,7 +37,7 @@ static inline void fatal(const rzip_control *control, unsigned int line, const c vfprintf(stderr, format, ap); perror(NULL); } else - control->log_cb(control->log_data, line, file, format, ap); + control->log_cb(control->log_data, 0, line, file, func, format, ap); va_end(ap); if (!control->library_mode) fatal_exit((rzip_control*)control); @@ -45,7 +45,7 @@ static inline void fatal(const rzip_control *control, unsigned int line, const c #ifdef fatal # undef fatal #endif -#define fatal(stuff...) fatal(control, __LINE__, __FILE__, stuff) +#define fatal(stuff...) fatal(control, __LINE__, __FILE__, __func__, stuff) #define fatal_return(stuff, ...) do { \ fatal stuff; \ return __VA_ARGS__; \ @@ -54,7 +54,7 @@ static inline void fatal(const rzip_control *control, unsigned int line, const c fatal stuff; \ goto label; \ } while (0) -static inline void failure(const rzip_control *control, unsigned int line, const char *file, const char *format, ...) +static inline void failure(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) { va_list ap; @@ -62,7 +62,7 @@ static inline void failure(const rzip_control *control, unsigned int line, const if (!control->log_cb) vfprintf(stderr, format, ap); else - control->log_cb(control->log_data, line, file, format, ap); + control->log_cb(control->log_data, 0, line, file, func, format, ap); va_end(ap); if (!control->library_mode) fatal_exit((rzip_control*)control); @@ -70,7 +70,7 @@ static inline void failure(const rzip_control *control, unsigned int line, const #ifdef failure # undef failure #endif -#define failure(stuff...) failure(control, __LINE__, __FILE__, stuff) +#define failure(stuff...) failure(control, __LINE__, __FILE__, __func__, stuff) #define failure_return(stuff, ...) do { \ failure stuff; \ return __VA_ARGS__; \ From 63141a103568eb43996ebf5e48697fc4dcb11629 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 03:35:36 -0400 Subject: [PATCH 15/30] add some gross bucketing to implement a queue for freeing stream_info structs after an operation has completed (in library mode ONLY; no changes will occur during regular execution) to fix some leaks --- lrzip_private.h | 4 ++++ rzip.c | 17 +++++++++++++---- stream.c | 25 +++++++++++++++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/lrzip_private.h b/lrzip_private.h index 3dcb9398..0778c89d 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -337,6 +337,10 @@ struct rzip_control { const char *util_infile; char delete_infile; const char *util_outfile; +#define STREAM_BUCKET_SIZE 20 + size_t sinfo_buckets; + size_t sinfo_idx; + struct stream_info **sinfo_queue; char delete_outfile; FILE *outputfile; char library_mode : 1; diff --git a/rzip.c b/rzip.c index 9a3a00e2..ad7419c4 100644 --- a/rzip.c +++ b/rzip.c @@ -1117,9 +1117,18 @@ bool rzip_fd(rzip_control *control, int fd_in, int fd_out) void rzip_control_free(rzip_control *control) { - if (!control) return; + size_t x; + if (!control) return; - free(control->tmpdir); - if (control->suffix && control->suffix[0]) free(control->suffix); - free(control); + free(control->tmpdir); + free(control->outname); + free(control->outdir); + if (control->suffix && control->suffix[0]) free(control->suffix); + + for (x = 0; x < control->sinfo_idx; x++) { + free(control->sinfo_queue[x]->s); + free(control->sinfo_queue[x]); + } + free(control->sinfo_queue); + free(control); } diff --git a/stream.c b/stream.c index c946bfa3..2153c279 100644 --- a/stream.c +++ b/stream.c @@ -1797,11 +1797,28 @@ int close_stream_out(rzip_control *control, void *ss) for (i = 0; i < sinfo->num_streams; i++) rewrite_encrypted(control, sinfo, sinfo->s[i].last_headofs); } - + if (control->library_mode) { + if (!control->sinfo_buckets) { + /* no streams added */ + control->sinfo_queue = calloc(STREAM_BUCKET_SIZE + 1, sizeof(void*)); + if (!control->sinfo_queue) return -1; + control->sinfo_buckets++; + } else if (control->sinfo_idx == STREAM_BUCKET_SIZE * control->sinfo_buckets + 1) { + /* all buckets full, create new bucket */ + void *tmp; + + tmp = realloc(control->sinfo_queue, (++control->sinfo_buckets * STREAM_BUCKET_SIZE + 1) * sizeof(void*)); + if (!tmp) return -1; + control->sinfo_queue = tmp; + memset(control->sinfo_queue + control->sinfo_idx, 0, ((control->sinfo_buckets * STREAM_BUCKET_SIZE + 1) - control->sinfo_idx) * sizeof(void*)); + } + control->sinfo_queue[control->sinfo_idx++] = sinfo; + } #if 0 - /* These cannot be freed because their values are read after the next - * stream has started so they're not properly freed and just dropped on - * program exit! FIXME */ + /* These cannot be freed immediately because their values are read after the next + * stream has started. Instead (in library mode), they are stored and only freed + * after the entire operation has completed. + */ free(sinfo->s); free(sinfo); #endif From f32191bee1b8469750185dbe2981df45612a3027 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 03:43:19 -0400 Subject: [PATCH 16/30] v0.00000001 of liblrzip --- liblrzip.c | 505 +++++++++++++++++++++++++++++++++++---------- liblrzip.h | 76 +++++-- liblrzip_private.h | 8 +- 3 files changed, 461 insertions(+), 128 deletions(-) diff --git a/liblrzip.c b/liblrzip.c index ac01c98f..b9cfee1f 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -19,6 +19,77 @@ #include "lrzip.h" #include "rzip.h" +static void liblrzip_index_update(size_t x, size_t *idx, void **queue) +{ + for (; x < *idx; x++) + queue[x] = queue[x] + 1; + (*idx)--; +} + +static bool liblrzip_setup_flags(Lrzip *lr) +{ + if (!lr) return false; +#define MODE_CHECK(X) \ + case LRZIP_MODE_COMPRESS_##X: \ + lr->control->flags ^= FLAG_NOT_LZMA; \ + lr->control->flags |= FLAG_##X##_COMPRESS; \ + break + + + switch (lr->mode) { + case LRZIP_MODE_DECOMPRESS: + lr->control->flags |= FLAG_DECOMPRESS; + break; + case LRZIP_MODE_TEST: + lr->control->flags |= FLAG_TEST_ONLY; + break; + case LRZIP_MODE_INFO: + lr->control->flags |= FLAG_INFO; + break; + case LRZIP_MODE_COMPRESS_NONE: + lr->control->flags ^= FLAG_NOT_LZMA; + lr->control->flags |= FLAG_NO_COMPRESS; + break; + case LRZIP_MODE_COMPRESS_LZMA: + lr->control->flags ^= FLAG_NOT_LZMA; + break; + MODE_CHECK(LZO); + MODE_CHECK(BZIP2); + MODE_CHECK(ZLIB); + MODE_CHECK(ZPAQ); +#undef MODE_CHECK + default: + return false; + } + setup_overhead(lr->control); + if (lr->flags & LRZIP_FLAG_VERIFY) { + lr->control->flags |= FLAG_CHECK; + lr->control->flags |= FLAG_HASH; + } + if (lr->flags & LRZIP_FLAG_REMOVE_DESTINATION) + lr->control->flags |= FLAG_FORCE_REPLACE; + if (lr->flags & LRZIP_FLAG_REMOVE_SOURCE) + lr->control->flags &= ~FLAG_KEEP_FILES; + if (lr->flags & LRZIP_FLAG_KEEP_BROKEN) + lr->control->flags |= FLAG_KEEP_BROKEN; + if (lr->flags & LRZIP_FLAG_DISABLE_LZO_CHECK) + lr->control->flags &= ~FLAG_THRESHOLD; + if (lr->flags & LRZIP_FLAG_UNLIMITED_RAM) + lr->control->flags |= FLAG_UNLIMITED; + if (lr->flags & LRZIP_FLAG_ENCRYPT) + lr->control->flags |= FLAG_ENCRYPT; + if (lr->control->log_level > 0) { + lr->control->flags |= FLAG_SHOW_PROGRESS; + if (lr->control->log_level > 1) { + lr->control->flags |= FLAG_VERBOSITY; + if (lr->control->log_level > 2) + lr->control->flags |= FLAG_VERBOSITY_MAX; + } + } else lr->control->flags ^= (FLAG_VERBOSE | FLAG_SHOW_PROGRESS); + return true; +} + + bool lrzip_init(void) { /* generate crc table */ @@ -42,12 +113,14 @@ void lrzip_config_env(Lrzip *lr) void lrzip_free(Lrzip *lr) { - if (!lr) return; + size_t x; + + if ((!lr) || (!lr->infilename_buckets)) return; rzip_control_free(lr->control); - for (; lr->infiles && *lr->infiles; lr->infiles++) - free(*lr->infiles); + for (x = 0; x < lr->infilename_idx; x++) + free(lr->infilenames[x]); + free(lr->infilenames); free(lr->infiles); - free(lr->outfile); free(lr); } @@ -56,29 +129,12 @@ Lrzip *lrzip_new(Lrzip_Mode mode) Lrzip *lr; lr = calloc(1, sizeof(Lrzip)); - if (unlikely(!lr)) return NULL; + if (!lr) return NULL; lr->control = calloc(1, sizeof(rzip_control)); - if (unlikely(!lr->control)) goto error; - if (unlikely(!initialize_control(lr->control))) goto error; + if (!lr->control) goto error; + if (!initialize_control(lr->control)) goto error; lr->mode = mode; lr->control->library_mode = 1; -#define MODE_CHECK(X) \ - case LRZIP_MODE_##X: \ - lr->control->flags |= FLAG_##X; \ - break - - switch (mode) { - MODE_CHECK(DECOMPRESS); - MODE_CHECK(NO_COMPRESS); - MODE_CHECK(LZO_COMPRESS); - MODE_CHECK(BZIP2_COMPRESS); - MODE_CHECK(ZLIB_COMPRESS); - MODE_CHECK(ZPAQ_COMPRESS); -#undef MODE_CHECK - default: - goto error; - } - setup_overhead(lr->control); return lr; error: lrzip_free(lr); @@ -91,66 +147,269 @@ Lrzip_Mode lrzip_mode_get(Lrzip *lr) return lr->mode; } -char **lrzip_files_get(Lrzip *lr) +bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode) +{ + if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ)) return false; + lr->mode = mode; + return true; +} + +bool lrzip_compression_level_set(Lrzip *lr, unsigned int level) +{ + if ((!lr) || (!level) || (level > 9)) return false; + lr->control->compression_level = level; + return true; +} + +unsigned int lrzip_compression_level_get(Lrzip *lr) +{ + if (!lr) return 0; + return lr->control->compression_level; +} + +void lrzip_flags_set(Lrzip *lr, unsigned int flags) +{ + if (!lr) return; + lr->flags = flags; +} + +unsigned int lrzip_flags_get(Lrzip *lr) +{ + if (!lr) return 0; + return lr->flags; +} + +void lrzip_nice_set(Lrzip *lr, int nice) +{ + if ((!lr) || (nice < -19) || (nice > 20)) return; + lr->control->nice_val = nice; +} + +int lrzip_nice_get(Lrzip *lr) +{ + if (!lr) return 0; + return lr->control->nice_val; +} + +void lrzip_threads_set(Lrzip *lr, unsigned int threads) +{ + if ((!lr) || (!threads)) return; + lr->control->threads = threads; +} + +unsigned int lrzip_threads_get(Lrzip *lr) +{ + if (!lr) return 0; + return lr->control->threads; +} + +void lrzip_compression_window_max_set(Lrzip *lr, int64_t size) +{ + if (!lr) return; + lr->control->window = size; +} + +int64_t lrzip_compression_window_max_get(Lrzip *lr) +{ + if (!lr) return -1; + return lr->control->window; +} + +unsigned int lrzip_files_count(Lrzip *lr) +{ + if (!lr) return 0; + return lr->infile_idx; +} + +unsigned int lrzip_filenames_count(Lrzip *lr) +{ + if (!lr) return 0; + return lr->infilename_idx; +} + +FILE **lrzip_files_get(Lrzip *lr) { if (!lr) return NULL; return lr->infiles; } -bool lrzip_file_add(Lrzip *lr, const char *file) +char **lrzip_filenames_get(Lrzip *lr) { - if ((!lr) || (!file) || (!file[0])) return false; + if (!lr) return NULL; + return lr->infilenames; +} +bool lrzip_file_add(Lrzip *lr, FILE *file) +{ + if ((!lr) || (!file)) return false; + if (lr->infilenames) return false; if (!lr->infile_buckets) { /* no files added */ - lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(char*)); + lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*)); lr->infile_buckets++; } else if (lr->infile_idx == INFILE_BUCKET_SIZE * lr->infile_buckets + 1) { /* all buckets full, create new bucket */ - char **tmp; + FILE **tmp; - tmp = realloc(lr->infiles, ++lr->infile_buckets * INFILE_BUCKET_SIZE + 1); + tmp = realloc(lr->infiles, (++lr->infile_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); if (!tmp) return false; lr->infiles = tmp; } - lr->infiles[lr->infile_idx++] = strdup(file); - return true; + lr->infiles[lr->infile_idx++] = file; + return true; } -bool lrzip_file_del(Lrzip *lr, const char *file) +bool lrzip_file_del(Lrzip *lr, FILE *file) { size_t x; - if ((!lr) || (!file) || (!file[0])) return false; + if ((!lr) || (!file)) return false; if (!lr->infile_buckets) return true; for (x = 0; x <= lr->infile_idx + 1; x++) { if (!lr->infiles[x]) return true; /* not found */ - if (strcmp(lr->infiles[x], file)) continue; /* not a match */ - free(lr->infiles[x]); + if (lr->infiles[x] != file) continue; /* not a match */ + break; + } + /* update index */ + liblrzip_index_update(x, &lr->infile_idx, (void**)lr->infiles); + return true; +} + +void lrzip_files_clear(Lrzip *lr) +{ + if ((!lr) || (!lr->infile_buckets)) return; + free(lr->infiles); + lr->infiles = NULL; +} + +bool lrzip_filename_add(Lrzip *lr, const char *file) +{ + struct stat st; + + if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-"))) return false; + if (lr->infiles) return false; + if (stat(file, &st)) return false; + if (S_ISDIR(st.st_mode)) return false; + + if (!lr->infilename_buckets) { + /* no files added */ + lr->infilenames = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*)); + lr->infilename_buckets++; + } else if (lr->infilename_idx == INFILE_BUCKET_SIZE * lr->infilename_buckets + 1) { + /* all buckets full, create new bucket */ + char **tmp; + + tmp = realloc(lr->infilenames, (++lr->infilename_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); + if (!tmp) return false; + lr->infilenames = tmp; + } + + lr->infilenames[lr->infilename_idx++] = strdup(file); + return true; +} + +bool lrzip_filename_del(Lrzip *lr, const char *file) +{ + size_t x; + + if ((!lr) || (!file) || (!file[0])) return false; + if (!lr->infilename_buckets) return true; + + for (x = 0; x <= lr->infilename_idx + 1; x++) { + if (!lr->infilenames[x]) return true; /* not found */ + if (strcmp(lr->infilenames[x], file)) continue; /* not a match */ + free(lr->infilenames[x]); break; } /* update index */ - for (; x < lr->infile_idx; x++) - lr->infiles[x] = lr->infiles[x] + 1; - lr->infile_idx--; + liblrzip_index_update(x, &lr->infilename_idx, (void**)lr->infilenames); return true; } -void lrzip_outfile_set(Lrzip *lr, const char *file) +void lrzip_filenames_clear(Lrzip *lr) +{ + size_t x; + if ((!lr) || (!lr->infilename_buckets)) return; + for (x = 0; x < lr->infilename_idx; x++) + free(lr->infilenames[x]); + free(lr->infilenames); + lr->infilenames = NULL; +} + +void lrzip_suffix_set(Lrzip *lr, const char *suffix) +{ + if ((!lr) || (!suffix) || (!suffix[0])) return; + free(lr->control->suffix); + lr->control->suffix = strdup(suffix); +} + +const char *lrzip_suffix_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->control->suffix; +} + +void lrzip_outdir_set(Lrzip *lr, const char *dir) +{ + const char *slash; + char *buf; + size_t len; + if ((!lr) || (!dir) || (!dir[0])) return; + free(lr->control->outdir); + slash = strrchr(dir, '/'); + if (slash && (slash[1] == 0)) { + lr->control->outdir = strdup(dir); + return; + } + len = strlen(dir); + buf = malloc(len + 2); + if (!buf) return; + memcpy(buf, dir, len); + buf[len] = '/'; + buf[len + 1] = 0; + lr->control->outdir = buf; +} + +const char *lrzip_outdir_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->control->outdir; +} + +void lrzip_outfile_set(Lrzip *lr, FILE *file) +{ + if ((!lr) || (file && (file == stderr))) return; + if (lr->control->outname) return; + lr->control->outFILE = file; +} + +FILE *lrzip_outfile_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->control->outFILE; +} + +void lrzip_outfilename_set(Lrzip *lr, const char *file) +{ + if ((!lr) || (file && (!file[0]))) return; + if (lr->control->outFILE) return; + if (lr->control->outname && file && (!strcmp(lr->control->outname, file))) return; + free(lr->control->outname); + lr->control->outname = file ? strdup(file) : NULL; +} + +const char *lrzip_outfilename_get(Lrzip *lr) { - if ((!lr) || (!file) || (!file[0])) return; - if (lr->outfile && (!strcmp(lr->outfile, file))) return; - free(lr->outfile); - lr->outfile = strdup(file); - lr->control->outname = lr->outfile; + if (!lr) return NULL; + return lr->control->outname; } -const char *lrzip_outfile_get(Lrzip *lr) +const unsigned char *lrzip_md5digest_get(Lrzip *lr) { if (!lr) return NULL; - return lr->outfile; + return lr->control->md5_resblock; } bool lrzip_run(Lrzip *lr) @@ -158,83 +417,103 @@ bool lrzip_run(Lrzip *lr) struct timeval start_time, end_time; rzip_control *control; double seconds,total_time; // for timers - size_t i; int hours,minutes; - if (!lr) return false; + if (!liblrzip_setup_flags(lr)) return false; control = lr->control; - /* One extra iteration for the case of no parameters means we will default to stdin/out */ - for (i = 0; i < lr->infile_idx; i++) { - lr->control->infile = lr->infiles[i]; - if (lr->control->infile) { - if ((strcmp(lr->control->infile, "-") == 0)) - lr->control->flags |= FLAG_STDIN; - else { - struct stat infile_stat; - - stat(lr->control->infile, &infile_stat); - if (unlikely(S_ISDIR(infile_stat.st_mode))) { - print_err("lrzip only works directly on FILES.\n" - "Use lrztar or pipe through tar for compressing directories.\n"); - return false; - } - } - } - if (lr->control->outname && (strcmp(lr->control->outname, "-") == 0)) { - lr->control->flags |= FLAG_STDOUT; + if ((!lr->infile_idx) && (!lr->infilename_idx)) return false; + if (lr->control->outFILE) { + if (lr->control->outFILE == lr->control->msgout) lr->control->msgout = stderr; - register_outputfile(lr->control, lr->control->msgout); - } + lr->control->flags |= FLAG_STDOUT; + register_outputfile(lr->control, lr->control->msgout); + } - /* If no output filename is specified, and we're using stdin, - * use stdout */ - if (!lr->control->outname && STDIN) { - lr->control->flags |= FLAG_STDOUT; - lr->control->msgout = stderr; - register_outputfile(lr->control, lr->control->msgout); - } + if (lr->infilenames) + lr->control->infile = lr->infilenames[0]; + else { + lr->control->inFILE = lr->infiles[0]; + control->flags |= FLAG_STDIN; + } - if (!STDOUT) { - lr->control->msgout = stdout; - register_outputfile(lr->control, lr->control->msgout); - } + if ((!STDOUT) && (!lr->control->msgout)) lr->control->msgout = stdout; + register_outputfile(lr->control, lr->control->msgout); - if (CHECK_FILE) { - if (!DECOMPRESS) { - print_err("Can only check file written on decompression.\n"); - lr->control->flags &= ~FLAG_CHECK; - } else if (STDOUT) { - print_err("Can't check file written when writing to stdout. Checking disabled.\n"); - lr->control->flags &= ~FLAG_CHECK; - } - } + setup_ram(lr->control); - setup_ram(lr->control); + gettimeofday(&start_time, NULL); - gettimeofday(&start_time, NULL); + if (ENCRYPT && (!lr->control->pass_cb)) { + print_err("No password callback set!\n"); + return false; + } - if (unlikely(ENCRYPT && (!lr->control->pass_cb))) { - print_err("No password callback set!\n"); - return false; - } + if (DECOMPRESS || TEST_ONLY) { + if (!decompress_file(lr->control)) return false; + } else if (INFO) { + if (!get_fileinfo(lr->control)) return false; + } else if (!compress_file(lr->control)) return false; + + /* compute total time */ + gettimeofday(&end_time, NULL); + total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) - + (start_time.tv_sec + (double)start_time.tv_usec / 1000000); + hours = (int)total_time / 3600; + minutes = (int)(total_time / 60) % 60; + seconds = total_time - hours * 3600 - minutes * 60; + if (!INFO) + print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds); - if (DECOMPRESS || TEST_ONLY) - decompress_file(lr->control); - else if (INFO) - get_fileinfo(lr->control); - else - compress_file(lr->control); - - /* compute total time */ - gettimeofday(&end_time, NULL); - total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) - - (start_time.tv_sec + (double)start_time.tv_usec / 1000000); - hours = (int)total_time / 3600; - minutes = (int)(total_time / 60) % 60; - seconds = total_time - hours * 3600 - minutes * 60; - if (!INFO) - print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds); - } return true; } + +void lrzip_log_level_set(Lrzip *lr, int level) +{ + if (!lr) return; + lr->control->log_level = level; +} + +int lrzip_log_level_get(Lrzip *lr) +{ + if (!lr) return 0; + return lr->control->log_level; +} + +void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data) +{ + if (!lr) return; + lr->control->log_cb = (void*)cb; + lr->control->log_data = log_data; +} + +void lrzip_log_stdout_set(Lrzip *lr, FILE *out) +{ + if (!lr) return; + lr->control->msgout = out; +} + +FILE *lrzip_log_stdout_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->control->msgout; +} + +void lrzip_log_stderr_set(Lrzip *lr, FILE *err) +{ + if (!lr) return; + lr->control->msgerr = err; +} + +FILE *lrzip_log_stderr_get(Lrzip *lr) +{ + if (!lr) return NULL; + return lr->control->msgerr; +} + +void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data) +{ + if (!lr) return; + lr->control->pass_cb = (void*)cb; + lr->control->pass_data = data; +} diff --git a/liblrzip.h b/liblrzip.h index 5ab2927a..7e4642a5 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -21,19 +21,39 @@ #define LIBLRZIP_H #include +#include +#ifdef _WIN32 +# include +#else +# include +#endif typedef struct Lrzip Lrzip; typedef enum { - LRZIP_MODE_NONE, + LRZIP_MODE_NONE = 0, + LRZIP_MODE_INFO, + LRZIP_MODE_TEST, LRZIP_MODE_DECOMPRESS, - LRZIP_MODE_NO_COMPRESS, - LRZIP_MODE_LZO_COMPRESS, - LRZIP_MODE_BZIP2_COMPRESS, - LRZIP_MODE_ZLIB_COMPRESS, - LRZIP_MODE_ZPAQ_COMPRESS + LRZIP_MODE_COMPRESS_NONE, + LRZIP_MODE_COMPRESS_LZO, + LRZIP_MODE_COMPRESS_ZLIB, + LRZIP_MODE_COMPRESS_BZIP2, + LRZIP_MODE_COMPRESS_LZMA, + LRZIP_MODE_COMPRESS_ZPAQ } Lrzip_Mode; +typedef enum { + LRZIP_FLAG_REMOVE_SOURCE = (1 << 0), + LRZIP_FLAG_REMOVE_DESTINATION = (1 << 1), + LRZIP_FLAG_KEEP_BROKEN = (1 << 2), + LRZIP_FLAG_VERIFY = (1 << 3), + LRZIP_FLAG_DISABLE_LZO_CHECK = (1 << 4), + LRZIP_FLAG_UNLIMITED_RAM = (1 << 5), + LRZIP_FLAG_ENCRYPT = (1 << 6) +} Lrzip_Flag; + +typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *format, va_list args); typedef void (*Lrzip_Password_Cb)(void *, char **, size_t); bool lrzip_init(void); @@ -41,12 +61,44 @@ void lrzip_config_env(Lrzip *lr); void lrzip_free(Lrzip *lr); Lrzip *lrzip_new(Lrzip_Mode mode); Lrzip_Mode lrzip_mode_get(Lrzip *lr); -char **lrzip_files_get(Lrzip *lr); -bool lrzip_file_add(Lrzip *lr, const char *file); -bool lrzip_file_del(Lrzip *lr, const char *file); -void lrzip_outfile_set(Lrzip *lr, const char *file); -const char *lrzip_outfile_get(Lrzip *lr); +bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode); +bool lrzip_compression_level_set(Lrzip *lr, unsigned int level); +unsigned int lrzip_compression_level_get(Lrzip *lr); +void lrzip_flags_set(Lrzip *lr, unsigned int flags); +unsigned int lrzip_flags_get(Lrzip *lr); +void lrzip_nice_set(Lrzip *lr, int nice); +int lrzip_nice_get(Lrzip *lr); +void lrzip_threads_set(Lrzip *lr, unsigned int threads); +unsigned int lrzip_threads_get(Lrzip *lr); +void lrzip_compression_window_max_set(Lrzip *lr, int64_t size); +int64_t lrzip_compression_window_max_get(Lrzip *lr); +unsigned int lrzip_files_count(Lrzip *lr); +unsigned int lrzip_filenames_count(Lrzip *lr); +FILE **lrzip_files_get(Lrzip *lr); +char **lrzip_filenames_get(Lrzip *lr); +bool lrzip_file_add(Lrzip *lr, FILE *file); +bool lrzip_file_del(Lrzip *lr, FILE *file); +void lrzip_files_clear(Lrzip *lr); +bool lrzip_filename_add(Lrzip *lr, const char *file); +bool lrzip_filename_del(Lrzip *lr, const char *file); +void lrzip_filenames_clear(Lrzip *lr); +void lrzip_suffix_set(Lrzip *lr, const char *suffix); +const char *lrzip_suffix_get(Lrzip *lr); +void lrzip_outdir_set(Lrzip *lr, const char *dir); +const char *lrzip_outdir_get(Lrzip *lr); +void lrzip_outfile_set(Lrzip *lr, FILE *file); +FILE *lrzip_outfile_get(Lrzip *lr); +void lrzip_outfilename_set(Lrzip *lr, const char *file); +const char *lrzip_outfilename_get(Lrzip *lr); +const unsigned char *lrzip_md5digest_get(Lrzip *lr); bool lrzip_run(Lrzip *lr); - +void lrzip_log_level_set(Lrzip *lr, int level); +int lrzip_log_level_get(Lrzip *lr); +void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data); +void lrzip_log_stdout_set(Lrzip *lr, FILE *out); +FILE *lrzip_log_stdout_get(Lrzip *lr); +void lrzip_log_stderr_set(Lrzip *lr, FILE *err); +FILE *lrzip_log_stderr_get(Lrzip *lr); +void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data); #endif diff --git a/liblrzip_private.h b/liblrzip_private.h index 4ba061d5..d75f1504 100644 --- a/liblrzip_private.h +++ b/liblrzip_private.h @@ -9,12 +9,14 @@ struct Lrzip { Lrzip_Mode mode; + unsigned int flags; rzip_control *control; - char *outfile; - /* bucket allocation is used here to avoid frequent calls to realloc */ - char **infiles; + char **infilenames; + size_t infilename_idx; + size_t infilename_buckets; + FILE **infiles; size_t infile_idx; size_t infile_buckets; }; From 120153c975e0779c3e1606317bb08bc3a6fdd36a Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 03:43:33 -0400 Subject: [PATCH 17/30] new liblrzip demo which mimics lrzip, updates to simple decompress demo --- Makefile.am | 5 +- compress_demo.c | 42 ------ decompress_demo.c | 9 +- liblrzip_demo.c | 327 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 335 insertions(+), 48 deletions(-) delete mode 100644 compress_demo.c create mode 100644 liblrzip_demo.c diff --git a/Makefile.am b/Makefile.am index a71422c9..8df5f3ae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -70,10 +70,13 @@ if STATIC lrzip_LDFLAGS = -all-static endif -noinst_PROGRAMS = decompress_demo +noinst_PROGRAMS = decompress_demo liblrzip_demo decompress_demo_SOURCES = decompress_demo.c decompress_demo_LDADD = liblrzip.la +liblrzip_demo_SOURCES = liblrzip_demo.c +liblrzip_demo_LDADD = liblrzip.la + dist_doc_DATA = \ AUTHORS \ BUGS \ diff --git a/compress_demo.c b/compress_demo.c deleted file mode 100644 index 5f3bf4c6..00000000 --- a/compress_demo.c +++ /dev/null @@ -1,42 +0,0 @@ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#undef NDEBUG -#include -#include -#include -#include -#include - -static const char *suffix_me(const char *file) -{ - const char *p; - static char buf[4096]; - - p = strrchr(file, '.'); - if (p && (strlen(p + 1) < 4)) - strncat(buf, file, p - file); - else - strcat(buf, file); - return &buf[0]; -} - -int main(int argc, char *argv[]) -{ - Lrzip *lr; - if ((argc != 2) && (argc != 3)) { - fprintf(stderr, "Usage: %s file [file [file [... [outfile]]]]\n", argv[0]); - exit(1); - } - lr = lrzip_new(LRZIP_MODE_DECOMPRESS); - assert(lr); - lrzip_config_env(lr); - assert(lrzip_file_add(lr, argv[1])); - if (argc == 2) { - lrzip_outfile_set(lr, suffix_me(argv[1])); - } - else - lrzip_outfile_set(lr, argv[2]); - assert(lrzip_run(lr)); - return 0; -} diff --git a/decompress_demo.c b/decompress_demo.c index 3e036260..a980dd57 100644 --- a/decompress_demo.c +++ b/decompress_demo.c @@ -31,12 +31,11 @@ int main(int argc, char *argv[]) lr = lrzip_new(LRZIP_MODE_DECOMPRESS); assert(lr); lrzip_config_env(lr); - assert(lrzip_file_add(lr, argv[1])); - if (argc == 2) { - lrzip_outfile_set(lr, suffix_me(argv[1])); - } + assert(lrzip_filename_add(lr, argv[1])); + if (argc == 2) + lrzip_outfilename_set(lr, suffix_me(argv[1])); else - lrzip_outfile_set(lr, argv[2]); + lrzip_outfilename_set(lr, argv[2]); assert(lrzip_run(lr)); return 0; } diff --git a/liblrzip_demo.c b/liblrzip_demo.c new file mode 100644 index 00000000..aec4a2bb --- /dev/null +++ b/liblrzip_demo.c @@ -0,0 +1,327 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#undef NDEBUG +#include +#include +#ifdef HAVE_STRING_H +# include +#endif +#include +#ifdef HAVE_ERRNO_H +# include +#else +#ifdef HAVE_UNISTD_H +# include +#endif +extern int errno; +#endif +#include +#include + +#define failure(format, args...) do { \ + fprintf(stderr, format, ##args); \ + exit(1); \ +} while (0) + +static void usage(void) +{ + printf("lrzip version %s\n", PACKAGE_VERSION); + printf("Copyright (C) Con Kolivas 2006-2011\n"); + printf("Based on rzip "); + printf("Copyright (C) Andrew Tridgell 1998-2003\n\n"); + printf("Usage: lrzip [options] \n"); + printf("General options:\n"); + printf(" -c check integrity of file written on decompression\n"); + printf(" -d decompress\n"); + printf(" -e password protected sha512/aes128 encryption on compression\n"); + printf(" -h|-? show help\n"); + printf(" -H display md5 hash integrity information\n"); + printf(" -i show compressed file information\n"); + printf(" -q don't show compression progress\n"); + printf(" -t test compressed file integrity\n"); + printf(" -v[v] Increase verbosity\n"); + printf(" -V show version\n"); + printf("Options affecting output:\n"); + printf(" -D delete existing files\n"); + printf(" -f force overwrite of any existing files\n"); + printf(" -k keep broken or damaged output files\n"); + printf(" -o filename specify the output file name and/or path\n"); + printf(" -O directory specify the output directory when -o is not used\n"); + printf(" -S suffix specify compressed suffix (default '.lrz')\n"); + printf("Options affecting compression:\n"); + printf(" -b bzip2 compression\n"); + printf(" -g gzip compression using zlib\n"); + printf(" -l lzo compression (ultra fast)\n"); + printf(" -n no backend compression - prepare for other compressor\n"); + printf(" -z zpaq compression (best, extreme compression, extremely slow)\n"); + printf("Low level options:\n"); + printf(" -L level set lzma/bzip2/gzip compression level (1-9, default 7)\n"); + printf(" -N value Set nice value to value (default 19)\n"); + printf(" -p value Set processor count to override number of threads\n"); + printf(" -T Disable LZO compressibility testing\n"); + printf(" -U Use unlimited window size beyond ramsize (potentially much slower)\n"); + printf(" -w size maximum compression window in hundreds of MB\n"); + printf(" default chosen by heuristic dependent on ram and chosen compression\n"); + printf("\nLRZIP=NOCONFIG environment variable setting can be used to bypass lrzip.conf.\n"); + printf("TMP environment variable will be used for storage of temporary files when needed.\n"); + printf("TMPDIR may also be stored in lrzip.conf file.\n"); + printf("\nIf no filenames or \"-\" is specified, stdin/out will be used.\n"); +} + +static int get_pass(char *s, size_t slen) +{ + int len; + + memset(s, 0, slen); + if (!fgets(s, slen, stdin)) { + fprintf(stderr, "Failed to retrieve passphrase\n"); + return -1; + } + len = strlen(s); + if (len > 0 && ('\r' == s[len - 1] || '\n' == s[len - 1])) + s[len - 1] = '\0'; + if (len > 1 && ('\r' == s[len - 2] || '\n' == s[len - 2])) + s[len - 2] = '\0'; + len = strlen(s); + if (!len) { + fprintf(stderr, "Empty passphrase\n"); + return -1; + } + return len; +} + +static void pass_cb(void *data __UNUSED__, char **pass_string, size_t pass_len) +{ + int len; + struct termios termios_p; + /* Disable stdin echo to screen */ + tcgetattr(fileno(stdin), &termios_p); + termios_p.c_lflag &= ~ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); + + printf("Enter passphrase: "); + len = get_pass(*pass_string, pass_len); + printf("\n"); + + if (len < 1) exit(1); + + termios_p.c_lflag |= ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); +} + +static void mode_check(Lrzip *lr, Lrzip_Mode mode) { + Lrzip_Mode current = lrzip_mode_get(lr); + if (current && (current != mode)) + failure("Can only use one of -l, -b, -g, -z or -n\n"); + lrzip_mode_set(lr, mode); +} + +int main(int argc, char *argv[]) +{ + Lrzip *lr; + extern int optind; + extern char *optarg; + int64_t x; + int c; + bool get_hash = false; + + lrzip_init(); + lr = lrzip_new(LRZIP_MODE_NONE); + assert(lr); + lrzip_config_env(lr); + lrzip_log_level_set(lr, 2); + while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) { + switch (c) { + case 'b': + mode_check(lr, LRZIP_MODE_COMPRESS_BZIP2); + break; + case 'c': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_VERIFY); + break; + case 'd': + mode_check(lr, LRZIP_MODE_DECOMPRESS); + break; + case 'D': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_SOURCE); + break; + case 'e': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_ENCRYPT); + break; + case 'f': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_DESTINATION); + break; + case 'g': + mode_check(lr, LRZIP_MODE_COMPRESS_ZLIB); + break; + case 'h': + case '?': + usage(); + return -1; + case 'H': + get_hash = true; + break; + case 'i': + mode_check(lr, LRZIP_MODE_INFO); + break; + case 'k': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_KEEP_BROKEN); + break; + case 'l': + mode_check(lr, LRZIP_MODE_COMPRESS_LZO); + break; + case 'L': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || ((x < 1) || (x > 9))) + failure("Invalid compression level (must be 1-9)\n"); + lrzip_compression_level_set(lr, (unsigned int)x); + break; + case 'n': + mode_check(lr, LRZIP_MODE_COMPRESS_NONE); + break; + case 'N': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || (x < -20 || x > 19)) + failure("Invalid nice value (must be -20..19)\n"); + lrzip_nice_set(lr, x); + break; + case 'o': + if (lrzip_outdir_get(lr)) + failure("Cannot have -o and -O together\n"); + if (!strcmp(optarg, "-")) + lrzip_outfile_set(lr, stdout); + else + lrzip_outfilename_set(lr, optarg); + break; + case 'O': + if (lrzip_outfilename_get(lr)) /* can't mix -o and -O */ + failure("Cannot have options -o and -O together\n"); + if (lrzip_outfile_get(lr)) + failure("Cannot specify an output directory when outputting to stdout\n"); + lrzip_outdir_set(lr, optarg); + break; + case 'p': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || (x < 1)) + failure("Must have at least one thread\n"); + lrzip_threads_set(lr, (unsigned int)x); + break; + case 'q': + lrzip_log_level_set(lr, lrzip_log_level_get(lr) - 1); + break; + case 'S': + if (lrzip_outfilename_get(lr)) + failure("Specified output filename already, can't specify an extension.\n"); + if (lrzip_outfile_get(lr)) + failure("Cannot specify a filename suffix when outputting to stdout\n"); + lrzip_suffix_set(lr, optarg); + break; + case 't': + if (lrzip_outfilename_get(lr)) + failure("Cannot specify an output file name when just testing.\n"); + if (lrzip_flags_get(lr) & LRZIP_FLAG_REMOVE_SOURCE) + failure("Doubt that you want to delete a file when just testing.\n"); + mode_check(lr, LRZIP_MODE_TEST); + break; + case 'T': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_DISABLE_LZO_CHECK); + break; + case 'U': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_UNLIMITED_RAM); + break; + case 'v': + lrzip_log_level_set(lr, lrzip_log_level_get(lr) + 1); + break; + case 'V': + printf("lrzip version %s\n", PACKAGE_VERSION); + exit(0); + break; + case 'w': + errno = 0; + x = strtoll(optarg, NULL, 10); + if (errno || (x < 1)) + failure("Invalid compression window '%s'!\n", optarg); + lrzip_compression_window_max_set(lr, x); + break; + case 'z': + mode_check(lr, LRZIP_MODE_COMPRESS_ZPAQ); + break; + } + } + /* LZMA is the default */ + if (!lrzip_mode_get(lr)) lrzip_mode_set(lr, LRZIP_MODE_COMPRESS_LZMA); + argc -= optind, argv += optind; + + if (lrzip_outfilename_get(lr) && (argc > 1)) + failure("Cannot specify output filename with more than 1 file\n"); + + if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_compression_window_max_get(lr)) { + fprintf(stderr, "If -U used, cannot specify a window size with -w.\n"); + lrzip_compression_window_max_set(lr, 0); + } + + if (argc < 1) lrzip_file_add(lr, stdin); + + if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_files_count(lr)) { + fprintf(stderr, "Cannot have -U and stdin, unlimited mode disabled.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_UNLIMITED_RAM); + } + + /* If no output filename is specified, and we're using stdin, + * use stdout */ + if (lrzip_files_count(lr) && (!lrzip_outfilename_get(lr))) + lrzip_outfile_set(lr, stdout); + + if (lrzip_flags_get(lr) & LRZIP_FLAG_VERIFY) { + if (lrzip_mode_get(lr) != LRZIP_MODE_DECOMPRESS) { + fprintf(stderr, "Can only check file written on decompression.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY); + } else if (lrzip_outfile_get(lr)) { + fprintf(stderr, "Can't check file written when writing to stdout. Checking disabled.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY); + } + } + + for (x = 0; x < argc; x++) { + if (argv[x][0] != '-') { + assert(lrzip_filename_add(lr, argv[x])); + continue; + } + if (argv[x][1] == 0) { + assert(lrzip_file_add(lr, stdin)); + continue; + } + } + if (argc == 1) { + if (!lrzip_files_count(lr)) lrzip_file_add(lr, stdin); + if (lrzip_filenames_count(lr)) { + if (!lrzip_outfilename_get(lr)) { + char buf[4096]; + const char *infile; + size_t len; + + infile = lrzip_filenames_get(lr)[0]; + len = strlen(infile); + if (!strcmp(infile + len - 4, ".lrz")) + strncat(buf, infile, len - 4); + else + snprintf(buf, sizeof(buf), "%s.out", infile); + lrzip_outfilename_set(lr, buf); + } + } else if (!lrzip_outfile_get(lr)) lrzip_outfile_set(lr, stdout); + } + lrzip_log_stdout_set(lr, stdout); + lrzip_log_stderr_set(lr, stderr); + lrzip_pass_cb_set(lr, pass_cb, NULL); + if (!lrzip_run(lr)) exit(1); + if (get_hash) { + const unsigned char *digest = lrzip_md5digest_get(lr); + for (x = 0; x < 16; x++) + fprintf(stdout, "%02x", digest[x] & 0xFF); + } + lrzip_free(lr); + return 0; +} From 6e702f49c42c50812d842b61ee781014eb85d59c Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 03:51:41 -0400 Subject: [PATCH 18/30] add lrzip log level enum --- liblrzip.h | 8 ++++++++ liblrzip_demo.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/liblrzip.h b/liblrzip.h index 7e4642a5..562c643a 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -30,6 +30,14 @@ typedef struct Lrzip Lrzip; +typedef enum { + LRZIP_LOG_LEVEL_ERROR = 0, + LRZIP_LOG_LEVEL_INFO, + LRZIP_LOG_LEVEL_PROGRESS, + LRZIP_LOG_LEVEL_VERBOSE, + LRZIP_LOG_LEVEL_DEBUG +} Lrzip_Log_Level; + typedef enum { LRZIP_MODE_NONE = 0, LRZIP_MODE_INFO, diff --git a/liblrzip_demo.c b/liblrzip_demo.c index aec4a2bb..4565e74b 100644 --- a/liblrzip_demo.c +++ b/liblrzip_demo.c @@ -130,7 +130,7 @@ int main(int argc, char *argv[]) lr = lrzip_new(LRZIP_MODE_NONE); assert(lr); lrzip_config_env(lr); - lrzip_log_level_set(lr, 2); + lrzip_log_level_set(lr, LRZIP_LOG_LEVEL_PROGRESS); while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) { switch (c) { case 'b': From 223a1107ea8f288c47832f97030a7229273a7ce7 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 04:05:01 -0400 Subject: [PATCH 19/30] add info callback which provides percentage completion of operation --- liblrzip.c | 8 ++++++++ liblrzip.h | 2 ++ lrzip_private.h | 2 ++ rzip.c | 3 +++ 4 files changed, 15 insertions(+) diff --git a/liblrzip.c b/liblrzip.c index b9cfee1f..2e2d00cd 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -517,3 +517,11 @@ void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data) lr->control->pass_cb = (void*)cb; lr->control->pass_data = data; } + +void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data) +{ + if (!lr) return; + lr->control->info_cb = (void*)cb; + lr->control->info_data = data; +} + diff --git a/liblrzip.h b/liblrzip.h index 562c643a..39aeb152 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -61,6 +61,7 @@ typedef enum { LRZIP_FLAG_ENCRYPT = (1 << 6) } Lrzip_Flag; +typedef void (*Lrzip_Info_Cb)(void *data, int pct, int chunk_pct); typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *format, va_list args); typedef void (*Lrzip_Password_Cb)(void *, char **, size_t); @@ -108,5 +109,6 @@ FILE *lrzip_log_stdout_get(Lrzip *lr); void lrzip_log_stderr_set(Lrzip *lr, FILE *err); FILE *lrzip_log_stderr_get(Lrzip *lr); void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data); +void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data); #endif diff --git a/lrzip_private.h b/lrzip_private.h index 0778c89d..be63ac56 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -345,6 +345,8 @@ struct rzip_control { FILE *outputfile; char library_mode : 1; int log_level; + void (*info_cb)(void *data, int pct, int chunk_pct); + void *info_data; void (*log_cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *func, const char *format, va_list); void *log_data; }; diff --git a/rzip.c b/rzip.c index ad7419c4..affabd3a 100644 --- a/rzip.c +++ b/rzip.c @@ -635,6 +635,9 @@ static bool hash_search(rzip_control *control, struct rzip_state *st, double pct if (!STDIN || st->stdin_eof) print_progress("Total: %2d%% ", pct); print_progress("Chunk: %2d%%\r", chunk_pct); + if (control->info_cb) + control->info_cb(control->info_data, + (!STDIN || st->stdin_eof) ? pct : -1, chunk_pct); lastpct = pct; last_chunkpct = chunk_pct; } From 3316dd5505abaf55fbe6201de8edc248cf62b023 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Sat, 13 Aug 2011 04:06:30 -0400 Subject: [PATCH 20/30] remove trailing whitespace --- aes.c | 10 +++++----- liblrzip.c | 8 ++++---- lrzip.c | 2 +- rzip.c | 2 +- stream.c | 8 ++++---- util.c | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/aes.c b/aes.c index 7dde41b4..f10eb795 100644 --- a/aes.c +++ b/aes.c @@ -61,10 +61,10 @@ * Forward S-box & tables */ static unsigned char FSb[256]; -static unsigned long FT0[256]; -static unsigned long FT1[256]; -static unsigned long FT2[256]; -static unsigned long FT3[256]; +static unsigned long FT0[256]; +static unsigned long FT1[256]; +static unsigned long FT2[256]; +static unsigned long FT3[256]; /* * Reverse S-box & tables @@ -501,7 +501,7 @@ int aes_crypt_cbc( aes_context *ctx, { if( padlock_xcryptcbc( ctx, mode, length, iv, input, output ) == 0 ) return( 0 ); - + // If padlock data misaligned, we just fall back to // unaccelerated mode // diff --git a/liblrzip.c b/liblrzip.c index 2e2d00cd..4bfd8bba 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -35,7 +35,7 @@ static bool liblrzip_setup_flags(Lrzip *lr) lr->control->flags |= FLAG_##X##_COMPRESS; \ break - + switch (lr->mode) { case LRZIP_MODE_DECOMPRESS: lr->control->flags |= FLAG_DECOMPRESS; @@ -257,13 +257,13 @@ bool lrzip_file_add(Lrzip *lr, FILE *file) } lr->infiles[lr->infile_idx++] = file; - return true; + return true; } bool lrzip_file_del(Lrzip *lr, FILE *file) { size_t x; - + if ((!lr) || (!file)) return false; if (!lr->infile_buckets) return true; @@ -313,7 +313,7 @@ bool lrzip_filename_add(Lrzip *lr, const char *file) bool lrzip_filename_del(Lrzip *lr, const char *file) { size_t x; - + if ((!lr) || (!file) || (!file[0])) return false; if (!lr->infilename_buckets) return true; diff --git a/lrzip.c b/lrzip.c index 0c3463eb..e1569e67 100644 --- a/lrzip.c +++ b/lrzip.c @@ -1148,7 +1148,7 @@ bool compress_file(rzip_control *control) control->fd_out = fd_out; if (!STDIN) if (unlikely(!preserve_perms(control, fd_in, fd_out))) goto error; - } else + } else if (unlikely(!open_tmpoutbuf(control))) goto error; /* Write zeroes to header at beginning of file */ diff --git a/rzip.c b/rzip.c index affabd3a..e4a73484 100644 --- a/rzip.c +++ b/rzip.c @@ -725,7 +725,7 @@ static bool mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) buf = (uchar *)mremap(buf, st->chunk_size, total, 0); st->mmap_size = st->chunk_size = total; } else { - /* Empty file */ + /* Empty file */ buf = (uchar *)mremap(buf, st->chunk_size, control->page_size, 0); st->mmap_size = control->page_size; st->chunk_size = 0; diff --git a/stream.c b/stream.c index 2153c279..9fe5cc5c 100644 --- a/stream.c +++ b/stream.c @@ -981,7 +981,7 @@ bool close_streamout_threads(rzip_control *control) free(threads); return false; } - + if (++close_thread == control->threads) close_thread = 0; } @@ -1283,7 +1283,7 @@ static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, if (unlikely(write_buf(control, head, SALT_LEN))) failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - + failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error); if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error; @@ -1314,7 +1314,7 @@ static void *compthread(void *data) int write_len; /* Make sure this thread doesn't already exist */ - + free(data); cti = &cthread[i]; ctis = cti->sinfo; @@ -1501,7 +1501,7 @@ static bool clear_buffer(rzip_control *control, struct stream_info *sinfo, int s unlock_mutex(control, &cthread[i].mutex); return false; } - + if (newbuf) { /* The stream buffer has been given to the thread, allocate a diff --git a/util.c b/util.c index 58b1fa6a..ea3dfb19 100644 --- a/util.c +++ b/util.c @@ -365,7 +365,7 @@ bool lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa if (unlikely(aes_setkey_enc(&aes_ctx, key, 128))) failure_goto(("Failed to aes_setkey_enc in lrz_crypt\n"), error); aes_crypt_cbc(&aes_ctx, AES_ENCRYPT, N, iv, buf, buf); - + if (M) { memset(tmp0, 0, CBC_LEN); memcpy(tmp0, buf + N, M); From e111d890eeb83b26eb784b5a4a2fbaebc7685304 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Tue, 16 Aug 2011 02:13:41 -0400 Subject: [PATCH 21/30] whoops typo --- liblrzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblrzip.c b/liblrzip.c index 4bfd8bba..576f89b7 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -22,7 +22,7 @@ static void liblrzip_index_update(size_t x, size_t *idx, void **queue) { for (; x < *idx; x++) - queue[x] = queue[x] + 1; + queue[x] = queue[x + 1]; (*idx)--; } From 8d299570e99e41074dba214f180d29058605eb91 Mon Sep 17 00:00:00 2001 From: discomfitor Date: Tue, 16 Aug 2011 19:04:25 -0400 Subject: [PATCH 22/30] add queue popping functions --- liblrzip.c | 18 ++++++++++++++++++ liblrzip.h | 2 ++ 2 files changed, 20 insertions(+) diff --git a/liblrzip.c b/liblrzip.c index 576f89b7..2895da9b 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -277,6 +277,15 @@ bool lrzip_file_del(Lrzip *lr, FILE *file) return true; } +FILE *lrzip_file_pop(Lrzip *lr) +{ + FILE *ret; + if ((!lr) || (!lr->infile_buckets)) return NULL; + ret = lr->infiles[0]; + lrzip_file_del(lr, ret); + return ret; +} + void lrzip_files_clear(Lrzip *lr) { if ((!lr) || (!lr->infile_buckets)) return; @@ -328,6 +337,15 @@ bool lrzip_filename_del(Lrzip *lr, const char *file) return true; } +const char *lrzip_filename_pop(Lrzip *lr) +{ + static char buf[4096]; + if ((!lr) || (!lr->infilename_buckets)) return NULL; + strcat(buf, lr->infilenames[0]); + lrzip_filename_del(lr, buf); + return &buf[0]; +} + void lrzip_filenames_clear(Lrzip *lr) { size_t x; diff --git a/liblrzip.h b/liblrzip.h index 39aeb152..4c293193 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -87,9 +87,11 @@ FILE **lrzip_files_get(Lrzip *lr); char **lrzip_filenames_get(Lrzip *lr); bool lrzip_file_add(Lrzip *lr, FILE *file); bool lrzip_file_del(Lrzip *lr, FILE *file); +FILE *lrzip_file_pop(Lrzip *lr); void lrzip_files_clear(Lrzip *lr); bool lrzip_filename_add(Lrzip *lr, const char *file); bool lrzip_filename_del(Lrzip *lr, const char *file); +const char *lrzip_filename_pop(Lrzip *lr); void lrzip_filenames_clear(Lrzip *lr); void lrzip_suffix_set(Lrzip *lr, const char *suffix); const char *lrzip_suffix_get(Lrzip *lr); From 651c4f6252f8aa5fa51b40a4af0856cd99f829b9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 17 Aug 2011 21:03:02 +1000 Subject: [PATCH 23/30] Add simple high level liblrzip interfaces. --- liblrzip.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ liblrzip.h | 38 ++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/liblrzip.c b/liblrzip.c index 2895da9b..7871ad31 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -19,6 +19,23 @@ #include "lrzip.h" #include "rzip.h" +#ifdef __APPLE__ +# define fmemopen(s, len, modes) fake_fmemopen((s), (len), (modes)) +static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode) +{ + FILE *in; + + in = tmpfile(); + if (!in) return NULL; + if (fwrite(buf, buflen, 1, in) != 1) { + fclose(in); + return NULL; + } + rewind(in); + return in; +} +#endif + static void liblrzip_index_update(size_t x, size_t *idx, void **queue) { for (; x < *idx; x++) @@ -543,3 +560,81 @@ void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data) lr->control->info_data = data; } +bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level) +{ + Lrzip *lr; + FILE *s, *d; + struct stat st; + int fd; + + if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE)) return false; + + lrzip_init(); + if (!mode) mode = LRZIP_MODE_COMPRESS_LZMA; + lr = lrzip_new(mode); + if (!lr) return false; + lrzip_config_env(lr); + + s = fmemopen((void*)source, source_len, "r"); + d = tmpfile(); + if ((!s) || (!d)) goto error; + + if (!lrzip_file_add(lr, s)) goto error; + lrzip_outfile_set(lr, d); + if (!lrzip_compression_level_set(lr, compress_level)) goto error; + if (!lrzip_run(lr)) goto error; + + fd = fileno(d); + if (fstat(fd, &st)) goto error; + *dest_len = st.st_size; + fread(dest, sizeof(char), st.st_size, d); + if (ferror(d)) goto error; + fclose(s); + fclose(d); + return true; + +error: + if (s) fclose(s); + if (d) fclose(d); + lrzip_free(lr); + return false; +} + + +bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ + Lrzip *lr; + FILE *s, *d; + struct stat st; + int fd; + + if ((!dest) || (!dest_len) || (!source) || (!source_len)) return false; + + lrzip_init(); + lr = lrzip_new(LRZIP_MODE_DECOMPRESS); + if (!lr) return false; + lrzip_config_env(lr); + + s = fmemopen((void*)source, source_len, "r"); + d = tmpfile(); + if ((!s) || (!d)) goto error; + + if (!lrzip_file_add(lr, s)) goto error; + lrzip_outfile_set(lr, d); + if (!lrzip_run(lr)) goto error; + + fd = fileno(d); + if (fstat(fd, &st)) goto error; + *dest_len = st.st_size; + fread(dest, sizeof(char), st.st_size, d); + if (ferror(d)) goto error; + fclose(s); + fclose(d); + return true; + +error: + if (s) fclose(s); + if (d) fclose(d); + lrzip_free(lr); + return false; +} diff --git a/liblrzip.h b/liblrzip.h index 4c293193..89c7c4fa 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -112,5 +112,43 @@ void lrzip_log_stderr_set(Lrzip *lr, FILE *err); FILE *lrzip_log_stderr_get(Lrzip *lr); void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data); void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data); +bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len); +bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level); + +static inline bool lrzip_compress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, 7); } + +static inline bool lrzip_lcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, 7); } + +static inline bool lrzip_gcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, 7); } + +static inline bool lrzip_zcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, 7); } + +static inline bool lrzip_bcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, 7); } + +static inline bool lrzip_rcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, 7); } + +static inline bool lrzip_compress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, compress_level); } + +static inline bool lrzip_lcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, compress_level); } + +static inline bool lrzip_gcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, compress_level); } + +static inline bool lrzip_zcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, compress_level); } + +static inline bool lrzip_bcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, compress_level); } + +static inline bool lrzip_rcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, compress_level); } #endif From 8f5fb91682f8e3deff03dec161a947276e66b4a8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 17 Aug 2011 21:16:37 +1000 Subject: [PATCH 24/30] Fix warning. --- liblrzip.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/liblrzip.c b/liblrzip.c index 7871ad31..06b81f83 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -587,8 +587,10 @@ bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source fd = fileno(d); if (fstat(fd, &st)) goto error; *dest_len = st.st_size; - fread(dest, sizeof(char), st.st_size, d); - if (ferror(d)) goto error; + if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) + goto error; + if (unlikely(ferror(d))) + goto error; fclose(s); fclose(d); return true; @@ -626,8 +628,10 @@ bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, u fd = fileno(d); if (fstat(fd, &st)) goto error; *dest_len = st.st_size; - fread(dest, sizeof(char), st.st_size, d); - if (ferror(d)) goto error; + if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) + goto error; + if (unlikely(ferror(d))) + goto error; fclose(s); fclose(d); return true; From f8661d4356e73f1492591ea2aed2869a206836b2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 17 Aug 2011 21:28:43 +1000 Subject: [PATCH 25/30] Add freebsd to fake memopen as well. --- liblrzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblrzip.c b/liblrzip.c index 06b81f83..a03ca725 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -19,7 +19,7 @@ #include "lrzip.h" #include "rzip.h" -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__FreeBSD__) # define fmemopen(s, len, modes) fake_fmemopen((s), (len), (modes)) static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode) { From 9f5498b6d3cb256902d18ac3d50ec957afa6185f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Sep 2011 14:40:35 +1000 Subject: [PATCH 26/30] Fix some encryption typos --- liblrzip.h | 2 +- liblrzip_demo.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/liblrzip.h b/liblrzip.h index 89c7c4fa..2a1e9448 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -63,7 +63,7 @@ typedef enum { typedef void (*Lrzip_Info_Cb)(void *data, int pct, int chunk_pct); typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *format, va_list args); -typedef void (*Lrzip_Password_Cb)(void *, char **, size_t); +typedef void (*Lrzip_Password_Cb)(void *, char *, size_t); bool lrzip_init(void); void lrzip_config_env(Lrzip *lr); diff --git a/liblrzip_demo.c b/liblrzip_demo.c index 4565e74b..f4a7af10 100644 --- a/liblrzip_demo.c +++ b/liblrzip_demo.c @@ -91,7 +91,7 @@ static int get_pass(char *s, size_t slen) return len; } -static void pass_cb(void *data __UNUSED__, char **pass_string, size_t pass_len) +static void pass_cb(void *data __UNUSED__, char *pass_string, size_t pass_len) { int len; struct termios termios_p; @@ -101,7 +101,7 @@ static void pass_cb(void *data __UNUSED__, char **pass_string, size_t pass_len) tcsetattr(fileno(stdin), 0, &termios_p); printf("Enter passphrase: "); - len = get_pass(*pass_string, pass_len); + len = get_pass(pass_string, pass_len); printf("\n"); if (len < 1) exit(1); From c6656650204c77c72c358c8a24041e783c5e0227 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Sep 2011 14:40:51 +1000 Subject: [PATCH 27/30] Formatting fixes --- liblrzip.c | 8 ++++---- liblrzip_demo.c | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/liblrzip.c b/liblrzip.c index a03ca725..9b50971b 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -32,7 +32,7 @@ static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode) return NULL; } rewind(in); - return in; + return in; } #endif @@ -180,8 +180,8 @@ bool lrzip_compression_level_set(Lrzip *lr, unsigned int level) unsigned int lrzip_compression_level_get(Lrzip *lr) { - if (!lr) return 0; - return lr->control->compression_level; + if (!lr) return 0; + return lr->control->compression_level; } void lrzip_flags_set(Lrzip *lr, unsigned int flags) @@ -493,7 +493,7 @@ bool lrzip_run(Lrzip *lr) /* compute total time */ gettimeofday(&end_time, NULL); total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) - - (start_time.tv_sec + (double)start_time.tv_usec / 1000000); + (start_time.tv_sec + (double)start_time.tv_usec / 1000000); hours = (int)total_time / 3600; minutes = (int)(total_time / 60) % 60; seconds = total_time - hours * 3600 - minutes * 60; diff --git a/liblrzip_demo.c b/liblrzip_demo.c index f4a7af10..83d30c9d 100644 --- a/liblrzip_demo.c +++ b/liblrzip_demo.c @@ -110,7 +110,8 @@ static void pass_cb(void *data __UNUSED__, char *pass_string, size_t pass_len) tcsetattr(fileno(stdin), 0, &termios_p); } -static void mode_check(Lrzip *lr, Lrzip_Mode mode) { +static void mode_check(Lrzip *lr, Lrzip_Mode mode) +{ Lrzip_Mode current = lrzip_mode_get(lr); if (current && (current != mode)) failure("Can only use one of -l, -b, -g, -z or -n\n"); @@ -320,7 +321,7 @@ int main(int argc, char *argv[]) if (get_hash) { const unsigned char *digest = lrzip_md5digest_get(lr); for (x = 0; x < 16; x++) - fprintf(stdout, "%02x", digest[x] & 0xFF); + fprintf(stdout, "%02x", digest[x] & 0xFF); } lrzip_free(lr); return 0; From b1c5d5504aadf930d22338cdae334f8a1af399fb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Sep 2011 14:53:11 +1000 Subject: [PATCH 28/30] Convert liblrzip.c into 1TBS style. --- liblrzip.c | 246 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 164 insertions(+), 82 deletions(-) diff --git a/liblrzip.c b/liblrzip.c index 9b50971b..ec5d0bd0 100644 --- a/liblrzip.c +++ b/liblrzip.c @@ -26,7 +26,8 @@ static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode) FILE *in; in = tmpfile(); - if (!in) return NULL; + if (!in) + return NULL; if (fwrite(buf, buflen, 1, in) != 1) { fclose(in); return NULL; @@ -45,7 +46,8 @@ static void liblrzip_index_update(size_t x, size_t *idx, void **queue) static bool liblrzip_setup_flags(Lrzip *lr) { - if (!lr) return false; + if (!lr) + return false; #define MODE_CHECK(X) \ case LRZIP_MODE_COMPRESS_##X: \ lr->control->flags ^= FLAG_NOT_LZMA; \ @@ -132,7 +134,8 @@ void lrzip_free(Lrzip *lr) { size_t x; - if ((!lr) || (!lr->infilename_buckets)) return; + if ((!lr) || (!lr->infilename_buckets)) + return; rzip_control_free(lr->control); for (x = 0; x < lr->infilename_idx; x++) free(lr->infilenames[x]); @@ -146,10 +149,13 @@ Lrzip *lrzip_new(Lrzip_Mode mode) Lrzip *lr; lr = calloc(1, sizeof(Lrzip)); - if (!lr) return NULL; + if (!lr) + return NULL; lr->control = calloc(1, sizeof(rzip_control)); - if (!lr->control) goto error; - if (!initialize_control(lr->control)) goto error; + if (!lr->control) + goto error; + if (!initialize_control(lr->control)) + goto error; lr->mode = mode; lr->control->library_mode = 1; return lr; @@ -160,106 +166,124 @@ Lrzip *lrzip_new(Lrzip_Mode mode) Lrzip_Mode lrzip_mode_get(Lrzip *lr) { - if (!lr) return LRZIP_MODE_NONE; + if (!lr) + return LRZIP_MODE_NONE; return lr->mode; } bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode) { - if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ)) return false; + if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ)) + return false; lr->mode = mode; return true; } bool lrzip_compression_level_set(Lrzip *lr, unsigned int level) { - if ((!lr) || (!level) || (level > 9)) return false; + if ((!lr) || (!level) || (level > 9)) + return false; lr->control->compression_level = level; return true; } unsigned int lrzip_compression_level_get(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->control->compression_level; } void lrzip_flags_set(Lrzip *lr, unsigned int flags) { - if (!lr) return; + if (!lr) + return; lr->flags = flags; } unsigned int lrzip_flags_get(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->flags; } void lrzip_nice_set(Lrzip *lr, int nice) { - if ((!lr) || (nice < -19) || (nice > 20)) return; + if ((!lr) || (nice < -19) || (nice > 20)) + return; lr->control->nice_val = nice; } int lrzip_nice_get(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->control->nice_val; } void lrzip_threads_set(Lrzip *lr, unsigned int threads) { - if ((!lr) || (!threads)) return; + if ((!lr) || (!threads)) + return; lr->control->threads = threads; } unsigned int lrzip_threads_get(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->control->threads; } void lrzip_compression_window_max_set(Lrzip *lr, int64_t size) { - if (!lr) return; + if (!lr) + return; lr->control->window = size; } int64_t lrzip_compression_window_max_get(Lrzip *lr) { - if (!lr) return -1; + if (!lr) + return -1; return lr->control->window; } unsigned int lrzip_files_count(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->infile_idx; } unsigned int lrzip_filenames_count(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->infilename_idx; } FILE **lrzip_files_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->infiles; } char **lrzip_filenames_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->infilenames; } bool lrzip_file_add(Lrzip *lr, FILE *file) { - if ((!lr) || (!file)) return false; - if (lr->infilenames) return false; + if ((!lr) || (!file)) + return false; + if (lr->infilenames) + return false; if (!lr->infile_buckets) { /* no files added */ lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*)); @@ -269,7 +293,8 @@ bool lrzip_file_add(Lrzip *lr, FILE *file) FILE **tmp; tmp = realloc(lr->infiles, (++lr->infile_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); - if (!tmp) return false; + if (!tmp) + return false; lr->infiles = tmp; } @@ -281,12 +306,16 @@ bool lrzip_file_del(Lrzip *lr, FILE *file) { size_t x; - if ((!lr) || (!file)) return false; - if (!lr->infile_buckets) return true; + if ((!lr) || (!file)) + return false; + if (!lr->infile_buckets) + return true; for (x = 0; x <= lr->infile_idx + 1; x++) { - if (!lr->infiles[x]) return true; /* not found */ - if (lr->infiles[x] != file) continue; /* not a match */ + if (!lr->infiles[x]) + return true; /* not found */ + if (lr->infiles[x] != file) + continue; /* not a match */ break; } /* update index */ @@ -297,7 +326,8 @@ bool lrzip_file_del(Lrzip *lr, FILE *file) FILE *lrzip_file_pop(Lrzip *lr) { FILE *ret; - if ((!lr) || (!lr->infile_buckets)) return NULL; + if ((!lr) || (!lr->infile_buckets)) + return NULL; ret = lr->infiles[0]; lrzip_file_del(lr, ret); return ret; @@ -305,7 +335,8 @@ FILE *lrzip_file_pop(Lrzip *lr) void lrzip_files_clear(Lrzip *lr) { - if ((!lr) || (!lr->infile_buckets)) return; + if ((!lr) || (!lr->infile_buckets)) + return; free(lr->infiles); lr->infiles = NULL; } @@ -314,10 +345,14 @@ bool lrzip_filename_add(Lrzip *lr, const char *file) { struct stat st; - if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-"))) return false; - if (lr->infiles) return false; - if (stat(file, &st)) return false; - if (S_ISDIR(st.st_mode)) return false; + if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-"))) + return false; + if (lr->infiles) + return false; + if (stat(file, &st)) + return false; + if (S_ISDIR(st.st_mode)) + return false; if (!lr->infilename_buckets) { /* no files added */ @@ -328,7 +363,8 @@ bool lrzip_filename_add(Lrzip *lr, const char *file) char **tmp; tmp = realloc(lr->infilenames, (++lr->infilename_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); - if (!tmp) return false; + if (!tmp) + return false; lr->infilenames = tmp; } @@ -340,12 +376,16 @@ bool lrzip_filename_del(Lrzip *lr, const char *file) { size_t x; - if ((!lr) || (!file) || (!file[0])) return false; - if (!lr->infilename_buckets) return true; + if ((!lr) || (!file) || (!file[0])) + return false; + if (!lr->infilename_buckets) + return true; for (x = 0; x <= lr->infilename_idx + 1; x++) { - if (!lr->infilenames[x]) return true; /* not found */ - if (strcmp(lr->infilenames[x], file)) continue; /* not a match */ + if (!lr->infilenames[x]) + return true; /* not found */ + if (strcmp(lr->infilenames[x], file)) + continue; /* not a match */ free(lr->infilenames[x]); break; } @@ -357,7 +397,8 @@ bool lrzip_filename_del(Lrzip *lr, const char *file) const char *lrzip_filename_pop(Lrzip *lr) { static char buf[4096]; - if ((!lr) || (!lr->infilename_buckets)) return NULL; + if ((!lr) || (!lr->infilename_buckets)) + return NULL; strcat(buf, lr->infilenames[0]); lrzip_filename_del(lr, buf); return &buf[0]; @@ -366,7 +407,8 @@ const char *lrzip_filename_pop(Lrzip *lr) void lrzip_filenames_clear(Lrzip *lr) { size_t x; - if ((!lr) || (!lr->infilename_buckets)) return; + if ((!lr) || (!lr->infilename_buckets)) + return; for (x = 0; x < lr->infilename_idx; x++) free(lr->infilenames[x]); free(lr->infilenames); @@ -375,14 +417,16 @@ void lrzip_filenames_clear(Lrzip *lr) void lrzip_suffix_set(Lrzip *lr, const char *suffix) { - if ((!lr) || (!suffix) || (!suffix[0])) return; + if ((!lr) || (!suffix) || (!suffix[0])) + return; free(lr->control->suffix); lr->control->suffix = strdup(suffix); } const char *lrzip_suffix_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->suffix; } @@ -391,7 +435,8 @@ void lrzip_outdir_set(Lrzip *lr, const char *dir) const char *slash; char *buf; size_t len; - if ((!lr) || (!dir) || (!dir[0])) return; + if ((!lr) || (!dir) || (!dir[0])) + return; free(lr->control->outdir); slash = strrchr(dir, '/'); if (slash && (slash[1] == 0)) { @@ -400,7 +445,8 @@ void lrzip_outdir_set(Lrzip *lr, const char *dir) } len = strlen(dir); buf = malloc(len + 2); - if (!buf) return; + if (!buf) + return; memcpy(buf, dir, len); buf[len] = '/'; buf[len + 1] = 0; @@ -409,41 +455,50 @@ void lrzip_outdir_set(Lrzip *lr, const char *dir) const char *lrzip_outdir_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->outdir; } void lrzip_outfile_set(Lrzip *lr, FILE *file) { - if ((!lr) || (file && (file == stderr))) return; - if (lr->control->outname) return; + if ((!lr) || (file && (file == stderr))) + return; + if (lr->control->outname) + return; lr->control->outFILE = file; } FILE *lrzip_outfile_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->outFILE; } void lrzip_outfilename_set(Lrzip *lr, const char *file) { - if ((!lr) || (file && (!file[0]))) return; - if (lr->control->outFILE) return; - if (lr->control->outname && file && (!strcmp(lr->control->outname, file))) return; + if ((!lr) || (file && (!file[0]))) + return; + if (lr->control->outFILE) + return; + if (lr->control->outname && file && (!strcmp(lr->control->outname, file))) + return; free(lr->control->outname); lr->control->outname = file ? strdup(file) : NULL; } const char *lrzip_outfilename_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->outname; } const unsigned char *lrzip_md5digest_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->md5_resblock; } @@ -454,10 +509,12 @@ bool lrzip_run(Lrzip *lr) double seconds,total_time; // for timers int hours,minutes; - if (!liblrzip_setup_flags(lr)) return false; + if (!liblrzip_setup_flags(lr)) + return false; control = lr->control; - if ((!lr->infile_idx) && (!lr->infilename_idx)) return false; + if ((!lr->infile_idx) && (!lr->infilename_idx)) + return false; if (lr->control->outFILE) { if (lr->control->outFILE == lr->control->msgout) lr->control->msgout = stderr; @@ -485,10 +542,13 @@ bool lrzip_run(Lrzip *lr) } if (DECOMPRESS || TEST_ONLY) { - if (!decompress_file(lr->control)) return false; + if (!decompress_file(lr->control)) + return false; } else if (INFO) { - if (!get_fileinfo(lr->control)) return false; - } else if (!compress_file(lr->control)) return false; + if (!get_fileinfo(lr->control)) + return false; + } else if (!compress_file(lr->control)) + return false; /* compute total time */ gettimeofday(&end_time, NULL); @@ -505,57 +565,66 @@ bool lrzip_run(Lrzip *lr) void lrzip_log_level_set(Lrzip *lr, int level) { - if (!lr) return; + if (!lr) + return; lr->control->log_level = level; } int lrzip_log_level_get(Lrzip *lr) { - if (!lr) return 0; + if (!lr) + return 0; return lr->control->log_level; } void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data) { - if (!lr) return; + if (!lr) + return; lr->control->log_cb = (void*)cb; lr->control->log_data = log_data; } void lrzip_log_stdout_set(Lrzip *lr, FILE *out) { - if (!lr) return; + if (!lr) + return; lr->control->msgout = out; } FILE *lrzip_log_stdout_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->msgout; } void lrzip_log_stderr_set(Lrzip *lr, FILE *err) { - if (!lr) return; + if (!lr) + return; lr->control->msgerr = err; } FILE *lrzip_log_stderr_get(Lrzip *lr) { - if (!lr) return NULL; + if (!lr) + return NULL; return lr->control->msgerr; } void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data) { - if (!lr) return; + if (!lr) + return; lr->control->pass_cb = (void*)cb; lr->control->pass_data = data; } void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data) { - if (!lr) return; + if (!lr) + return; lr->control->info_cb = (void*)cb; lr->control->info_data = data; } @@ -567,25 +636,32 @@ bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source struct stat st; int fd; - if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE)) return false; + if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE)) + return false; lrzip_init(); if (!mode) mode = LRZIP_MODE_COMPRESS_LZMA; lr = lrzip_new(mode); - if (!lr) return false; + if (!lr) + return false; lrzip_config_env(lr); s = fmemopen((void*)source, source_len, "r"); d = tmpfile(); - if ((!s) || (!d)) goto error; + if ((!s) || (!d)) + goto error; - if (!lrzip_file_add(lr, s)) goto error; + if (!lrzip_file_add(lr, s)) + goto error; lrzip_outfile_set(lr, d); - if (!lrzip_compression_level_set(lr, compress_level)) goto error; - if (!lrzip_run(lr)) goto error; + if (!lrzip_compression_level_set(lr, compress_level)) + goto error; + if (!lrzip_run(lr)) + goto error; fd = fileno(d); - if (fstat(fd, &st)) goto error; + if (fstat(fd, &st)) + goto error; *dest_len = st.st_size; if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) goto error; @@ -610,23 +686,29 @@ bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, u struct stat st; int fd; - if ((!dest) || (!dest_len) || (!source) || (!source_len)) return false; + if ((!dest) || (!dest_len) || (!source) || (!source_len)) + return false; lrzip_init(); lr = lrzip_new(LRZIP_MODE_DECOMPRESS); - if (!lr) return false; + if (!lr) + return false; lrzip_config_env(lr); s = fmemopen((void*)source, source_len, "r"); d = tmpfile(); - if ((!s) || (!d)) goto error; + if ((!s) || (!d)) + goto error; - if (!lrzip_file_add(lr, s)) goto error; + if (!lrzip_file_add(lr, s)) + goto error; lrzip_outfile_set(lr, d); - if (!lrzip_run(lr)) goto error; + if (!lrzip_run(lr)) + goto error; fd = fileno(d); - if (fstat(fd, &st)) goto error; + if (fstat(fd, &st)) + goto error; *dest_len = st.st_size; if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) goto error; From d9fb70bffc24b2775689a5c2519c88242fee896a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 16 Sep 2011 23:06:17 +1000 Subject: [PATCH 29/30] Fixes. --- liblrzip_demo.c | 4 ++-- lrzip.c | 1 + lrzip.h | 2 +- runzip.c | 8 ++++---- stream.h | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/liblrzip_demo.c b/liblrzip_demo.c index 83d30c9d..76e4d219 100644 --- a/liblrzip_demo.c +++ b/liblrzip_demo.c @@ -11,11 +11,11 @@ #ifdef HAVE_ERRNO_H # include #else +extern int errno; +#endif #ifdef HAVE_UNISTD_H # include #endif -extern int errno; -#endif #include #include diff --git a/lrzip.c b/lrzip.c index 34075b42..18202973 100644 --- a/lrzip.c +++ b/lrzip.c @@ -276,6 +276,7 @@ bool preserve_perms(rzip_control *control, int fd_in, int fd_out) /* chown fail is not fatal_return(( */ if (unlikely(fchown(fd_out, st.st_uid, st.st_gid))) print_verbose("Warning, unable to set owner on %s\n", control->outfile); + return true; } /* Open a temporary outputfile to emulate stdout */ diff --git a/lrzip.h b/lrzip.h index a2b1e0f9..d7937761 100644 --- a/lrzip.h +++ b/lrzip.h @@ -25,7 +25,7 @@ inline i64 get_ram(rzip_control *control); i64 nloops(i64 seconds, uchar *b1, uchar *b2); bool write_magic(rzip_control *control); bool read_magic(rzip_control *control, int fd_in, i64 *expected_size); -void preserve_perms(rzip_control *control, int fd_in, int fd_out); +bool preserve_perms(rzip_control *control, int fd_in, int fd_out); int open_tmpoutfile(rzip_control *control); bool dump_tmpoutfile(rzip_control *control, int fd_out); int open_tmpinfile(rzip_control *control); diff --git a/runzip.c b/runzip.c index 1c2469d6..452077ac 100644 --- a/runzip.c +++ b/runzip.c @@ -314,7 +314,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 case 0: u = unzip_literal(control, ss, len, &cksum); if (unlikely(u == -1)) { - close_stream_in(ss); + close_stream_in(control, ss); return -1; } total += u; @@ -323,7 +323,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 default: u = unzip_match(control, ss, len, &cksum, chunk_bytes); if (unlikely(u == -1)) { - close_stream_in(ss); + close_stream_in(control, ss); return -1; } total += u; @@ -343,11 +343,11 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 if (!HAS_MD5) { good_cksum = read_u32(control, ss, 0, &err); if (unlikely(err)) { - close_stream_in(ss); + close_stream_in(control, ss); return -1; } if (unlikely(good_cksum != cksum)) { - close_stream_in(ss); + close_stream_in(control, ss); failure_return(("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum), -1); } print_maxverbose("Checksum for block: 0x%08x\n", cksum); diff --git a/stream.h b/stream.h index bb089d23..abf00c52 100644 --- a/stream.h +++ b/stream.h @@ -29,8 +29,8 @@ bool join_pthread(pthread_t th, void **thread_return); ssize_t write_1g(rzip_control *control, void *buf, i64 len); ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len); i64 get_readseek(rzip_control *control, int fd); -void prepare_streamout_threads(rzip_control *control); -void close_streamout_threads(rzip_control *control); +bool prepare_streamout_threads(rzip_control *control); +bool close_streamout_threads(rzip_control *control); void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes); void *open_stream_in(rzip_control *control, int f, int n, char cbytes); bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); From 91311882334dbf270e0e589f4ca4a8c3b2f97219 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 18 Sep 2011 09:34:54 +1000 Subject: [PATCH 30/30] Add missing header file to makefile definition. --- Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.am b/Makefile.am index 8c462855..898a2275 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,6 +34,7 @@ libtmplrzip_la_SOURCES = \ zpipe.cpp \ zpipe.h \ lrzip_private.h \ + liblrzip_private.h \ liblrzip.h \ lrzip.c \ lrzip.h \