diff --git a/Makefile.am b/Makefile.am index dc9a3cd9..e5db2979 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,20 +23,21 @@ m4/ltversion.m4 SUBDIRS = lzma man doc -AM_CFLAGS = -I lzma/C -DNDEBUG +AM_CFLAGS = -I. -I lzma/C -DNDEBUG +AM_CXXFLAGS = $(AM_CFLAGS) lrztardir = $(bindir) lrztar_SCRIPTS = lrztar -bin_PROGRAMS = lrzip -lrzip_SOURCES = \ +noinst_LTLIBRARIES = libtmplrzip.la +libtmplrzip_la_SOURCES = \ zpipe.cpp \ zpipe.h \ lrzip_private.h \ + liblrzip_private.h \ liblrzip.h \ lrzip.c \ lrzip.h \ - main.c \ rzip.h \ rzip.c \ runzip.c \ @@ -51,12 +52,32 @@ lrzip_SOURCES = \ aes.h \ sha4.c \ sha4.h +libtmplrzip_la_LIBADD = lzma/C/liblzma.la + -lrzip_LDADD = lzma/C/liblzma.la +lib_LTLIBRARIES = liblrzip.la +liblrzip_la_SOURCES = \ + liblrzip.c +nodist_EXTRA_liblrzip_la_SOURCES = dummy.cxx +liblrzip_la_LIBADD = libtmplrzip.la + +bin_PROGRAMS = lrzip +lrzip_SOURCES = \ + main.c +nodist_EXTRA_lrzip_SOURCES = dummyy.cxx + +lrzip_LDADD = libtmplrzip.la if STATIC lrzip_LDFLAGS = -all-static endif +noinst_PROGRAMS = decompress_demo liblrzip_demo +decompress_demo_SOURCES = decompress_demo.c +decompress_demo_LDADD = liblrzip.la + +liblrzip_demo_SOURCES = liblrzip_demo.c +liblrzip_demo_LDADD = liblrzip.la + dist_doc_DATA = \ AUTHORS \ BUGS \ diff --git a/aes.c b/aes.c index 7dde41b4..f10eb795 100644 --- a/aes.c +++ b/aes.c @@ -61,10 +61,10 @@ * Forward S-box & tables */ static unsigned char FSb[256]; -static unsigned long FT0[256]; -static unsigned long FT1[256]; -static unsigned long FT2[256]; -static unsigned long FT3[256]; +static unsigned long FT0[256]; +static unsigned long FT1[256]; +static unsigned long FT2[256]; +static unsigned long FT3[256]; /* * Reverse S-box & tables @@ -501,7 +501,7 @@ int aes_crypt_cbc( aes_context *ctx, { if( padlock_xcryptcbc( ctx, mode, length, iv, input, output ) == 0 ) return( 0 ); - + // If padlock data misaligned, we just fall back to // unaccelerated mode // diff --git a/configure.ac b/configure.ac index 701e25f0..9d777a5c 100644 --- a/configure.ac +++ b/configure.ac @@ -52,6 +52,7 @@ AC_PROG_INSTALL AC_PROG_LN_S AC_SUBST(SHELL) AC_SYS_LARGEFILE +AC_FUNC_ALLOCA AC_CHECK_PROG([HAVE_POD2MAN], [pod2man], [yes]) AS_IF([test "$HAVE_POD2MAN" != "yes"], diff --git a/decompress_demo.c b/decompress_demo.c new file mode 100644 index 00000000..a980dd57 --- /dev/null +++ b/decompress_demo.c @@ -0,0 +1,41 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#undef NDEBUG +#include +#include +#include +#include +#include + +static const char *suffix_me(const char *file) +{ + const char *p; + static char buf[4096]; + + p = strrchr(file, '.'); + if (p && (strlen(p + 1) < 4)) + strncat(buf, file, p - file); + else + strcat(buf, file); + return &buf[0]; +} + +int main(int argc, char *argv[]) +{ + Lrzip *lr; + if ((argc != 2) && (argc != 3)) { + fprintf(stderr, "Usage: %s file [outfile]\n", argv[0]); + exit(1); + } + lr = lrzip_new(LRZIP_MODE_DECOMPRESS); + assert(lr); + lrzip_config_env(lr); + assert(lrzip_filename_add(lr, argv[1])); + if (argc == 2) + lrzip_outfilename_set(lr, suffix_me(argv[1])); + else + lrzip_outfilename_set(lr, argv[2]); + assert(lrzip_run(lr)); + return 0; +} diff --git a/liblrzip.c b/liblrzip.c new file mode 100644 index 00000000..ec5d0bd0 --- /dev/null +++ b/liblrzip.c @@ -0,0 +1,726 @@ +#include + +#ifdef HAVE_SYS_TIME_H +# include +#endif +#ifdef HAVE_SYS_RESOURCE_H +# include +#endif +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif + +/* needed for CRC routines */ +#include "lzma/C/7zCrc.h" +#include "util.h" +#include "lrzip.h" +#include "rzip.h" + +#if defined(__APPLE__) || defined(__FreeBSD__) +# define fmemopen(s, len, modes) fake_fmemopen((s), (len), (modes)) +static FILE *fake_fmemopen(void *buf, size_t buflen, const char *mode) +{ + FILE *in; + + in = tmpfile(); + if (!in) + return NULL; + if (fwrite(buf, buflen, 1, in) != 1) { + fclose(in); + return NULL; + } + rewind(in); + return in; +} +#endif + +static void liblrzip_index_update(size_t x, size_t *idx, void **queue) +{ + for (; x < *idx; x++) + queue[x] = queue[x + 1]; + (*idx)--; +} + +static bool liblrzip_setup_flags(Lrzip *lr) +{ + if (!lr) + return false; +#define MODE_CHECK(X) \ + case LRZIP_MODE_COMPRESS_##X: \ + lr->control->flags ^= FLAG_NOT_LZMA; \ + lr->control->flags |= FLAG_##X##_COMPRESS; \ + break + + + switch (lr->mode) { + case LRZIP_MODE_DECOMPRESS: + lr->control->flags |= FLAG_DECOMPRESS; + break; + case LRZIP_MODE_TEST: + lr->control->flags |= FLAG_TEST_ONLY; + break; + case LRZIP_MODE_INFO: + lr->control->flags |= FLAG_INFO; + break; + case LRZIP_MODE_COMPRESS_NONE: + lr->control->flags ^= FLAG_NOT_LZMA; + lr->control->flags |= FLAG_NO_COMPRESS; + break; + case LRZIP_MODE_COMPRESS_LZMA: + lr->control->flags ^= FLAG_NOT_LZMA; + break; + MODE_CHECK(LZO); + MODE_CHECK(BZIP2); + MODE_CHECK(ZLIB); + MODE_CHECK(ZPAQ); +#undef MODE_CHECK + default: + return false; + } + setup_overhead(lr->control); + if (lr->flags & LRZIP_FLAG_VERIFY) { + lr->control->flags |= FLAG_CHECK; + lr->control->flags |= FLAG_HASH; + } + if (lr->flags & LRZIP_FLAG_REMOVE_DESTINATION) + lr->control->flags |= FLAG_FORCE_REPLACE; + if (lr->flags & LRZIP_FLAG_REMOVE_SOURCE) + lr->control->flags &= ~FLAG_KEEP_FILES; + if (lr->flags & LRZIP_FLAG_KEEP_BROKEN) + lr->control->flags |= FLAG_KEEP_BROKEN; + if (lr->flags & LRZIP_FLAG_DISABLE_LZO_CHECK) + lr->control->flags &= ~FLAG_THRESHOLD; + if (lr->flags & LRZIP_FLAG_UNLIMITED_RAM) + lr->control->flags |= FLAG_UNLIMITED; + if (lr->flags & LRZIP_FLAG_ENCRYPT) + lr->control->flags |= FLAG_ENCRYPT; + if (lr->control->log_level > 0) { + lr->control->flags |= FLAG_SHOW_PROGRESS; + if (lr->control->log_level > 1) { + lr->control->flags |= FLAG_VERBOSITY; + if (lr->control->log_level > 2) + lr->control->flags |= FLAG_VERBOSITY_MAX; + } + } else lr->control->flags ^= (FLAG_VERBOSE | FLAG_SHOW_PROGRESS); + return true; +} + + +bool lrzip_init(void) +{ + /* generate crc table */ + CrcGenerateTable(); + return true; +} + +void lrzip_config_env(Lrzip *lr) +{ + const char *eptr; + /* Get Preloaded Defaults from lrzip.conf + * Look in ., $HOME/.lrzip/, /etc/lrzip. + * If LRZIP=NOCONFIG is set, then ignore config + */ + eptr = getenv("LRZIP"); + if (!eptr) + read_config(lr->control); + else if (!strstr(eptr,"NOCONFIG")) + read_config(lr->control); +} + +void lrzip_free(Lrzip *lr) +{ + size_t x; + + if ((!lr) || (!lr->infilename_buckets)) + return; + rzip_control_free(lr->control); + for (x = 0; x < lr->infilename_idx; x++) + free(lr->infilenames[x]); + free(lr->infilenames); + free(lr->infiles); + free(lr); +} + +Lrzip *lrzip_new(Lrzip_Mode mode) +{ + Lrzip *lr; + + lr = calloc(1, sizeof(Lrzip)); + if (!lr) + return NULL; + lr->control = calloc(1, sizeof(rzip_control)); + if (!lr->control) + goto error; + if (!initialize_control(lr->control)) + goto error; + lr->mode = mode; + lr->control->library_mode = 1; + return lr; +error: + lrzip_free(lr); + return NULL; +} + +Lrzip_Mode lrzip_mode_get(Lrzip *lr) +{ + if (!lr) + return LRZIP_MODE_NONE; + return lr->mode; +} + +bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode) +{ + if ((!lr) || (mode > LRZIP_MODE_COMPRESS_ZPAQ)) + return false; + lr->mode = mode; + return true; +} + +bool lrzip_compression_level_set(Lrzip *lr, unsigned int level) +{ + if ((!lr) || (!level) || (level > 9)) + return false; + lr->control->compression_level = level; + return true; +} + +unsigned int lrzip_compression_level_get(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->control->compression_level; +} + +void lrzip_flags_set(Lrzip *lr, unsigned int flags) +{ + if (!lr) + return; + lr->flags = flags; +} + +unsigned int lrzip_flags_get(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->flags; +} + +void lrzip_nice_set(Lrzip *lr, int nice) +{ + if ((!lr) || (nice < -19) || (nice > 20)) + return; + lr->control->nice_val = nice; +} + +int lrzip_nice_get(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->control->nice_val; +} + +void lrzip_threads_set(Lrzip *lr, unsigned int threads) +{ + if ((!lr) || (!threads)) + return; + lr->control->threads = threads; +} + +unsigned int lrzip_threads_get(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->control->threads; +} + +void lrzip_compression_window_max_set(Lrzip *lr, int64_t size) +{ + if (!lr) + return; + lr->control->window = size; +} + +int64_t lrzip_compression_window_max_get(Lrzip *lr) +{ + if (!lr) + return -1; + return lr->control->window; +} + +unsigned int lrzip_files_count(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->infile_idx; +} + +unsigned int lrzip_filenames_count(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->infilename_idx; +} + +FILE **lrzip_files_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->infiles; +} + +char **lrzip_filenames_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->infilenames; +} + +bool lrzip_file_add(Lrzip *lr, FILE *file) +{ + if ((!lr) || (!file)) + return false; + if (lr->infilenames) + return false; + if (!lr->infile_buckets) { + /* no files added */ + lr->infiles = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*)); + lr->infile_buckets++; + } else if (lr->infile_idx == INFILE_BUCKET_SIZE * lr->infile_buckets + 1) { + /* all buckets full, create new bucket */ + FILE **tmp; + + tmp = realloc(lr->infiles, (++lr->infile_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); + if (!tmp) + return false; + lr->infiles = tmp; + } + + lr->infiles[lr->infile_idx++] = file; + return true; +} + +bool lrzip_file_del(Lrzip *lr, FILE *file) +{ + size_t x; + + if ((!lr) || (!file)) + return false; + if (!lr->infile_buckets) + return true; + + for (x = 0; x <= lr->infile_idx + 1; x++) { + if (!lr->infiles[x]) + return true; /* not found */ + if (lr->infiles[x] != file) + continue; /* not a match */ + break; + } + /* update index */ + liblrzip_index_update(x, &lr->infile_idx, (void**)lr->infiles); + return true; +} + +FILE *lrzip_file_pop(Lrzip *lr) +{ + FILE *ret; + if ((!lr) || (!lr->infile_buckets)) + return NULL; + ret = lr->infiles[0]; + lrzip_file_del(lr, ret); + return ret; +} + +void lrzip_files_clear(Lrzip *lr) +{ + if ((!lr) || (!lr->infile_buckets)) + return; + free(lr->infiles); + lr->infiles = NULL; +} + +bool lrzip_filename_add(Lrzip *lr, const char *file) +{ + struct stat st; + + if ((!lr) || (!file) || (!file[0]) || (!strcmp(file, "-"))) + return false; + if (lr->infiles) + return false; + if (stat(file, &st)) + return false; + if (S_ISDIR(st.st_mode)) + return false; + + if (!lr->infilename_buckets) { + /* no files added */ + lr->infilenames = calloc(INFILE_BUCKET_SIZE + 1, sizeof(void*)); + lr->infilename_buckets++; + } else if (lr->infilename_idx == INFILE_BUCKET_SIZE * lr->infilename_buckets + 1) { + /* all buckets full, create new bucket */ + char **tmp; + + tmp = realloc(lr->infilenames, (++lr->infilename_buckets * INFILE_BUCKET_SIZE + 1) * sizeof(void*)); + if (!tmp) + return false; + lr->infilenames = tmp; + } + + lr->infilenames[lr->infilename_idx++] = strdup(file); + return true; +} + +bool lrzip_filename_del(Lrzip *lr, const char *file) +{ + size_t x; + + if ((!lr) || (!file) || (!file[0])) + return false; + if (!lr->infilename_buckets) + return true; + + for (x = 0; x <= lr->infilename_idx + 1; x++) { + if (!lr->infilenames[x]) + return true; /* not found */ + if (strcmp(lr->infilenames[x], file)) + continue; /* not a match */ + free(lr->infilenames[x]); + break; + } + /* update index */ + liblrzip_index_update(x, &lr->infilename_idx, (void**)lr->infilenames); + return true; +} + +const char *lrzip_filename_pop(Lrzip *lr) +{ + static char buf[4096]; + if ((!lr) || (!lr->infilename_buckets)) + return NULL; + strcat(buf, lr->infilenames[0]); + lrzip_filename_del(lr, buf); + return &buf[0]; +} + +void lrzip_filenames_clear(Lrzip *lr) +{ + size_t x; + if ((!lr) || (!lr->infilename_buckets)) + return; + for (x = 0; x < lr->infilename_idx; x++) + free(lr->infilenames[x]); + free(lr->infilenames); + lr->infilenames = NULL; +} + +void lrzip_suffix_set(Lrzip *lr, const char *suffix) +{ + if ((!lr) || (!suffix) || (!suffix[0])) + return; + free(lr->control->suffix); + lr->control->suffix = strdup(suffix); +} + +const char *lrzip_suffix_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->suffix; +} + +void lrzip_outdir_set(Lrzip *lr, const char *dir) +{ + const char *slash; + char *buf; + size_t len; + if ((!lr) || (!dir) || (!dir[0])) + return; + free(lr->control->outdir); + slash = strrchr(dir, '/'); + if (slash && (slash[1] == 0)) { + lr->control->outdir = strdup(dir); + return; + } + len = strlen(dir); + buf = malloc(len + 2); + if (!buf) + return; + memcpy(buf, dir, len); + buf[len] = '/'; + buf[len + 1] = 0; + lr->control->outdir = buf; +} + +const char *lrzip_outdir_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->outdir; +} + +void lrzip_outfile_set(Lrzip *lr, FILE *file) +{ + if ((!lr) || (file && (file == stderr))) + return; + if (lr->control->outname) + return; + lr->control->outFILE = file; +} + +FILE *lrzip_outfile_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->outFILE; +} + +void lrzip_outfilename_set(Lrzip *lr, const char *file) +{ + if ((!lr) || (file && (!file[0]))) + return; + if (lr->control->outFILE) + return; + if (lr->control->outname && file && (!strcmp(lr->control->outname, file))) + return; + free(lr->control->outname); + lr->control->outname = file ? strdup(file) : NULL; +} + +const char *lrzip_outfilename_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->outname; +} + +const unsigned char *lrzip_md5digest_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->md5_resblock; +} + +bool lrzip_run(Lrzip *lr) +{ + struct timeval start_time, end_time; + rzip_control *control; + double seconds,total_time; // for timers + int hours,minutes; + + if (!liblrzip_setup_flags(lr)) + return false; + control = lr->control; + + if ((!lr->infile_idx) && (!lr->infilename_idx)) + return false; + if (lr->control->outFILE) { + if (lr->control->outFILE == lr->control->msgout) + lr->control->msgout = stderr; + lr->control->flags |= FLAG_STDOUT; + register_outputfile(lr->control, lr->control->msgout); + } + + if (lr->infilenames) + lr->control->infile = lr->infilenames[0]; + else { + lr->control->inFILE = lr->infiles[0]; + control->flags |= FLAG_STDIN; + } + + if ((!STDOUT) && (!lr->control->msgout)) lr->control->msgout = stdout; + register_outputfile(lr->control, lr->control->msgout); + + setup_ram(lr->control); + + gettimeofday(&start_time, NULL); + + if (ENCRYPT && (!lr->control->pass_cb)) { + print_err("No password callback set!\n"); + return false; + } + + if (DECOMPRESS || TEST_ONLY) { + if (!decompress_file(lr->control)) + return false; + } else if (INFO) { + if (!get_fileinfo(lr->control)) + return false; + } else if (!compress_file(lr->control)) + return false; + + /* compute total time */ + gettimeofday(&end_time, NULL); + total_time = (end_time.tv_sec + (double)end_time.tv_usec / 1000000) - + (start_time.tv_sec + (double)start_time.tv_usec / 1000000); + hours = (int)total_time / 3600; + minutes = (int)(total_time / 60) % 60; + seconds = total_time - hours * 3600 - minutes * 60; + if (!INFO) + print_progress("Total time: %02d:%02d:%05.2f\n", hours, minutes, seconds); + + return true; +} + +void lrzip_log_level_set(Lrzip *lr, int level) +{ + if (!lr) + return; + lr->control->log_level = level; +} + +int lrzip_log_level_get(Lrzip *lr) +{ + if (!lr) + return 0; + return lr->control->log_level; +} + +void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data) +{ + if (!lr) + return; + lr->control->log_cb = (void*)cb; + lr->control->log_data = log_data; +} + +void lrzip_log_stdout_set(Lrzip *lr, FILE *out) +{ + if (!lr) + return; + lr->control->msgout = out; +} + +FILE *lrzip_log_stdout_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->msgout; +} + +void lrzip_log_stderr_set(Lrzip *lr, FILE *err) +{ + if (!lr) + return; + lr->control->msgerr = err; +} + +FILE *lrzip_log_stderr_get(Lrzip *lr) +{ + if (!lr) + return NULL; + return lr->control->msgerr; +} + +void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data) +{ + if (!lr) + return; + lr->control->pass_cb = (void*)cb; + lr->control->pass_data = data; +} + +void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data) +{ + if (!lr) + return; + lr->control->info_cb = (void*)cb; + lr->control->info_data = data; +} + +bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level) +{ + Lrzip *lr; + FILE *s, *d; + struct stat st; + int fd; + + if ((!dest) || (!dest_len) || (!source) || (!source_len) || (mode < LRZIP_MODE_COMPRESS_NONE)) + return false; + + lrzip_init(); + if (!mode) mode = LRZIP_MODE_COMPRESS_LZMA; + lr = lrzip_new(mode); + if (!lr) + return false; + lrzip_config_env(lr); + + s = fmemopen((void*)source, source_len, "r"); + d = tmpfile(); + if ((!s) || (!d)) + goto error; + + if (!lrzip_file_add(lr, s)) + goto error; + lrzip_outfile_set(lr, d); + if (!lrzip_compression_level_set(lr, compress_level)) + goto error; + if (!lrzip_run(lr)) + goto error; + + fd = fileno(d); + if (fstat(fd, &st)) + goto error; + *dest_len = st.st_size; + if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) + goto error; + if (unlikely(ferror(d))) + goto error; + fclose(s); + fclose(d); + return true; + +error: + if (s) fclose(s); + if (d) fclose(d); + lrzip_free(lr); + return false; +} + + +bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ + Lrzip *lr; + FILE *s, *d; + struct stat st; + int fd; + + if ((!dest) || (!dest_len) || (!source) || (!source_len)) + return false; + + lrzip_init(); + lr = lrzip_new(LRZIP_MODE_DECOMPRESS); + if (!lr) + return false; + lrzip_config_env(lr); + + s = fmemopen((void*)source, source_len, "r"); + d = tmpfile(); + if ((!s) || (!d)) + goto error; + + if (!lrzip_file_add(lr, s)) + goto error; + lrzip_outfile_set(lr, d); + if (!lrzip_run(lr)) + goto error; + + fd = fileno(d); + if (fstat(fd, &st)) + goto error; + *dest_len = st.st_size; + if (unlikely(fread(dest, sizeof(char), st.st_size, d) != st.st_size)) + goto error; + if (unlikely(ferror(d))) + goto error; + fclose(s); + fclose(d); + return true; + +error: + if (s) fclose(s); + if (d) fclose(d); + lrzip_free(lr); + return false; +} diff --git a/liblrzip.h b/liblrzip.h index d817af0f..2a1e9448 100644 --- a/liblrzip.h +++ b/liblrzip.h @@ -20,54 +20,135 @@ #ifndef LIBLRZIP_H #define LIBLRZIP_H -#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) -#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) -#define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA)) - -#define SHOW_PROGRESS (control->flags & FLAG_SHOW_PROGRESS) -#define KEEP_FILES (control->flags & FLAG_KEEP_FILES) -#define TEST_ONLY (control->flags & FLAG_TEST_ONLY) -#define FORCE_REPLACE (control->flags & FLAG_FORCE_REPLACE) -#define DECOMPRESS (control->flags & FLAG_DECOMPRESS) -#define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS) -#define LZO_COMPRESS (control->flags & FLAG_LZO_COMPRESS) -#define BZIP2_COMPRESS (control->flags & FLAG_BZIP2_COMPRESS) -#define ZLIB_COMPRESS (control->flags & FLAG_ZLIB_COMPRESS) -#define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS) -#define VERBOSE (control->flags & FLAG_VERBOSE) -#define VERBOSITY (control->flags & FLAG_VERBOSITY) -#define MAX_VERBOSE (control->flags & FLAG_VERBOSITY_MAX) -#define STDIN (control->flags & FLAG_STDIN) -#define STDOUT (control->flags & FLAG_STDOUT) -#define INFO (control->flags & FLAG_INFO) -#define UNLIMITED (control->flags & FLAG_UNLIMITED) -#define HASH_CHECK (control->flags & FLAG_HASH) -#define HAS_MD5 (control->flags & FLAG_MD5) -#define CHECK_FILE (control->flags & FLAG_CHECK) -#define KEEP_BROKEN (control->flags & FLAG_KEEP_BROKEN) -#define LZO_TEST (control->flags & FLAG_THRESHOLD) -#define TMP_OUTBUF (control->flags & FLAG_TMP_OUTBUF) -#define TMP_INBUF (control->flags & FLAG_TMP_INBUF) -#define ENCRYPT (control->flags & FLAG_ENCRYPT) - -#define print_output(format, args...) do {\ - fprintf(control->msgout, format, ##args); \ - fflush(control->msgout); \ -} while (0) - -#define print_progress(format, args...) do {\ - if (SHOW_PROGRESS) \ - print_output(format, ##args); \ -} while (0) - -#define print_verbose(format, args...) do {\ - if (VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -#define print_maxverbose(format, args...) do {\ - if (MAX_VERBOSE) \ - print_output(format, ##args); \ -} while (0) +#include +#include +#ifdef _WIN32 +# include +#else +# include +#endif + +typedef struct Lrzip Lrzip; + +typedef enum { + LRZIP_LOG_LEVEL_ERROR = 0, + LRZIP_LOG_LEVEL_INFO, + LRZIP_LOG_LEVEL_PROGRESS, + LRZIP_LOG_LEVEL_VERBOSE, + LRZIP_LOG_LEVEL_DEBUG +} Lrzip_Log_Level; + +typedef enum { + LRZIP_MODE_NONE = 0, + LRZIP_MODE_INFO, + LRZIP_MODE_TEST, + LRZIP_MODE_DECOMPRESS, + LRZIP_MODE_COMPRESS_NONE, + LRZIP_MODE_COMPRESS_LZO, + LRZIP_MODE_COMPRESS_ZLIB, + LRZIP_MODE_COMPRESS_BZIP2, + LRZIP_MODE_COMPRESS_LZMA, + LRZIP_MODE_COMPRESS_ZPAQ +} Lrzip_Mode; + +typedef enum { + LRZIP_FLAG_REMOVE_SOURCE = (1 << 0), + LRZIP_FLAG_REMOVE_DESTINATION = (1 << 1), + LRZIP_FLAG_KEEP_BROKEN = (1 << 2), + LRZIP_FLAG_VERIFY = (1 << 3), + LRZIP_FLAG_DISABLE_LZO_CHECK = (1 << 4), + LRZIP_FLAG_UNLIMITED_RAM = (1 << 5), + LRZIP_FLAG_ENCRYPT = (1 << 6) +} Lrzip_Flag; + +typedef void (*Lrzip_Info_Cb)(void *data, int pct, int chunk_pct); +typedef void (*Lrzip_Log_Cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *format, va_list args); +typedef void (*Lrzip_Password_Cb)(void *, char *, size_t); + +bool lrzip_init(void); +void lrzip_config_env(Lrzip *lr); +void lrzip_free(Lrzip *lr); +Lrzip *lrzip_new(Lrzip_Mode mode); +Lrzip_Mode lrzip_mode_get(Lrzip *lr); +bool lrzip_mode_set(Lrzip *lr, Lrzip_Mode mode); +bool lrzip_compression_level_set(Lrzip *lr, unsigned int level); +unsigned int lrzip_compression_level_get(Lrzip *lr); +void lrzip_flags_set(Lrzip *lr, unsigned int flags); +unsigned int lrzip_flags_get(Lrzip *lr); +void lrzip_nice_set(Lrzip *lr, int nice); +int lrzip_nice_get(Lrzip *lr); +void lrzip_threads_set(Lrzip *lr, unsigned int threads); +unsigned int lrzip_threads_get(Lrzip *lr); +void lrzip_compression_window_max_set(Lrzip *lr, int64_t size); +int64_t lrzip_compression_window_max_get(Lrzip *lr); +unsigned int lrzip_files_count(Lrzip *lr); +unsigned int lrzip_filenames_count(Lrzip *lr); +FILE **lrzip_files_get(Lrzip *lr); +char **lrzip_filenames_get(Lrzip *lr); +bool lrzip_file_add(Lrzip *lr, FILE *file); +bool lrzip_file_del(Lrzip *lr, FILE *file); +FILE *lrzip_file_pop(Lrzip *lr); +void lrzip_files_clear(Lrzip *lr); +bool lrzip_filename_add(Lrzip *lr, const char *file); +bool lrzip_filename_del(Lrzip *lr, const char *file); +const char *lrzip_filename_pop(Lrzip *lr); +void lrzip_filenames_clear(Lrzip *lr); +void lrzip_suffix_set(Lrzip *lr, const char *suffix); +const char *lrzip_suffix_get(Lrzip *lr); +void lrzip_outdir_set(Lrzip *lr, const char *dir); +const char *lrzip_outdir_get(Lrzip *lr); +void lrzip_outfile_set(Lrzip *lr, FILE *file); +FILE *lrzip_outfile_get(Lrzip *lr); +void lrzip_outfilename_set(Lrzip *lr, const char *file); +const char *lrzip_outfilename_get(Lrzip *lr); +const unsigned char *lrzip_md5digest_get(Lrzip *lr); +bool lrzip_run(Lrzip *lr); +void lrzip_log_level_set(Lrzip *lr, int level); +int lrzip_log_level_get(Lrzip *lr); +void lrzip_log_cb_set(Lrzip *lr, Lrzip_Log_Cb cb, void *log_data); +void lrzip_log_stdout_set(Lrzip *lr, FILE *out); +FILE *lrzip_log_stdout_get(Lrzip *lr); +void lrzip_log_stderr_set(Lrzip *lr, FILE *err); +FILE *lrzip_log_stderr_get(Lrzip *lr); +void lrzip_pass_cb_set(Lrzip *lr, Lrzip_Password_Cb cb, void *data); +void lrzip_info_cb_set(Lrzip *lr, Lrzip_Info_Cb cb, void *data); +bool lrzip_decompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len); +bool lrzip_compress_full(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, Lrzip_Mode mode, int compress_level); + +static inline bool lrzip_compress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, 7); } + +static inline bool lrzip_lcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, 7); } + +static inline bool lrzip_gcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, 7); } +static inline bool lrzip_zcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, 7); } + +static inline bool lrzip_bcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, 7); } + +static inline bool lrzip_rcompress(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, 7); } + +static inline bool lrzip_compress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZMA, compress_level); } + +static inline bool lrzip_lcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_LZO, compress_level); } + +static inline bool lrzip_gcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZLIB, compress_level); } + +static inline bool lrzip_zcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_ZPAQ, compress_level); } + +static inline bool lrzip_bcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_BZIP2, compress_level); } + +static inline bool lrzip_rcompress2(void *dest, unsigned long *dest_len, const void *source, unsigned long source_len, int compress_level) +{ return lrzip_compress_full(dest, dest_len, source, source_len, LRZIP_MODE_COMPRESS_NONE, compress_level); } #endif + diff --git a/liblrzip_demo.c b/liblrzip_demo.c new file mode 100644 index 00000000..76e4d219 --- /dev/null +++ b/liblrzip_demo.c @@ -0,0 +1,328 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#undef NDEBUG +#include +#include +#ifdef HAVE_STRING_H +# include +#endif +#include +#ifdef HAVE_ERRNO_H +# include +#else +extern int errno; +#endif +#ifdef HAVE_UNISTD_H +# include +#endif +#include +#include + +#define failure(format, args...) do { \ + fprintf(stderr, format, ##args); \ + exit(1); \ +} while (0) + +static void usage(void) +{ + printf("lrzip version %s\n", PACKAGE_VERSION); + printf("Copyright (C) Con Kolivas 2006-2011\n"); + printf("Based on rzip "); + printf("Copyright (C) Andrew Tridgell 1998-2003\n\n"); + printf("Usage: lrzip [options] \n"); + printf("General options:\n"); + printf(" -c check integrity of file written on decompression\n"); + printf(" -d decompress\n"); + printf(" -e password protected sha512/aes128 encryption on compression\n"); + printf(" -h|-? show help\n"); + printf(" -H display md5 hash integrity information\n"); + printf(" -i show compressed file information\n"); + printf(" -q don't show compression progress\n"); + printf(" -t test compressed file integrity\n"); + printf(" -v[v] Increase verbosity\n"); + printf(" -V show version\n"); + printf("Options affecting output:\n"); + printf(" -D delete existing files\n"); + printf(" -f force overwrite of any existing files\n"); + printf(" -k keep broken or damaged output files\n"); + printf(" -o filename specify the output file name and/or path\n"); + printf(" -O directory specify the output directory when -o is not used\n"); + printf(" -S suffix specify compressed suffix (default '.lrz')\n"); + printf("Options affecting compression:\n"); + printf(" -b bzip2 compression\n"); + printf(" -g gzip compression using zlib\n"); + printf(" -l lzo compression (ultra fast)\n"); + printf(" -n no backend compression - prepare for other compressor\n"); + printf(" -z zpaq compression (best, extreme compression, extremely slow)\n"); + printf("Low level options:\n"); + printf(" -L level set lzma/bzip2/gzip compression level (1-9, default 7)\n"); + printf(" -N value Set nice value to value (default 19)\n"); + printf(" -p value Set processor count to override number of threads\n"); + printf(" -T Disable LZO compressibility testing\n"); + printf(" -U Use unlimited window size beyond ramsize (potentially much slower)\n"); + printf(" -w size maximum compression window in hundreds of MB\n"); + printf(" default chosen by heuristic dependent on ram and chosen compression\n"); + printf("\nLRZIP=NOCONFIG environment variable setting can be used to bypass lrzip.conf.\n"); + printf("TMP environment variable will be used for storage of temporary files when needed.\n"); + printf("TMPDIR may also be stored in lrzip.conf file.\n"); + printf("\nIf no filenames or \"-\" is specified, stdin/out will be used.\n"); +} + +static int get_pass(char *s, size_t slen) +{ + int len; + + memset(s, 0, slen); + if (!fgets(s, slen, stdin)) { + fprintf(stderr, "Failed to retrieve passphrase\n"); + return -1; + } + len = strlen(s); + if (len > 0 && ('\r' == s[len - 1] || '\n' == s[len - 1])) + s[len - 1] = '\0'; + if (len > 1 && ('\r' == s[len - 2] || '\n' == s[len - 2])) + s[len - 2] = '\0'; + len = strlen(s); + if (!len) { + fprintf(stderr, "Empty passphrase\n"); + return -1; + } + return len; +} + +static void pass_cb(void *data __UNUSED__, char *pass_string, size_t pass_len) +{ + int len; + struct termios termios_p; + /* Disable stdin echo to screen */ + tcgetattr(fileno(stdin), &termios_p); + termios_p.c_lflag &= ~ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); + + printf("Enter passphrase: "); + len = get_pass(pass_string, pass_len); + printf("\n"); + + if (len < 1) exit(1); + + termios_p.c_lflag |= ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); +} + +static void mode_check(Lrzip *lr, Lrzip_Mode mode) +{ + Lrzip_Mode current = lrzip_mode_get(lr); + if (current && (current != mode)) + failure("Can only use one of -l, -b, -g, -z or -n\n"); + lrzip_mode_set(lr, mode); +} + +int main(int argc, char *argv[]) +{ + Lrzip *lr; + extern int optind; + extern char *optarg; + int64_t x; + int c; + bool get_hash = false; + + lrzip_init(); + lr = lrzip_new(LRZIP_MODE_NONE); + assert(lr); + lrzip_config_env(lr); + lrzip_log_level_set(lr, LRZIP_LOG_LEVEL_PROGRESS); + while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) { + switch (c) { + case 'b': + mode_check(lr, LRZIP_MODE_COMPRESS_BZIP2); + break; + case 'c': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_VERIFY); + break; + case 'd': + mode_check(lr, LRZIP_MODE_DECOMPRESS); + break; + case 'D': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_SOURCE); + break; + case 'e': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_ENCRYPT); + break; + case 'f': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_REMOVE_DESTINATION); + break; + case 'g': + mode_check(lr, LRZIP_MODE_COMPRESS_ZLIB); + break; + case 'h': + case '?': + usage(); + return -1; + case 'H': + get_hash = true; + break; + case 'i': + mode_check(lr, LRZIP_MODE_INFO); + break; + case 'k': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_KEEP_BROKEN); + break; + case 'l': + mode_check(lr, LRZIP_MODE_COMPRESS_LZO); + break; + case 'L': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || ((x < 1) || (x > 9))) + failure("Invalid compression level (must be 1-9)\n"); + lrzip_compression_level_set(lr, (unsigned int)x); + break; + case 'n': + mode_check(lr, LRZIP_MODE_COMPRESS_NONE); + break; + case 'N': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || (x < -20 || x > 19)) + failure("Invalid nice value (must be -20..19)\n"); + lrzip_nice_set(lr, x); + break; + case 'o': + if (lrzip_outdir_get(lr)) + failure("Cannot have -o and -O together\n"); + if (!strcmp(optarg, "-")) + lrzip_outfile_set(lr, stdout); + else + lrzip_outfilename_set(lr, optarg); + break; + case 'O': + if (lrzip_outfilename_get(lr)) /* can't mix -o and -O */ + failure("Cannot have options -o and -O together\n"); + if (lrzip_outfile_get(lr)) + failure("Cannot specify an output directory when outputting to stdout\n"); + lrzip_outdir_set(lr, optarg); + break; + case 'p': + errno = 0; + x = strtol(optarg, NULL, 10); + if (errno || (x < 1)) + failure("Must have at least one thread\n"); + lrzip_threads_set(lr, (unsigned int)x); + break; + case 'q': + lrzip_log_level_set(lr, lrzip_log_level_get(lr) - 1); + break; + case 'S': + if (lrzip_outfilename_get(lr)) + failure("Specified output filename already, can't specify an extension.\n"); + if (lrzip_outfile_get(lr)) + failure("Cannot specify a filename suffix when outputting to stdout\n"); + lrzip_suffix_set(lr, optarg); + break; + case 't': + if (lrzip_outfilename_get(lr)) + failure("Cannot specify an output file name when just testing.\n"); + if (lrzip_flags_get(lr) & LRZIP_FLAG_REMOVE_SOURCE) + failure("Doubt that you want to delete a file when just testing.\n"); + mode_check(lr, LRZIP_MODE_TEST); + break; + case 'T': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_DISABLE_LZO_CHECK); + break; + case 'U': + lrzip_flags_set(lr, lrzip_flags_get(lr) | LRZIP_FLAG_UNLIMITED_RAM); + break; + case 'v': + lrzip_log_level_set(lr, lrzip_log_level_get(lr) + 1); + break; + case 'V': + printf("lrzip version %s\n", PACKAGE_VERSION); + exit(0); + break; + case 'w': + errno = 0; + x = strtoll(optarg, NULL, 10); + if (errno || (x < 1)) + failure("Invalid compression window '%s'!\n", optarg); + lrzip_compression_window_max_set(lr, x); + break; + case 'z': + mode_check(lr, LRZIP_MODE_COMPRESS_ZPAQ); + break; + } + } + /* LZMA is the default */ + if (!lrzip_mode_get(lr)) lrzip_mode_set(lr, LRZIP_MODE_COMPRESS_LZMA); + argc -= optind, argv += optind; + + if (lrzip_outfilename_get(lr) && (argc > 1)) + failure("Cannot specify output filename with more than 1 file\n"); + + if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_compression_window_max_get(lr)) { + fprintf(stderr, "If -U used, cannot specify a window size with -w.\n"); + lrzip_compression_window_max_set(lr, 0); + } + + if (argc < 1) lrzip_file_add(lr, stdin); + + if ((lrzip_flags_get(lr) & LRZIP_FLAG_UNLIMITED_RAM) && lrzip_files_count(lr)) { + fprintf(stderr, "Cannot have -U and stdin, unlimited mode disabled.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_UNLIMITED_RAM); + } + + /* If no output filename is specified, and we're using stdin, + * use stdout */ + if (lrzip_files_count(lr) && (!lrzip_outfilename_get(lr))) + lrzip_outfile_set(lr, stdout); + + if (lrzip_flags_get(lr) & LRZIP_FLAG_VERIFY) { + if (lrzip_mode_get(lr) != LRZIP_MODE_DECOMPRESS) { + fprintf(stderr, "Can only check file written on decompression.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY); + } else if (lrzip_outfile_get(lr)) { + fprintf(stderr, "Can't check file written when writing to stdout. Checking disabled.\n"); + lrzip_flags_set(lr, lrzip_flags_get(lr) & ~LRZIP_FLAG_VERIFY); + } + } + + for (x = 0; x < argc; x++) { + if (argv[x][0] != '-') { + assert(lrzip_filename_add(lr, argv[x])); + continue; + } + if (argv[x][1] == 0) { + assert(lrzip_file_add(lr, stdin)); + continue; + } + } + if (argc == 1) { + if (!lrzip_files_count(lr)) lrzip_file_add(lr, stdin); + if (lrzip_filenames_count(lr)) { + if (!lrzip_outfilename_get(lr)) { + char buf[4096]; + const char *infile; + size_t len; + + infile = lrzip_filenames_get(lr)[0]; + len = strlen(infile); + if (!strcmp(infile + len - 4, ".lrz")) + strncat(buf, infile, len - 4); + else + snprintf(buf, sizeof(buf), "%s.out", infile); + lrzip_outfilename_set(lr, buf); + } + } else if (!lrzip_outfile_get(lr)) lrzip_outfile_set(lr, stdout); + } + lrzip_log_stdout_set(lr, stdout); + lrzip_log_stderr_set(lr, stderr); + lrzip_pass_cb_set(lr, pass_cb, NULL); + if (!lrzip_run(lr)) exit(1); + if (get_hash) { + const unsigned char *digest = lrzip_md5digest_get(lr); + for (x = 0; x < 16; x++) + fprintf(stdout, "%02x", digest[x] & 0xFF); + } + lrzip_free(lr); + return 0; +} diff --git a/liblrzip_private.h b/liblrzip_private.h new file mode 100644 index 00000000..d75f1504 --- /dev/null +++ b/liblrzip_private.h @@ -0,0 +1,22 @@ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include +#include + +#define INFILE_BUCKET_SIZE 10 + +struct Lrzip +{ + Lrzip_Mode mode; + unsigned int flags; + rzip_control *control; + + /* bucket allocation is used here to avoid frequent calls to realloc */ + char **infilenames; + size_t infilename_idx; + size_t infilename_buckets; + FILE **infiles; + size_t infile_idx; + size_t infile_buckets; +}; diff --git a/lrzip.c b/lrzip.c index 675b9371..18202973 100644 --- a/lrzip.c +++ b/lrzip.c @@ -45,16 +45,18 @@ #ifdef HAVE_ARPA_INET_H # include #endif +#include #include "md5.h" #include "rzip.h" #include "runzip.h" #include "util.h" #include "stream.h" -#include "liblrzip.h" /* flag defines */ #define MAGIC_LEN (24) +static void release_hashes(rzip_control *control); + static i64 fdout_seekto(rzip_control *control, i64 pos) { if (TMP_OUTBUF) { @@ -69,7 +71,69 @@ static i64 fdout_seekto(rzip_control *control, i64 pos) return lseek(control->fd_out, pos, SEEK_SET); } -void write_magic(rzip_control *control) +#ifdef __APPLE__ +# include +inline i64 get_ram(void) +{ + int mib[2]; + size_t len; + i64 *p, ramsize; + + mib[0] = CTL_HW; + mib[1] = HW_MEMSIZE; + sysctl(mib, 2, NULL, &len, NULL, 0); + p = malloc(len); + sysctl(mib, 2, p, &len, NULL, 0); + ramsize = *p; + + return ramsize; +} +#else /* __APPLE__ */ +inline i64 get_ram(rzip_control *control) +{ + i64 ramsize; + FILE *meminfo; + char aux[256]; + + ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE; + if (ramsize > 0) + return ramsize; + + /* Workaround for uclibc which doesn't properly support sysconf */ + if(!(meminfo = fopen("/proc/meminfo", "r"))) + fatal_return(("fopen\n"), -1); + + while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { + if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) { + fclose(meminfo); + fatal_return(("Failed to fgets in get_ram\n"), -1); + } + } + if (fclose(meminfo) == -1) + fatal_return(("fclose"), -1); + ramsize *= 1000; + + return ramsize; +} +#endif + +i64 nloops(i64 seconds, uchar *b1, uchar *b2) +{ + i64 nloops; + int nbits; + + nloops = ARBITRARY_AT_EPOCH * pow(MOORE_TIMES_PER_SECOND, seconds); + if (nloops < ARBITRARY) + nloops = ARBITRARY; + for (nbits = 0; nloops > 255; nbits ++) + nloops = nloops >> 1; + *b1 = nbits; + *b2 = nloops; + return nloops << nbits; +} + + +bool write_magic(rzip_control *control) { char magic[MAGIC_LEN]; @@ -107,11 +171,12 @@ void write_magic(rzip_control *control) magic[22] = 1; if (unlikely(fdout_seekto(control, 0))) - fatal("Failed to seek to BOF to write Magic Header\n"); + fatal_return(("Failed to seek to BOF to write Magic Header\n"), false); if (unlikely(put_fdout(control, magic, MAGIC_LEN) != MAGIC_LEN)) - fatal("Failed to write magic header\n"); + fatal_return(("Failed to write magic header\n"), false); control->magic_written = 1; + return true; } static inline i64 enc_loops(uchar b1, uchar b2) @@ -119,14 +184,14 @@ static inline i64 enc_loops(uchar b1, uchar b2) return (i64)b2 << (i64)b1; } -static void get_magic(rzip_control *control, char *magic) +static bool get_magic(rzip_control *control, char *magic) { int encrypted, md5, i; i64 expected_size; uint32_t v; if (unlikely(strncmp(magic, "LRZI", 4))) - failure("Not an lrzip file\n"); + failure_return(("Not an lrzip file\n"), false); memcpy(&control->major_version, &magic[4], 1); memcpy(&control->minor_version, &magic[5], 1); @@ -169,7 +234,7 @@ static void get_magic(rzip_control *control, char *magic) if (encrypted == 1) control->flags |= FLAG_ENCRYPT; else - failure("Unknown encryption\n"); + failure_return(("Unknown encryption\n"), false); /* In encrypted files, the size field is used to store the salt * instead and the size is unknown, just like a STDOUT chunked * file */ @@ -181,34 +246,37 @@ static void get_magic(rzip_control *control, char *magic) print_output("Asked to decrypt a non-encrypted archive. Bypassing decryption.\n"); control->flags &= ~FLAG_ENCRYPT; } + return true; } -void read_magic(rzip_control *control, int fd_in, i64 *expected_size) +bool read_magic(rzip_control *control, int fd_in, i64 *expected_size) { char magic[MAGIC_LEN]; memset(magic, 0, sizeof(magic)); /* Initially read only st_size; + return true; } /* preserve ownership and permissions where possible */ -void preserve_perms(rzip_control *control, int fd_in, int fd_out) +bool preserve_perms(rzip_control *control, int fd_in, int fd_out) { struct stat st; if (unlikely(fstat(fd_in, &st))) - fatal("Failed to fstat input file\n"); + fatal_return(("Failed to fstat input file\n"), false); if (unlikely(fchmod(fd_out, (st.st_mode & 0666)))) print_verbose("Warning, unable to set permissions on %s\n", control->outfile); - /* chown fail is not fatal */ + /* chown fail is not fatal_return(( */ if (unlikely(fchown(fd_out, st.st_uid, st.st_gid))) print_verbose("Warning, unable to set owner on %s\n", control->outfile); + return true; } /* Open a temporary outputfile to emulate stdout */ @@ -221,24 +289,24 @@ int open_tmpoutfile(rzip_control *control) if (control->tmpdir) { control->outfile = realloc(NULL, strlen(control->tmpdir) + 16); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), -1); strcpy(control->outfile, control->tmpdir); strcat(control->outfile, "lrzipout.XXXXXX"); } else { control->outfile = realloc(NULL, 16); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), -1); strcpy(control->outfile, "lrzipout.XXXXXX"); } fd_out = mkstemp(control->outfile); if (unlikely(fd_out == -1)) - fatal("Failed to create out tmpfile: %s\n", control->outfile); - register_outfile(control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); + fatal_return(("Failed to create out tmpfile: %s\n", control->outfile), -1); + register_outfile(control, control->outfile, TEST_ONLY || STDOUT || !KEEP_BROKEN); return fd_out; } -static void fwrite_stdout(void *buf, i64 len) +static bool fwrite_stdout(rzip_control *control, void *buf, i64 len) { uchar *offset_buf = buf; ssize_t ret; @@ -250,17 +318,18 @@ static void fwrite_stdout(void *buf, i64 len) ret = one_g; else ret = len; - ret = fwrite(offset_buf, 1, ret, stdout); + ret = fwrite(offset_buf, 1, ret, control->outFILE); if (unlikely(ret <= 0)) - fatal("Failed to fwrite in fwrite_stdout\n"); + fatal_return(("Failed to fwrite in fwrite_stdout\n"), false); len -= ret; offset_buf += ret; total += ret; } - fflush(stdout); + fflush(control->outFILE); + return true; } -void write_fdout(rzip_control *control, void *buf, i64 len) +bool write_fdout(rzip_control *control, void *buf, i64 len) { uchar *offset_buf = buf; ssize_t ret; @@ -269,27 +338,30 @@ void write_fdout(rzip_control *control, void *buf, i64 len) ret = MIN(len, one_g); ret = write(control->fd_out, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal("Failed to write to fd_out in write_fdout\n"); + fatal_return(("Failed to write to fd_out in write_fdout\n"), false);; len -= ret; offset_buf += ret; } + return true; } -void flush_tmpoutbuf(rzip_control *control) +bool flush_tmpoutbuf(rzip_control *control) { if (!TEST_ONLY) { print_maxverbose("Dumping buffer to physical file.\n"); - if (STDOUT) - fwrite_stdout(control->tmp_outbuf, control->out_len); - else - write_fdout(control, control->tmp_outbuf, control->out_len); + if (STDOUT) { + if (unlikely(!fwrite_stdout(control, control->tmp_outbuf, control->out_len))) return false; + } else { + if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) return false; + } } control->out_relofs += control->out_len; control->out_ofs = control->out_len = 0; + return true; } /* Dump temporary outputfile to perform stdout */ -void dump_tmpoutfile(rzip_control *control, int fd_out) +bool dump_tmpoutfile(rzip_control *control, int fd_out) { FILE *tmpoutfp; int tmpchar; @@ -298,24 +370,25 @@ void dump_tmpoutfile(rzip_control *control, int fd_out) fsync(fd_out); tmpoutfp = fdopen(fd_out, "r"); if (unlikely(tmpoutfp == NULL)) - fatal("Failed to fdopen out tmpfile\n"); + fatal_return(("Failed to fdopen out tmpfile\n"), false); rewind(tmpoutfp); if (!TEST_ONLY) { - print_verbose("Dumping temporary file to stdout.\n"); + print_verbose("Dumping temporary file to control->outFILE.\n"); while ((tmpchar = fgetc(tmpoutfp)) != EOF) putchar(tmpchar); - fflush(stdout); + fflush(control->outFILE); rewind(tmpoutfp); } if (unlikely(ftruncate(fd_out, 0))) - fatal("Failed to ftruncate fd_out in dump_tmpoutfile\n"); + fatal_return(("Failed to ftruncate fd_out in dump_tmpoutfile\n"), false); + return true; } /* Used if we're unable to read STDIN into the temporary buffer, shunts data * to temporary file */ -void write_fdin(rzip_control *control) +bool write_fdin(rzip_control *control) { uchar *offset_buf = control->tmp_inbuf; i64 len = control->in_len; @@ -325,10 +398,11 @@ void write_fdin(rzip_control *control) ret = MIN(len, one_g); ret = write(control->fd_in, offset_buf, (size_t)ret); if (unlikely(ret <= 0)) - fatal("Failed to write to fd_in in write_fdin\n"); + fatal_return(("Failed to write to fd_in in write_fdin\n"), false); len -= ret; offset_buf += ret; } + return true; } /* Open a temporary inputfile to perform stdin decompression */ @@ -339,28 +413,31 @@ int open_tmpinfile(rzip_control *control) if (control->tmpdir) { control->infile = malloc(strlen(control->tmpdir) + 15); if (unlikely(!control->infile)) - fatal("Failed to allocate infile name\n"); + fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, control->tmpdir); strcat(control->infile, "lrzipin.XXXXXX"); } else { control->infile = malloc(15); if (unlikely(!control->infile)) - fatal("Failed to allocate infile name\n"); + fatal_return(("Failed to allocate infile name\n"), -1); strcpy(control->infile, "lrzipin.XXXXXX"); } fd_in = mkstemp(control->infile); if (unlikely(fd_in == -1)) - fatal("Failed to create in tmpfile: %s\n", control->infile); - register_infile(control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); + fatal_return(("Failed to create in tmpfile: %s\n", control->infile), -1); + register_infile(control, control->infile, (DECOMPRESS || TEST_ONLY) && STDIN); /* Unlink temporary file immediately to minimise chance of files left - * lying around in cases of failure. */ - if (unlikely(unlink(control->infile))) + * lying around in cases of failure_return((. */ + if (unlikely(unlink(control->infile))) { fatal("Failed to unlink tmpfile: %s\n", control->infile); + close(fd_in); + return -1; + } return fd_in; } -static void read_tmpinmagic(rzip_control *control) +static bool read_tmpinmagic(rzip_control *control) { char magic[MAGIC_LEN]; int i, tmpchar; @@ -369,14 +446,14 @@ static void read_tmpinmagic(rzip_control *control) for (i = 0; i < 24; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure("Reached end of file on STDIN prematurely on v05 magic read\n"); + failure_return(("Reached end of file on STDIN prematurely on v05 magic read\n"), false); magic[i] = (char)tmpchar; } - get_magic(control, magic); + return get_magic(control, magic); } /* Read data from stdin into temporary inputfile */ -void read_tmpinfile(rzip_control *control, int fd_in) +bool read_tmpinfile(rzip_control *control, int fd_in) { FILE *tmpinfp; int tmpchar; @@ -385,18 +462,19 @@ void read_tmpinfile(rzip_control *control, int fd_in) fprintf(control->msgout, "Copying from stdin.\n"); tmpinfp = fdopen(fd_in, "w+"); if (unlikely(tmpinfp == NULL)) - fatal("Failed to fdopen in tmpfile\n"); + fatal_return(("Failed to fdopen in tmpfile\n"), false); while ((tmpchar = getchar()) != EOF) fputc(tmpchar, tmpinfp); fflush(tmpinfp); rewind(tmpinfp); + return true; } /* To perform STDOUT, we allocate a proportion of ram that is then used as * a pseudo-temporary file */ -static void open_tmpoutbuf(rzip_control *control) +static bool open_tmpoutbuf(rzip_control *control) { control->flags |= FLAG_TMP_OUTBUF; control->out_maxlen = control->maxram; @@ -404,9 +482,10 @@ static void open_tmpoutbuf(rzip_control *control) * fall back to a real temporary file */ control->tmp_outbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_outbuf)) - fatal("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"); + fatal_return(("Failed to malloc tmp_outbuf in open_tmpoutbuf\n"), false); if (!DECOMPRESS && !TEST_ONLY) - control->out_ofs = control->out_len = MAGIC_LEN; + control->out_ofs = control->out_len = MAGIC_LEN;\ + return true; } void close_tmpoutbuf(rzip_control *control) @@ -415,13 +494,14 @@ void close_tmpoutbuf(rzip_control *control) free(control->tmp_outbuf); } -static void open_tmpinbuf(rzip_control *control) +static bool open_tmpinbuf(rzip_control *control) { control->flags |= FLAG_TMP_INBUF; control->in_maxlen = control->maxram; control->tmp_inbuf = malloc(control->maxram + control->page_size); if (unlikely(!control->tmp_inbuf)) - fatal("Failed to malloc tmp_inbuf in open_tmpinbuf\n"); + fatal_return(("Failed to malloc tmp_inbuf in open_tmpinbuf\n"), false); + return true; } inline void clear_tmpinbuf(rzip_control *control) @@ -429,12 +509,13 @@ inline void clear_tmpinbuf(rzip_control *control) control->in_len = control->in_ofs = 0; } -void clear_tmpinfile(rzip_control *control) +bool clear_tmpinfile(rzip_control *control) { if (unlikely(lseek(control->fd_in, 0, SEEK_SET))) - fatal("Failed to lseek on fd_in in clear_tmpinfile\n"); + fatal_return(("Failed to lseek on fd_in in clear_tmpinfile\n"), false); if (unlikely(ftruncate(control->fd_in, 0))) - fatal("Failed to truncate fd_in in clear_tmpinfile\n"); + fatal_return(("Failed to truncate fd_in in clear_tmpinfile\n"), false); + return true; } void close_tmpinbuf(rzip_control *control) @@ -443,13 +524,13 @@ void close_tmpinbuf(rzip_control *control) free(control->tmp_inbuf); } -static int get_pass(char *s) +static int get_pass(rzip_control *control, char *s) { int len; memset(s, 0, PASS_LEN - SALT_LEN); if (unlikely(fgets(s, PASS_LEN - SALT_LEN, stdin) == NULL)) - failure("Failed to retrieve passphrase\n"); + failure_return(("Failed to retrieve passphrase\n"), -1); len = strlen(s); if (len > 0 && ('\r' == s[len - 1] || '\n' == s[len - 1])) s[len - 1] = '\0'; @@ -457,11 +538,11 @@ static int get_pass(char *s) s[len - 2] = '\0'; len = strlen(s); if (unlikely(0 == len)) - failure("Empty passphrase\n"); + failure_return(("Empty passphrase\n"), -1); return len; } -static void get_hash(rzip_control *control, int make_hash) +static bool get_hash(rzip_control *control, int make_hash) { char *passphrase, *testphrase; struct termios termios_p; @@ -470,42 +551,60 @@ static void get_hash(rzip_control *control, int make_hash) testphrase = calloc(PASS_LEN, 1); control->salt_pass = calloc(PASS_LEN, 1); control->hash = calloc(HASH_LEN, 1); - if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) + if (unlikely(!passphrase || !testphrase || !control->salt_pass || !control->hash)) { fatal("Failed to calloc encrypt buffers in compress_file\n"); + free(testphrase); + free(passphrase); + return false; + } mlock(passphrase, PASS_LEN); mlock(testphrase, PASS_LEN); mlock(control->salt_pass, PASS_LEN); mlock(control->hash, HASH_LEN); - /* Disable stdin echo to screen */ - tcgetattr(fileno(stdin), &termios_p); - termios_p.c_lflag &= ~ECHO; - tcsetattr(fileno(stdin), 0, &termios_p); + if (control->pass_cb) { + control->pass_cb(control->pass_data, passphrase, PASS_LEN - SALT_LEN); + if (!passphrase[0]) { + fatal("Supplied password was null!"); + munlock(passphrase, PASS_LEN); + munlock(testphrase, PASS_LEN); + free(testphrase); + free(passphrase); + release_hashes(control); + return false; + } + control->salt_pass_len = strlen(passphrase) + SALT_LEN; + } else { + /* Disable stdin echo to screen */ + tcgetattr(fileno(stdin), &termios_p); + termios_p.c_lflag &= ~ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); retry_pass: - print_output("Enter passphrase: "); - control->salt_pass_len = get_pass(passphrase) + SALT_LEN; - print_output("\n"); - if (make_hash) { - print_output("Re-enter passphrase: "); - get_pass(testphrase); + print_output("Enter passphrase: "); + control->salt_pass_len = get_pass(control, passphrase) + SALT_LEN; print_output("\n"); - if (strcmp(passphrase, testphrase)) { - print_output("Passwords do not match. Try again.\n"); - goto retry_pass; + if (make_hash) { + print_output("Re-enter passphrase: "); + get_pass(control, testphrase); + print_output("\n"); + if (strcmp(passphrase, testphrase)) { + print_output("Passwords do not match. Try again.\n"); + goto retry_pass; + } } + termios_p.c_lflag |= ECHO; + tcsetattr(fileno(stdin), 0, &termios_p); + memset(testphrase, 0, PASS_LEN); } - termios_p.c_lflag |= ECHO; - tcsetattr(fileno(stdin), 0, &termios_p); - memset(testphrase, 0, PASS_LEN); - munlock(testphrase, PASS_LEN); - free(testphrase); - memcpy(control->salt_pass, control->salt, SALT_LEN); memcpy(control->salt_pass + SALT_LEN, passphrase, PASS_LEN - SALT_LEN); lrz_stretch(control); memset(passphrase, 0, PASS_LEN); munlock(passphrase, PASS_LEN); + munlock(testphrase, PASS_LEN); + free(testphrase); free(passphrase); + return true; } static void release_hashes(rzip_control *control) @@ -521,7 +620,7 @@ static void release_hashes(rzip_control *control) /* decompress one file from the command line */ -void decompress_file(rzip_control *control) +bool decompress_file(rzip_control *control) { char *tmp, *tmpoutfile, *infilecopy = NULL; int fd_in, fd_out = -1, fd_hist = -1; @@ -537,15 +636,11 @@ void decompress_file(rzip_control *control) /* make sure infile has an extension. If not, add it * because manipulations may be made to input filename, set local ptr */ - infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); - if (unlikely(infilecopy == NULL)) - fatal("Failed to allocate memory for infile suffix\n"); - else { - strcpy(infilecopy, control->infile); - strcat(infilecopy, control->suffix); - } + infilecopy = alloca(strlen(control->infile) + strlen(control->suffix) + 1); + strcpy(infilecopy, control->infile); + strcat(infilecopy, control->suffix); } else - infilecopy = strdup(control->infile); + infilecopy = strdupa(control->infile); /* regardless, infilecopy has the input filename */ } @@ -559,9 +654,9 @@ void decompress_file(rzip_control *control) * infilecopy, then remove suffix. */ if (control->outdir && (tmp = strrchr(infilecopy, '/'))) - tmpoutfile = strdup(tmp + 1); + tmpoutfile = strdupa(tmp + 1); else - tmpoutfile = strdup(infilecopy); + tmpoutfile = strdupa(infilecopy); /* remove suffix to make outfile name */ if ((tmp = strrchr(tmpoutfile, '.')) && !strcmp(tmp, control->suffix)) @@ -569,14 +664,13 @@ void decompress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpoutfile) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal_return(("Failed to allocate outfile name\n"), false); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); strcat(control->outfile, tmpoutfile); } else strcpy(control->outfile, tmpoutfile); - free(tmpoutfile); } if (!STDOUT) @@ -585,15 +679,16 @@ void decompress_file(rzip_control *control) if (STDIN) { fd_in = open_tmpinfile(control); + if (unlikely(fd_in == -1)) return false; read_tmpinmagic(control); if (ENCRYPT) - failure("Cannot decompress encrypted file from STDIN\n"); + failure_return(("Cannot decompress encrypted file from STDIN\n"), false); expected_size = control->st_size; - open_tmpinbuf(control); + if (unlikely(!open_tmpinbuf(control))) return false; } else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) { - fatal("Failed to open %s\n", infilecopy); + fatal_return(("Failed to open %s\n", infilecopy), false); } } control->fd_in = fd_in; @@ -602,50 +697,50 @@ void decompress_file(rzip_control *control) fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink an existing file: %s\n", control->outfile); + fatal_return(("Failed to unlink an existing file: %s\n", control->outfile), false); fd_out = open(control->outfile, O_WRONLY | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal("Failed to create %s\n", control->outfile); + fatal_return(("Failed to create %s\n", control->outfile), false); } fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control->outfile); + fatal_return(("Failed to open history file %s\n", control->outfile), false); /* Can't copy permissions from STDIN */ if (!STDIN) - preserve_perms(control, fd_in, fd_out); + if (unlikely(!preserve_perms(control, fd_in, fd_out))) return false; } else { fd_out = open_tmpoutfile(control); if (unlikely(fd_out == -1)) - fatal("Failed to create %s\n", control->outfile); + fatal_return(("Failed to create %s\n", control->outfile), false); fd_hist = open(control->outfile, O_RDONLY); if (unlikely(fd_hist == -1)) - fatal("Failed to open history file %s\n", control->outfile); + fatal_return(("Failed to open history file %s\n", control->outfile), false); /* Unlink temporary file as soon as possible */ if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink tmpfile: %s\n", control->outfile); + fatal_return(("Failed to unlink tmpfile: %s\n", control->outfile), false); } - open_tmpoutbuf(control); + if (unlikely(!open_tmpoutbuf(control))) return false; if (!STDIN) - read_magic(control, fd_in, &expected_size); + if (unlikely(!read_magic(control, fd_in, &expected_size))) return false; if (!STDOUT) { /* Check if there's enough free space on the device chosen to fit the * decompressed file. */ if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal("Failed to fstatvfs in decompress_file\n"); + fatal_return(("Failed to fstatvfs in decompress_file\n"), false); free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < expected_size) { if (FORCE_REPLACE) print_err("Warning, inadequate free space detected, but attempting to decompress due to -f option being used.\n"); else - failure("Inadequate free space to decompress file, use -f to override.\n"); + failure_return(("Inadequate free space to decompress file, use -f to override.\n"), false); } } control->fd_out = fd_out; @@ -660,16 +755,16 @@ void decompress_file(rzip_control *control) print_verbose("being used for integrity testing.\n"); if (ENCRYPT) - get_hash(control, 0); + if (unlikely(!get_hash(control, 0))) return false; print_progress("Decompressing...\n"); - runzip_fd(control, fd_in, fd_out, fd_hist, expected_size); + if (unlikely(runzip_fd(control, fd_in, fd_out, fd_hist, expected_size) < 0)) return false; if (STDOUT && !TMP_OUTBUF) - dump_tmpoutfile(control, fd_out); + if (unlikely(!dump_tmpoutfile(control, fd_out))) return false; - /* if we get here, no fatal errors during decompression */ + /* if we get here, no fatal_return(( errors during decompression */ print_progress("\r"); if (!(STDOUT | TEST_ONLY)) print_output("Output filename is: %s: ", control->outfile); @@ -681,38 +776,38 @@ void decompress_file(rzip_control *control) print_progress("[OK] \n"); if (unlikely(close(fd_hist) || close(fd_out))) - fatal("Failed to close files\n"); + fatal_return(("Failed to close files\n"), false); close(fd_in); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal("Failed to unlink %s\n", infilecopy); + fatal_return(("Failed to unlink %s\n", infilecopy), false); } if (ENCRYPT) release_hashes(control); free(control->outfile); - free(infilecopy); + return true; } -void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, +bool get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head, int chunk_bytes) { if (unlikely(read(fd_in, ctype, 1) != 1)) - fatal("Failed to read in get_header_info\n"); + fatal_return(("Failed to read in get_header_info\n"), false); *c_len = *u_len = *last_head = 0; if (control->major_version == 0 && control->minor_version < 4) { u32 c_len32, u_len32, last_head32; if (unlikely(read(fd_in, &c_len32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, &u_len32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, &last_head32, 4) != 4)) - fatal("Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); c_len32 = le32toh(c_len32); u_len32 = le32toh(u_len32); last_head32 = le32toh(last_head32); @@ -727,15 +822,16 @@ void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, else read_len = chunk_bytes; if (unlikely(read(fd_in, c_len, read_len) != read_len)) - fatal("Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, u_len, read_len) != read_len)) - fatal("Failed to read in get_header_info"); + fatal_return(("Failed to read in get_header_info"), false); if (unlikely(read(fd_in, last_head, read_len) != read_len)) - fatal("Failed to read_i64 in get_header_info"); + fatal_return(("Failed to read_i64 in get_header_info"), false); *c_len = le64toh(*c_len); *u_len = le64toh(*u_len); *last_head = le64toh(*last_head); } + return true; } static double percentage(i64 num, i64 den) @@ -752,7 +848,7 @@ static double percentage(i64 num, i64 den) return d_num / d_den; } -void get_fileinfo(rzip_control *control) +bool get_fileinfo(rzip_control *control) { i64 u_len, c_len, last_head, utotal = 0, ctotal = 0, ofs = 25, stream_head[2]; i64 expected_size, infile_size, chunk_size = 0, chunk_total = 0; @@ -770,15 +866,11 @@ void get_fileinfo(rzip_control *control) stat(control->infile, &fdin_stat); if (!S_ISREG(fdin_stat.st_mode) && (tmp = strrchr(control->infile, '.')) && strcmp(tmp,control->suffix)) { - infilecopy = malloc(strlen(control->infile) + strlen(control->suffix) + 1); - if (unlikely(infilecopy == NULL)) - fatal("Failed to allocate memory for infile suffix\n"); - else { - strcpy(infilecopy, control->infile); - strcat(infilecopy, control->suffix); - } + infilecopy = alloca(strlen(control->infile) + strlen(control->suffix) + 1); + strcpy(infilecopy, control->infile); + strcat(infilecopy, control->suffix); } else - infilecopy = strdup(control->infile); + infilecopy = strdupa(control->infile); } if (STDIN) @@ -786,30 +878,31 @@ void get_fileinfo(rzip_control *control) else { fd_in = open(infilecopy, O_RDONLY); if (unlikely(fd_in == -1)) - fatal("Failed to open %s\n", infilecopy); + fatal_return(("Failed to open %s\n", infilecopy), false); } /* Get file size */ if (unlikely(fstat(fd_in, &st))) - fatal("bad magic file descriptor!?\n"); + fatal_goto(("bad magic file descriptor!?\n"), error); infile_size = st.st_size; /* Get decompressed size */ - read_magic(control, fd_in, &expected_size); + if (unlikely(!read_magic(control, fd_in, &expected_size))) goto error; if (ENCRYPT) { print_output("Encrypted lrzip archive. No further information available\n"); + if (!STDIN) close(fd_in); goto out; } if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal("Failed to read chunk_byte in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error); if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal("Failed to read eof in get_fileinfo\n"); + fatal_goto(("Failed to read eof in get_fileinfo\n"), error); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal("Failed to read chunk_size in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error); chunk_size = le64toh(chunk_size); } } @@ -846,8 +939,8 @@ void get_fileinfo(rzip_control *control) int block = 1; if (unlikely(lseek(fd_in, stream_head[stream] + ofs, SEEK_SET)) == -1) - fatal("Failed to seek to header data in get_fileinfo\n"); - get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte); + fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error); + if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len, &last_head, chunk_byte))) return false; print_verbose("Stream: %d\n", stream); print_maxverbose("Offset: %lld\n", ofs); @@ -856,13 +949,13 @@ void get_fileinfo(rzip_control *control) i64 head_off; if (unlikely(last_head + ofs > infile_size)) - failure("Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error); if (unlikely(head_off = lseek(fd_in, last_head + ofs, SEEK_SET)) == -1) - fatal("Failed to seek to header data in get_fileinfo\n"); - get_header_info(control, fd_in, &ctype, &c_len, &u_len, - &last_head, chunk_byte); + fatal_goto(("Failed to seek to header data in get_fileinfo\n"), error); + if (unlikely(!get_header_info(control, fd_in, &ctype, &c_len, &u_len, + &last_head, chunk_byte))) return false; if (unlikely(last_head < 0 || c_len < 0 || u_len < 0)) - failure("Entry negative, likely corrupted archive.\n"); + failure_goto(("Entry negative, likely corrupted archive.\n"), error); print_verbose("%d\t", block); if (ctype == CTYPE_NONE) print_verbose("none"); @@ -889,20 +982,20 @@ void get_fileinfo(rzip_control *control) } if (unlikely((ofs = lseek(fd_in, c_len, SEEK_CUR)) == -1)) - fatal("Failed to lseek c_len in get_fileinfo\n"); + fatal_goto(("Failed to lseek c_len in get_fileinfo\n"), error); if (ofs >= infile_size - (HAS_MD5 ? MD5_DIGEST_SIZE : 0)) goto done; /* Chunk byte entry */ if (control->major_version == 0 && control->minor_version > 4) { if (unlikely(read(fd_in, &chunk_byte, 1) != 1)) - fatal("Failed to read chunk_byte in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_byte in get_fileinfo\n"), error); ofs++; if (control->major_version == 0 && control->minor_version > 5) { if (unlikely(read(fd_in, &control->eof, 1) != 1)) - fatal("Failed to read eof in get_fileinfo\n"); + fatal_goto(("Failed to read eof in get_fileinfo\n"), error); if (unlikely(read(fd_in, &chunk_size, chunk_byte) != chunk_byte)) - fatal("Failed to read chunk_size in get_fileinfo\n"); + fatal_goto(("Failed to read chunk_size in get_fileinfo\n"), error); chunk_size = le64toh(chunk_size); ofs += 1 + chunk_byte; header_length = 1 + (chunk_byte * 3); @@ -911,7 +1004,7 @@ void get_fileinfo(rzip_control *control) goto next_chunk; done: if (unlikely(ofs > infile_size)) - failure("Offset greater than archive size, likely corrupted/truncated archive.\n"); + failure_goto(("Offset greater than archive size, likely corrupted/truncated archive.\n"), error); if (chunk_total > expected_size) expected_size = chunk_total; print_verbose("Rzip compression: %.1f%% %lld / %lld\n", @@ -953,9 +1046,9 @@ void get_fileinfo(rzip_control *control) print_output("MD5 used for integrity testing\n"); if (unlikely(lseek(fd_in, -MD5_DIGEST_SIZE, SEEK_END)) == -1) - fatal("Failed to seek to md5 data in runzip_fd\n"); + fatal_goto(("Failed to seek to md5 data in runzip_fd\n"), error); if (unlikely(read(fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to read md5 data in runzip_fd\n"); + fatal_goto(("Failed to read md5 data in runzip_fd\n"), error); print_output("MD5: "); for (i = 0; i < MD5_DIGEST_SIZE; i++) print_output("%02x", md5_stored[i] & 0xFF); @@ -963,40 +1056,43 @@ void get_fileinfo(rzip_control *control) } else print_output("CRC32 used for integrity testing\n"); if (unlikely(close(fd_in))) - fatal("Failed to close fd_in in get_fileinfo\n"); + fatal_return(("Failed to close fd_in in get_fileinfo\n"), false); out: free(control->outfile); - free(infilecopy); + return true; +error: + if (!STDIN) close(fd_in); + return false; } /* compress one file from the command line */ -void compress_file(rzip_control *control) +bool compress_file(rzip_control *control) { const char *tmp, *tmpinfile; /* we're just using this as a proxy for control->infile. * Spares a compiler warning */ - int fd_in, fd_out = -1; + int fd_in = -1, fd_out = -1; char header[MAGIC_LEN]; if (MD5_RELIABLE) control->flags |= FLAG_MD5; if (ENCRYPT) - get_hash(control, 1); + if (unlikely(!get_hash(control, 1))) return false; memset(header, 0, sizeof(header)); if (!STDIN) { /* is extension at end of infile? */ if ((tmp = strrchr(control->infile, '.')) && !strcmp(tmp, control->suffix)) { print_err("%s: already has %s suffix. Skipping...\n", control->infile, control->suffix); - return; + return false; } fd_in = open(control->infile, O_RDONLY); if (unlikely(fd_in == -1)) - fatal("Failed to open %s\n", control->infile); + fatal_return(("Failed to open %s\n", control->infile), false); } else fd_in = 0; @@ -1008,7 +1104,7 @@ void compress_file(rzip_control *control) else if ((tmp=strrchr(control->outname, '.')) && strcmp(tmp, control->suffix)) { control->outfile = malloc(strlen(control->outname) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal_goto(("Failed to allocate outfile name\n"), error); strcpy(control->outfile, control->outname); strcat(control->outfile, control->suffix); print_output("Suffix added to %s.\nFull pathname is: %s\n", control->outname, control->outfile); @@ -1026,7 +1122,7 @@ void compress_file(rzip_control *control) control->outfile = malloc((control->outdir == NULL? 0: strlen(control->outdir)) + strlen(tmpinfile) + strlen(control->suffix) + 1); if (unlikely(!control->outfile)) - fatal("Failed to allocate outfile name\n"); + fatal_goto(("Failed to allocate outfile name\n"), error); if (control->outdir) { /* prepend control->outdir */ strcpy(control->outfile, control->outdir); @@ -1040,45 +1136,96 @@ void compress_file(rzip_control *control) fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); if (FORCE_REPLACE && (-1 == fd_out) && (EEXIST == errno)) { if (unlikely(unlink(control->outfile))) - fatal("Failed to unlink an existing file: %s\n", control->outfile); + fatal_goto(("Failed to unlink an existing file: %s\n", control->outfile), error); fd_out = open(control->outfile, O_RDWR | O_CREAT | O_EXCL, 0666); } if (unlikely(fd_out == -1)) { /* We must ensure we don't delete a file that already * exists just because we tried to create a new one */ control->flags |= FLAG_KEEP_BROKEN; - fatal("Failed to create %s\n", control->outfile); + fatal_goto(("Failed to create %s\n", control->outfile), error); } control->fd_out = fd_out; if (!STDIN) - preserve_perms(control, fd_in, fd_out); - } else - open_tmpoutbuf(control); + if (unlikely(!preserve_perms(control, fd_in, fd_out))) goto error; + } else + if (unlikely(!open_tmpoutbuf(control))) goto error; /* Write zeroes to header at beginning of file */ if (unlikely(!STDOUT && write(fd_out, header, sizeof(header)) != sizeof(header))) - fatal("Cannot write file header\n"); + fatal_goto(("Cannot write file header\n"), error); rzip_fd(control, fd_in, fd_out); /* Wwrite magic at end b/c lzma does not tell us properties until it is done */ if (!STDOUT) - write_magic(control); + if (unlikely(!write_magic(control))) goto error; if (ENCRYPT) release_hashes(control); - if (unlikely(close(fd_in))) + if (unlikely(close(fd_in))) { fatal("Failed to close fd_in\n"); + fd_in = -1; + goto error; + } if (unlikely(!STDOUT && close(fd_out))) - fatal("Failed to close fd_out\n"); + fatal_return(("Failed to close fd_out\n"), false); if (TMP_OUTBUF) close_tmpoutbuf(control); if (!KEEP_FILES) { if (unlikely(unlink(control->infile))) - fatal("Failed to unlink %s\n", control->infile); + fatal_return(("Failed to unlink %s\n", control->infile), false); } free(control->outfile); + return true; +error: + if (STDIN && (fd_in > 0)) close(fd_in); + if ((!STDOUT) && (fd_out > 0)) close(fd_out); + return false; +} + +bool initialize_control(rzip_control *control) +{ + struct timeval tv; + char *eptr; /* for environment */ + + memset(control, 0, sizeof(rzip_control)); + control->msgout = stderr; + control->msgerr = stderr; + register_outputfile(control, control->msgout); + control->flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; + control->suffix = strdup(".lrz"); + control->compression_level = 7; + control->ramsize = get_ram(control); + if (unlikely(control->ramsize == -1)) return false; + /* for testing single CPU */ + control->threads = PROCESSORS; /* get CPUs for LZMA */ + control->page_size = PAGE_SIZE; + control->nice_val = 19; + + /* The first 5 bytes of the salt is the time in seconds. + * The next 2 bytes encode how many times to hash the password. + * The last 9 bytes are random data, making 16 bytes of salt */ + if (unlikely(gettimeofday(&tv, NULL))) + fatal_return(("Failed to gettimeofday in main\n"), false); + control->secs = tv.tv_sec; + control->encloops = nloops(control->secs, control->salt, control->salt + 1); + if (unlikely(!get_rand(control, control->salt + 2, 6))) return false; + + /* Get Temp Dir */ + eptr = getenv("TMP"); + if (eptr != NULL) { + size_t len = strlen(eptr); + control->tmpdir = malloc(len+2); + if (control->tmpdir == NULL) + fatal_return(("Failed to allocate for tmpdir\n"), false); + strcpy(control->tmpdir, eptr); + if (eptr[len - 2] != '/') + eptr[len - 2] = '/'; /* need a trailing slash */ + eptr[len - 1] = 0; + } + return true; } diff --git a/lrzip.h b/lrzip.h index 6b54c99c..d7937761 100644 --- a/lrzip.h +++ b/lrzip.h @@ -21,22 +21,25 @@ #include "lrzip_private.h" -void write_magic(rzip_control *control); -void read_magic(rzip_control *control, int fd_in, i64 *expected_size); -void preserve_perms(rzip_control *control, int fd_in, int fd_out); +inline i64 get_ram(rzip_control *control); +i64 nloops(i64 seconds, uchar *b1, uchar *b2); +bool write_magic(rzip_control *control); +bool read_magic(rzip_control *control, int fd_in, i64 *expected_size); +bool preserve_perms(rzip_control *control, int fd_in, int fd_out); int open_tmpoutfile(rzip_control *control); -void dump_tmpoutfile(rzip_control *control, int fd_out); +bool dump_tmpoutfile(rzip_control *control, int fd_out); int open_tmpinfile(rzip_control *control); -void read_tmpinfile(rzip_control *control, int fd_in); -void decompress_file(rzip_control *control); -void get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head); -void get_fileinfo(rzip_control *control); -void compress_file(rzip_control *control); -void write_fdout(rzip_control *control, void *buf, i64 len); -void write_fdin(rzip_control *control); -void flush_tmpoutbuf(rzip_control *control); +bool read_tmpinfile(rzip_control *control, int fd_in); +bool decompress_file(rzip_control *control); +bool get_header_info(rzip_control *control, int fd_in, uchar *ctype, i64 *c_len, i64 *u_len, i64 *last_head); +bool get_fileinfo(rzip_control *control); +bool compress_file(rzip_control *control); +bool write_fdout(rzip_control *control, void *buf, i64 len); +bool write_fdin(rzip_control *control); +bool flush_tmpoutbuf(rzip_control *control); void close_tmpoutbuf(rzip_control *control); void clear_tmpinbuf(rzip_control *control); -inline void clear_tmpinfile(rzip_control *control); +inline bool clear_tmpinfile(rzip_control *control); void close_tmpinbuf(rzip_control *control); +bool initialize_control(rzip_control *control); #endif diff --git a/lrzip_private.h b/lrzip_private.h index 6e3795b3..7eae4762 100644 --- a/lrzip_private.h +++ b/lrzip_private.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include #ifdef HAVE_STRING_H # include #endif @@ -34,6 +36,38 @@ # include #endif +#ifdef HAVE_ALLOCA_H +# include +#elif defined __GNUC__ +# define alloca __builtin_alloca +#elif defined _AIX +# define alloca __alloca +#elif defined _MSC_VER +# include +# define alloca _alloca +#else +# include +# ifdef __cplusplus +extern "C" +# endif +void *alloca (size_t); +#endif + +#ifndef MD5_DIGEST_SIZE +# define MD5_DIGEST_SIZE 16 +#endif + +#define free(X) do { free((X)); (X) = NULL; } while (0) + +#ifndef strdupa +# define strdupa(str) strcpy(alloca(strlen(str) + 1), str) +#endif + +#ifndef strndupa +# define strndupa(str, len) strncpy(alloca(len + 1), str, len) +#endif + + #ifndef uchar #define uchar unsigned char #endif @@ -176,9 +210,61 @@ typedef struct md5_ctx md5_ctx; #define SALT_LEN 8 #define CBC_LEN 16 -#define print_err(format, args...) do {\ - fprintf(stderr, format, ##args); \ -} while (0) +#define one_g (1000 * 1024 * 1024) + +#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN) +# define PROCESSORS (1) +#else +# define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN)) +#endif + +#ifdef _SC_PAGE_SIZE +# define PAGE_SIZE (sysconf(_SC_PAGE_SIZE)) +#else +# define PAGE_SIZE (4096) +#endif + +/* Determine how many times to hash the password when encrypting, based on + * the date such that we increase the number of loops according to Moore's + * law relative to when the data is encrypted. It is then stored as a two + * byte value in the header */ +#define MOORE 1.835 // world constant [TIMES per YEAR] +#define ARBITRARY 1000000 // number of sha2 calls per one second in 2011 +#define T_ZERO 1293840000 // seconds since epoch in 2011 + +#define SECONDS_IN_A_YEAR (365*86400) +#define MOORE_TIMES_PER_SECOND pow (MOORE, 1.0 / SECONDS_IN_A_YEAR) +#define ARBITRARY_AT_EPOCH (ARBITRARY * pow (MOORE_TIMES_PER_SECOND, -T_ZERO)) + +#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) +#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) +#define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA)) + +#define SHOW_PROGRESS (control->flags & FLAG_SHOW_PROGRESS) +#define KEEP_FILES (control->flags & FLAG_KEEP_FILES) +#define TEST_ONLY (control->flags & FLAG_TEST_ONLY) +#define FORCE_REPLACE (control->flags & FLAG_FORCE_REPLACE) +#define DECOMPRESS (control->flags & FLAG_DECOMPRESS) +#define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS) +#define LZO_COMPRESS (control->flags & FLAG_LZO_COMPRESS) +#define BZIP2_COMPRESS (control->flags & FLAG_BZIP2_COMPRESS) +#define ZLIB_COMPRESS (control->flags & FLAG_ZLIB_COMPRESS) +#define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS) +#define VERBOSE (control->flags & FLAG_VERBOSE) +#define VERBOSITY (control->flags & FLAG_VERBOSITY) +#define MAX_VERBOSE (control->flags & FLAG_VERBOSITY_MAX) +#define STDIN (control->flags & FLAG_STDIN) +#define STDOUT (control->flags & FLAG_STDOUT) +#define INFO (control->flags & FLAG_INFO) +#define UNLIMITED (control->flags & FLAG_UNLIMITED) +#define HASH_CHECK (control->flags & FLAG_HASH) +#define HAS_MD5 (control->flags & FLAG_MD5) +#define CHECK_FILE (control->flags & FLAG_CHECK) +#define KEEP_BROKEN (control->flags & FLAG_KEEP_BROKEN) +#define LZO_TEST (control->flags & FLAG_THRESHOLD) +#define TMP_OUTBUF (control->flags & FLAG_TMP_OUTBUF) +#define TMP_INBUF (control->flags & FLAG_TMP_INBUF) +#define ENCRYPT (control->flags & FLAG_ENCRYPT) /* Structure to save state of computation between the single steps. */ @@ -196,8 +282,10 @@ struct md5_ctx struct rzip_control { char *infile; + FILE *inFILE; // if a FILE is being read from char *outname; char *outfile; + FILE *outFILE; // if a FILE is being written to char *outdir; char *tmpdir; // when stdin, stdout, or test used uchar *tmp_outbuf; // Temporary file storage for stdout @@ -211,7 +299,8 @@ struct rzip_control { i64 in_len; i64 in_maxlen; FILE *msgout; //stream for output messages - const char *suffix; + FILE *msgerr; //stream for output errors + char *suffix; uchar compression_level; i64 overhead; // compressor overhead i64 usable_ram; // the most ram we'll try to use on one activity @@ -233,6 +322,8 @@ struct rzip_control { int fd_hist; i64 encloops; i64 secs; + void (*pass_cb)(void *, char *, size_t); /* callback to get password in lib */ + void *pass_data; uchar salt[SALT_LEN]; uchar *salt_pass; int salt_pass_len; @@ -240,7 +331,23 @@ struct rzip_control { unsigned char eof; unsigned char magic_written; md5_ctx ctx; + uchar md5_resblock[MD5_DIGEST_SIZE]; i64 md5_read; // How far into the file the md5 has done so far + const char *util_infile; + char delete_infile; + const char *util_outfile; +#define STREAM_BUCKET_SIZE 20 + size_t sinfo_buckets; + size_t sinfo_idx; + struct stream_info **sinfo_queue; + char delete_outfile; + FILE *outputfile; + char library_mode : 1; + int log_level; + void (*info_cb)(void *data, int pct, int chunk_pct); + void *info_data; + void (*log_cb)(void *data, unsigned int level, unsigned int line, const char *file, const char *func, const char *format, va_list); + void *log_data; }; struct stream { @@ -271,4 +378,60 @@ struct stream_info { int chunks; char chunk_bytes; }; + +static inline void print_stuff(const rzip_control *control, int level, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + if (control->library_mode && control->log_cb && (control->log_level >= level)) { + va_start(ap, format); + control->log_cb(control->log_data, level, line, file, func, format, ap); + va_end(ap); + } else if (control->msgout) { + va_start(ap, format); + vfprintf(control->msgout, format, ap); + fflush(control->msgout); + va_end(ap); + } +} + +static inline void print_err(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + if (control->library_mode && control->log_cb && (control->log_level >= 0)) { + va_start(ap, format); + control->log_cb(control->log_data, 0, line, file, func, format, ap); + va_end(ap); + } else if (control->msgerr) { + va_start(ap, format); + vfprintf(control->msgerr, format, ap); + va_end(ap); + } +} + +#define print_stuff(level, format, args...) do {\ + print_stuff(control, level, __LINE__, __FILE__, __func__, format, ##args); \ +} while (0) + +#define print_output(format, args...) do {\ + print_stuff(1, format, ##args); \ +} while (0) + +#define print_progress(format, args...) do {\ + if (SHOW_PROGRESS) \ + print_stuff(2, format, ##args); \ +} while (0) + +#define print_verbose(format, args...) do {\ + if (VERBOSE) \ + print_stuff(3, format, ##args); \ +} while (0) + +#define print_maxverbose(format, args...) do {\ + if (MAX_VERBOSE) \ + print_stuff(4, format, ##args); \ +} while (0) + +#define print_err(format, args...) do {\ + print_err(control, __LINE__, __FILE__, __func__, format, ##args); \ +} while (0) #endif diff --git a/main.c b/main.c index ae99f13d..bae4bb61 100644 --- a/main.c +++ b/main.c @@ -26,9 +26,6 @@ #ifdef HAVE_UNISTD_H # include #endif -#ifdef HAVE_CTYPE_H -# include -#endif #ifdef HAVE_SYS_TIME_H # include #endif @@ -41,7 +38,7 @@ #ifdef HAVE_SYS_STAT_H # include #endif -#include + #include #ifdef HAVE_ENDIAN_H # include @@ -60,119 +57,7 @@ /* needed for CRC routines */ #include "lzma/C/7zCrc.h" -/* Macros for testing parameters */ -#define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) -#define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) - -/* main() defines, different from liblrzip defines */ -#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX) -#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS) -#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA)) - -#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS) -#define KEEP_FILES (control.flags & FLAG_KEEP_FILES) -#define TEST_ONLY (control.flags & FLAG_TEST_ONLY) -#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE) -#define DECOMPRESS (control.flags & FLAG_DECOMPRESS) -#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS) -#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS) -#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS) -#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS) -#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS) -#define VERBOSE (control.flags & FLAG_VERBOSE) -#define VERBOSITY (control.flags & FLAG_VERBOSITY) -#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX) -#define STDIN (control.flags & FLAG_STDIN) -#define STDOUT (control.flags & FLAG_STDOUT) -#define INFO (control.flags & FLAG_INFO) -#define UNLIMITED (control.flags & FLAG_UNLIMITED) -#define HASH_CHECK (control.flags & FLAG_HASH) -#define HAS_MD5 (control.flags & FLAG_MD5) -#define CHECK_FILE (control.flags & FLAG_CHECK) -#define KEEP_BROKEN (control.flags & FLAG_KEEP_BROKEN) -#define LZO_TEST (control.flags & FLAG_THRESHOLD) -#define TMP_OUTBUF (control.flags & FLAG_TMP_OUTBUF) -#define TMP_INBUF (control.flags & FLAG_TMP_INBUF) -#define ENCRYPT (control.flags & FLAG_ENCRYPT) - -#define print_output(format, args...) do {\ - fprintf(control.msgout, format, ##args); \ - fflush(control.msgout); \ -} while (0) - -#define print_progress(format, args...) do {\ - if (SHOW_PROGRESS) \ - print_output(format, ##args); \ -} while (0) - -#define print_verbose(format, args...) do {\ - if (VERBOSE) \ - print_output(format, ##args); \ -} while (0) - -#define print_maxverbose(format, args...) do {\ - if (MAX_VERBOSE) \ - print_output(format, ##args); \ -} while (0) - - -#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN) -# define PROCESSORS (1) -#else -# define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN)) -#endif - -#ifdef _SC_PAGE_SIZE -# define PAGE_SIZE (sysconf(_SC_PAGE_SIZE)) -#else -# define PAGE_SIZE (4096) -#endif - -#ifdef __APPLE__ -# include -static inline i64 get_ram(void) -{ - int mib[2]; - size_t len; - i64 *p, ramsize; - - mib[0] = CTL_HW; - mib[1] = HW_MEMSIZE; - sysctl(mib, 2, NULL, &len, NULL, 0); - p = malloc(len); - sysctl(mib, 2, p, &len, NULL, 0); - ramsize = *p; - - return ramsize; -} -#else /* __APPLE__ */ -static inline i64 get_ram(void) -{ - i64 ramsize; - FILE *meminfo; - char aux[256]; - - ramsize = (i64)sysconf(_SC_PHYS_PAGES) * PAGE_SIZE; - if (ramsize > 0) - return ramsize; - - /* Workaround for uclibc which doesn't properly support sysconf */ - if(!(meminfo = fopen("/proc/meminfo", "r"))) - fatal("fopen\n"); - - while(!feof(meminfo) && !fscanf(meminfo, "MemTotal: %Lu kB", &ramsize)) { - if (unlikely(fgets(aux, sizeof(aux), meminfo) == NULL)) - fatal("Failed to fgets in get_ram\n"); - } - if (fclose(meminfo) == -1) - fatal("fclose"); - ramsize *= 1000; - - return ramsize; -} -#endif - -static rzip_control control; +static rzip_control controlstaticvariablehaha, *control; static void usage(void) { @@ -229,7 +114,7 @@ static void sighandler(int sig __UNUSED__) termios_p.c_lflag |= ECHO; tcsetattr(fileno(stdin), 0, &termios_p); - unlink_files(); + unlink_files(control); exit(0); } @@ -240,11 +125,11 @@ static void show_summary(void) if (!TEST_ONLY) print_verbose("The following options are in effect for this %s.\n", DECOMPRESS ? "DECOMPRESSION" : "COMPRESSION"); - print_verbose("Threading is %s. Number of CPUs detected: %d\n", control.threads > 1? "ENABLED" : "DISABLED", - control.threads); - print_verbose("Detected %lld bytes ram\n", control.ramsize); - print_verbose("Compression level %d\n", control.compression_level); - print_verbose("Nice Value: %d\n", control.nice_val); + print_verbose("Threading is %s. Number of CPUs detected: %d\n", control->threads > 1? "ENABLED" : "DISABLED", + control->threads); + print_verbose("Detected %lld bytes ram\n", control->ramsize); + print_verbose("Compression level %d\n", control->compression_level); + print_verbose("Nice Value: %d\n", control->nice_val); print_verbose("Show Progress\n"); print_maxverbose("Max "); print_verbose("Verbose\n"); @@ -252,14 +137,14 @@ static void show_summary(void) print_verbose("Overwrite Files\n"); if (!KEEP_FILES) print_verbose("Remove input files on completion\n"); - if (control.outdir) - print_verbose("Output Directory Specified: %s\n", control.outdir); - else if (control.outname) - print_verbose("Output Filename Specified: %s\n", control.outname); + if (control->outdir) + print_verbose("Output Directory Specified: %s\n", control->outdir); + else if (control->outname) + print_verbose("Output Filename Specified: %s\n", control->outname); if (TEST_ONLY) print_verbose("Test file integrity\n"); - if (control.tmpdir) - print_verbose("Temporary Directory set as: %s\n", control.tmpdir); + if (control->tmpdir) + print_verbose("Temporary Directory set as: %s\n", control->tmpdir); /* show compression options */ if (!DECOMPRESS && !TEST_ONLY) { @@ -276,16 +161,16 @@ static void show_summary(void) print_verbose("ZPAQ. LZO Compressibility testing %s\n", (LZO_TEST? "enabled" : "disabled")); else if (NO_COMPRESS) print_verbose("RZIP pre-processing only\n"); - if (control.window) - print_verbose("Compression Window: %lld = %lldMB\n", control.window, control.window * 100ull); + if (control->window) + print_verbose("Compression Window: %lld = %lldMB\n", control->window, control->window * 100ull); /* show heuristically computed window size */ - if (!control.window && !UNLIMITED) { + if (!control->window && !UNLIMITED) { i64 temp_chunk, temp_window; if (STDOUT || STDIN) - temp_chunk = control.maxram; + temp_chunk = control->maxram; else - temp_chunk = control.ramsize * 2 / 3; + temp_chunk = control->ramsize * 2 / 3; temp_window = temp_chunk / (100 * 1024 * 1024); print_verbose("Heuristically Computed Compression Window: %lld = %lldMB\n", temp_window, temp_window * 100ull); } @@ -293,200 +178,15 @@ static void show_summary(void) print_verbose("Using Unlimited Window size\n"); } if (!DECOMPRESS && !TEST_ONLY) - print_maxverbose("Storage time in seconds %lld\n", control.secs); + print_maxverbose("Storage time in seconds %lld\n", control->secs); if (ENCRYPT) - print_maxverbose("Encryption hash loops %lld\n", control.encloops); + print_maxverbose("Encryption hash loops %lld\n", control->encloops); } } -static void read_config(rzip_control *control) -{ - /* check for lrzip.conf in ., $HOME/.lrzip and /etc/lrzip */ - char *HOME, *homeconf; - char *parametervalue; - char *parameter; - char *line; - FILE *fp; - - line = malloc(255); - homeconf = malloc(255); - if (line == NULL || homeconf == NULL) - fatal("Fatal Memory Error in read_config"); - - fp = fopen("lrzip.conf", "r"); - if (fp) - fprintf(control->msgout, "Using configuration file ./lrzip.conf\n"); - if (fp == NULL) { - fp = fopen("/etc/lrzip/lrzip.conf", "r"); - if (fp) - fprintf(control->msgout, "Using configuration file /etc/lrzip/lrzip.conf\n"); - } - if (fp == NULL) { - HOME=getenv("HOME"); - if (HOME) { - strcpy(homeconf, HOME); - strcat(homeconf,"/.lrzip/lrzip.conf"); - fp = fopen(homeconf, "r"); - if (fp) - fprintf(control->msgout, "Using configuration file %s\n", homeconf); - } - } - if (fp == NULL) - goto out; - - /* if we get here, we have a file. read until no more. */ - - while ((fgets(line, 255, fp)) != NULL) { - if (strlen(line)) - line[strlen(line) - 1] = '\0'; - parameter = strtok(line, " ="); - if (parameter == NULL) - continue; - /* skip if whitespace or # */ - if (isspace(*parameter)) - continue; - if (*parameter == '#') - continue; - - parametervalue = strtok(NULL, " ="); - if (parametervalue == NULL) - continue; - - /* have valid parameter line, now assign to control */ - - if (isparameter(parameter, "window")) - control->window = atoi(parametervalue); - else if (isparameter(parameter, "unlimited")) { - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_UNLIMITED; - } else if (isparameter(parameter, "compressionlevel")) { - control->compression_level = atoi(parametervalue); - if ( control->compression_level < 1 || control->compression_level > 9 ) - failure("CONF.FILE error. Compression Level must between 1 and 9"); - } else if (isparameter(parameter, "compressionmethod")) { - /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ - if (control->flags & FLAG_NOT_LZMA) - failure("CONF.FILE error. Can only specify one compression method"); - if (isparameter(parametervalue, "bzip2")) - control->flags |= FLAG_BZIP2_COMPRESS; - else if (isparameter(parametervalue, "gzip")) - control->flags |= FLAG_ZLIB_COMPRESS; - else if (isparameter(parametervalue, "lzo")) - control->flags |= FLAG_LZO_COMPRESS; - else if (isparameter(parametervalue, "rzip")) - control->flags |= FLAG_NO_COMPRESS; - else if (isparameter(parametervalue, "zpaq")) - control->flags |= FLAG_ZPAQ_COMPRESS; - else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ - failure("CONF.FILE error. Invalid compression method %s specified\n",parametervalue); - } else if (isparameter(parameter, "lzotest")) { - /* default is yes */ - if (isparameter(parametervalue, "no")) - control->flags &= ~FLAG_THRESHOLD; - } else if (isparameter(parameter, "hashcheck")) { - if (isparameter(parametervalue, "yes")) { - control->flags |= FLAG_CHECK; - control->flags |= FLAG_HASH; - } - } else if (isparameter(parameter, "showhash")) { - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_HASH; - } else if (isparameter(parameter, "outputdirectory")) { - control->outdir = malloc(strlen(parametervalue) + 2); - if (!control->outdir) - fatal("Fatal Memory Error in read_config"); - strcpy(control->outdir, parametervalue); - if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) - strcat(control->outdir, "/"); - } else if (isparameter(parameter,"verbosity")) { - if (control->flags & FLAG_VERBOSE) - failure("CONF.FILE error. Verbosity already defined."); - if (isparameter(parametervalue, "yes")) - control->flags |= FLAG_VERBOSITY; - else if (isparameter(parametervalue,"max")) - control->flags |= FLAG_VERBOSITY_MAX; - else /* oops, unrecognized value */ - print_err("lrzip.conf: Unrecognized verbosity value %s. Ignored.\n", parametervalue); - } else if (isparameter(parameter, "showprogress")) { - /* Yes by default */ - if (isparameter(parametervalue, "NO")) - control->flags &= ~FLAG_SHOW_PROGRESS; - } else if (isparameter(parameter,"nice")) { - control->nice_val = atoi(parametervalue); - if (control->nice_val < -20 || control->nice_val > 19) - failure("CONF.FILE error. Nice must be between -20 and 19"); - } else if (isparameter(parameter, "keepbroken")) { - if (isparameter(parametervalue, "yes" )) - control->flags |= FLAG_KEEP_BROKEN; - } else if (iscaseparameter(parameter, "DELETEFILES")) { - /* delete files must be case sensitive */ - if (iscaseparameter(parametervalue, "YES")) - control->flags &= ~FLAG_KEEP_FILES; - } else if (iscaseparameter(parameter, "REPLACEFILE")) { - /* replace lrzip file must be case sensitive */ - if (iscaseparameter(parametervalue, "YES")) - control->flags |= FLAG_FORCE_REPLACE; - } else if (isparameter(parameter, "tmpdir")) { - control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); - if (!control->tmpdir) - fatal("Fatal Memory Error in read_config"); - strcpy(control->tmpdir, parametervalue); - if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) - strcat(control->tmpdir, "/"); - } else if (isparameter(parameter, "encrypt")) { - if (isparameter(parameter, "YES")) - control->flags |= FLAG_ENCRYPT; - } else - /* oops, we have an invalid parameter, display */ - print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ - parameter, parametervalue); - } - - if (unlikely(fclose(fp))) - fatal("Failed to fclose fp in read_config\n"); -out: - /* clean up */ - free(line); - free(homeconf); - -/* fprintf(stderr, "\nWindow = %d \ - \nCompression Level = %d \ - \nThreshold = %1.2f \ - \nOutput Directory = %s \ - \nFlags = %d\n", control->window,control->compression_level, control->threshold, control->outdir, control->flags); -*/ -} - -/* Determine how many times to hash the password when encrypting, based on - * the date such that we increase the number of loops according to Moore's - * law relative to when the data is encrypted. It is then stored as a two - * byte value in the header */ -#define MOORE 1.835 // world constant [TIMES per YEAR] -#define ARBITRARY 1000000 // number of sha2 calls per one second in 2011 -#define T_ZERO 1293840000 // seconds since epoch in 2011 - -#define SECONDS_IN_A_YEAR (365*86400) -#define MOORE_TIMES_PER_SECOND pow (MOORE, 1.0 / SECONDS_IN_A_YEAR) -#define ARBITRARY_AT_EPOCH (ARBITRARY * pow (MOORE_TIMES_PER_SECOND, -T_ZERO)) - -static i64 nloops(i64 seconds, uchar *b1, uchar *b2) -{ - i64 nloops; - int nbits; - - nloops = ARBITRARY_AT_EPOCH * pow(MOORE_TIMES_PER_SECOND, seconds); - if (nloops < ARBITRARY) - nloops = ARBITRARY; - for (nbits = 0; nloops > 255; nbits ++) - nloops = nloops >> 1; - *b1 = nbits; - *b2 = nloops; - return nloops << nbits; -} - int main(int argc, char *argv[]) { - struct timeval start_time, end_time, tv; + struct timeval start_time, end_time; struct sigaction handler; double seconds,total_time; // for timers int c, i; @@ -494,176 +194,146 @@ int main(int argc, char *argv[]) extern int optind; char *eptr; /* for environment */ - memset(&control, 0, sizeof(control)); + control = &controlstaticvariablehaha; - control.msgout = stderr; - register_outputfile(control.msgout); - control.flags = FLAG_SHOW_PROGRESS | FLAG_KEEP_FILES | FLAG_THRESHOLD; - control.suffix = ".lrz"; - control.outdir = NULL; - control.tmpdir = NULL; + initialize_control(control); if (strstr(argv[0], "lrunzip")) - control.flags |= FLAG_DECOMPRESS; + control->flags |= FLAG_DECOMPRESS; else if (strstr(argv[0], "lrzcat")) - control.flags |= FLAG_DECOMPRESS | FLAG_STDOUT; - - control.compression_level = 7; - control.ramsize = get_ram(); - /* for testing single CPU */ - control.threads = PROCESSORS; /* get CPUs for LZMA */ - control.page_size = PAGE_SIZE; - control.nice_val = 19; - - /* The first 5 bytes of the salt is the time in seconds. - * The next 2 bytes encode how many times to hash the password. - * The last 9 bytes are random data, making 16 bytes of salt */ - if (unlikely(gettimeofday(&tv, NULL))) - fatal("Failed to gettimeofday in main\n"); - control.secs = tv.tv_sec; - control.encloops = nloops(control.secs, control.salt, control.salt + 1); - get_rand(control.salt + 2, 6); + control->flags |= FLAG_DECOMPRESS | FLAG_STDOUT; /* generate crc table */ CrcGenerateTable(); - /* Get Temp Dir */ - eptr = getenv("TMP"); - if (eptr != NULL) { - control.tmpdir = malloc(strlen(eptr)+2); - if (control.tmpdir == NULL) - fatal("Failed to allocate for tmpdir\n"); - strcpy(control.tmpdir, eptr); - if (strcmp(eptr+strlen(eptr) - 1, "/")) /* need a trailing slash */ - strcat(control.tmpdir, "/"); - } - /* Get Preloaded Defaults from lrzip.conf * Look in ., $HOME/.lrzip/, /etc/lrzip. * If LRZIP=NOCONFIG is set, then ignore config */ eptr = getenv("LRZIP"); if (eptr == NULL) - read_config(&control); + read_config(control); else if (!strstr(eptr,"NOCONFIG")) - read_config(&control); + read_config(control); while ((c = getopt(argc, argv, "bcdDefghHiklL:nN:o:O:p:qS:tTUvVw:z?")) != -1) { switch (c) { case 'b': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_BZIP2_COMPRESS; + control->flags |= FLAG_BZIP2_COMPRESS; break; case 'c': - control.flags |= FLAG_CHECK; - control.flags |= FLAG_HASH; + control->flags |= FLAG_CHECK; + control->flags |= FLAG_HASH; break; case 'd': - control.flags |= FLAG_DECOMPRESS; + control->flags |= FLAG_DECOMPRESS; break; case 'D': - control.flags &= ~FLAG_KEEP_FILES; + control->flags &= ~FLAG_KEEP_FILES; break; case 'e': - control.flags |= FLAG_ENCRYPT; + control->flags |= FLAG_ENCRYPT; break; case 'f': - control.flags |= FLAG_FORCE_REPLACE; + control->flags |= FLAG_FORCE_REPLACE; break; case 'g': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_ZLIB_COMPRESS; + control->flags |= FLAG_ZLIB_COMPRESS; break; case 'h': case '?': usage(); return -1; case 'H': - control.flags |= FLAG_HASH; + control->flags |= FLAG_HASH; break; case 'i': - control.flags |= FLAG_INFO; + control->flags |= FLAG_INFO; break; case 'k': - control.flags |= FLAG_KEEP_BROKEN; + control->flags |= FLAG_KEEP_BROKEN; break; case 'l': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_LZO_COMPRESS; + control->flags |= FLAG_LZO_COMPRESS; break; case 'L': - control.compression_level = atoi(optarg); - if (control.compression_level < 1 || control.compression_level > 9) + control->compression_level = atoi(optarg); + if (control->compression_level < 1 || control->compression_level > 9) failure("Invalid compression level (must be 1-9)\n"); break; case 'n': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_NO_COMPRESS; + control->flags |= FLAG_NO_COMPRESS; break; case 'N': - control.nice_val = atoi(optarg); - if (control.nice_val < -20 || control.nice_val > 19) + control->nice_val = atoi(optarg); + if (control->nice_val < -20 || control->nice_val > 19) failure("Invalid nice value (must be -20..19)\n"); break; case 'o': - if (control.outdir) + if (control->outdir) failure("Cannot have -o and -O together\n"); if (unlikely(STDOUT)) failure("Cannot specify an output filename when outputting to stdout\n"); - control.outname = optarg; - control.suffix = ""; + control->outname = optarg; + free(control->suffix); + control->suffix = ""; break; case 'O': - if (control.outname) /* can't mix -o and -O */ + if (control->outname) /* can't mix -o and -O */ failure("Cannot have options -o and -O together\n"); if (unlikely(STDOUT)) failure("Cannot specify an output directory when outputting to stdout\n"); - control.outdir = malloc(strlen(optarg) + 2); - if (control.outdir == NULL) + control->outdir = malloc(strlen(optarg) + 2); + if (control->outdir == NULL) fatal("Failed to allocate for outdir\n"); - strcpy(control.outdir,optarg); + strcpy(control->outdir,optarg); if (strcmp(optarg+strlen(optarg) - 1, "/")) /* need a trailing slash */ - strcat(control.outdir, "/"); + strcat(control->outdir, "/"); break; case 'p': - control.threads = atoi(optarg); - if (control.threads < 1) + control->threads = atoi(optarg); + if (control->threads < 1) failure("Must have at least one thread\n"); break; case 'q': - control.flags &= ~FLAG_SHOW_PROGRESS; + control->flags &= ~FLAG_SHOW_PROGRESS; break; case 'S': - if (control.outname) + if (control->outname) failure("Specified output filename already, can't specify an extension.\n"); if (unlikely(STDOUT)) failure("Cannot specify a filename suffix when outputting to stdout\n"); - control.suffix = optarg; + free(control->suffix); + control->suffix = optarg; break; case 't': - if (control.outname) + if (control->outname) failure("Cannot specify an output file name when just testing.\n"); if (!KEEP_FILES) failure("Doubt that you want to delete a file when just testing.\n"); - control.flags |= FLAG_TEST_ONLY; + control->flags |= FLAG_TEST_ONLY; break; case 'T': - control.flags &= ~FLAG_THRESHOLD; + control->flags &= ~FLAG_THRESHOLD; break; case 'U': - control.flags |= FLAG_UNLIMITED; + control->flags |= FLAG_UNLIMITED; break; case 'v': /* set verbosity flag */ - if (!(control.flags & FLAG_VERBOSITY) && !(control.flags & FLAG_VERBOSITY_MAX)) - control.flags |= FLAG_VERBOSITY; - else if ((control.flags & FLAG_VERBOSITY)) { - control.flags &= ~FLAG_VERBOSITY; - control.flags |= FLAG_VERBOSITY_MAX; + if (!(control->flags & FLAG_VERBOSITY) && !(control->flags & FLAG_VERBOSITY_MAX)) + control->flags |= FLAG_VERBOSITY; + else if ((control->flags & FLAG_VERBOSITY)) { + control->flags &= ~FLAG_VERBOSITY; + control->flags |= FLAG_VERBOSITY_MAX; } break; case 'V': @@ -671,12 +341,12 @@ int main(int argc, char *argv[]) exit(0); break; case 'w': - control.window = atol(optarg); + control->window = atol(optarg); break; case 'z': - if (control.flags & FLAG_NOT_LZMA) + if (control->flags & FLAG_NOT_LZMA) failure("Can only use one of -l, -b, -g, -z or -n\n"); - control.flags |= FLAG_ZPAQ_COMPRESS; + control->flags |= FLAG_ZPAQ_COMPRESS; break; } } @@ -684,61 +354,52 @@ int main(int argc, char *argv[]) argc -= optind; argv += optind; - if (control.outname && argc > 1) + if (control->outname && argc > 1) failure("Cannot specify output filename with more than 1 file\n"); if (VERBOSE && !SHOW_PROGRESS) { print_err("Cannot have -v and -q options. -v wins.\n"); - control.flags |= FLAG_SHOW_PROGRESS; + control->flags |= FLAG_SHOW_PROGRESS; } - if (UNLIMITED && control.window) { + if (UNLIMITED && control->window) { print_err("If -U used, cannot specify a window size with -w.\n"); - control.window = 0; + control->window = 0; } if (argc < 1) - control.flags |= FLAG_STDIN; + control->flags |= FLAG_STDIN; if (UNLIMITED && STDIN) { print_err("Cannot have -U and stdin, unlimited mode disabled.\n"); - control.flags &= ~FLAG_UNLIMITED; + control->flags &= ~FLAG_UNLIMITED; } - /* Work out the compression overhead per compression thread for the - * compression back-ends that need a lot of ram */ - if (LZMA_COMPRESS) { - int level = control.compression_level * 7 / 9 ? : 1; - i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) : - (level == 6 ? (1 << 25) : (1 << 26))); - - control.overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); - } else if (ZPAQ_COMPRESS) - control.overhead = 112 * 1024 * 1024; + setup_overhead(control); /* Set the main nice value to half that of the backend threads since * the rzip stage is usually the rate limiting step */ - if (control.nice_val > 0 && !NO_COMPRESS) { - if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val / 2) == -1)) + if (control->nice_val > 0 && !NO_COMPRESS) { + if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val / 2) == -1)) print_err("Warning, unable to set nice value\n"); } else { - if (unlikely(setpriority(PRIO_PROCESS, 0, control.nice_val) == -1)) + if (unlikely(setpriority(PRIO_PROCESS, 0, control->nice_val) == -1)) print_err("Warning, unable to set nice value\n"); } /* One extra iteration for the case of no parameters means we will default to stdin/out */ for (i = 0; i <= argc; i++) { if (i < argc) - control.infile = argv[i]; + control->infile = argv[i]; else if (!(i == 0 && STDIN)) break; - if (control.infile) { - if ((strcmp(control.infile, "-") == 0)) - control.flags |= FLAG_STDIN; + if (control->infile) { + if ((strcmp(control->infile, "-") == 0)) + control->flags |= FLAG_STDIN; else { struct stat infile_stat; - stat(control.infile, &infile_stat); + stat(control->infile, &infile_stat); if (unlikely(S_ISDIR(infile_stat.st_mode))) failure("lrzip only works directly on FILES.\n" "Use lrztar or pipe through tar for compressing directories.\n"); @@ -748,24 +409,27 @@ int main(int argc, char *argv[]) if (INFO && STDIN) failure("Will not get file info from STDIN\n"); - if (control.outname && (strcmp(control.outname, "-") == 0)) { - control.flags |= FLAG_STDOUT; - control.msgout = stderr; - register_outputfile(control.msgout); + if (control->outname && (strcmp(control->outname, "-") == 0)) { + control->flags |= FLAG_STDOUT; + control->outFILE = stdout; + control->msgout = stderr; + register_outputfile(control, control->msgout); } /* If no output filename is specified, and we're using stdin, * use stdout */ - if (!control.outname && STDIN) { - control.flags |= FLAG_STDOUT; - control.msgout = stderr; - register_outputfile(control.msgout); + if (!control->outname && STDIN) { + control->flags |= FLAG_STDOUT; + control->msgout = stderr; + control->outFILE = stdout; + register_outputfile(control, control->msgout); } if (!STDOUT) { - control.msgout = stdout; - register_outputfile(control.msgout); + control->msgout = stdout; + register_outputfile(control, control->msgout); } + if (STDIN) control->inFILE = stdin; /* Implement signal handler only once flags are set */ handler.sa_handler = &sighandler; sigaction(SIGTERM, &handler, 0); @@ -787,34 +451,14 @@ int main(int argc, char *argv[]) if (CHECK_FILE) { if (!DECOMPRESS) { print_err("Can only check file written on decompression.\n"); - control.flags &= ~FLAG_CHECK; + control->flags &= ~FLAG_CHECK; } else if (STDOUT) { print_err("Can't check file written when writing to stdout. Checking disabled.\n"); - control.flags &= ~FLAG_CHECK; + control->flags &= ~FLAG_CHECK; } } - /* Use less ram when using STDOUT to store the temporary output - * file. */ - if (STDOUT && ((STDIN && DECOMPRESS) || !(DECOMPRESS || TEST_ONLY))) - control.maxram = control.ramsize * 2 / 9; - else - control.maxram = control.ramsize / 3; - if (BITS32) { - /* Decrease usable ram size on 32 bits due to kernel / - * userspace split. Cannot allocate larger than a 1 - * gigabyte chunk due to 32 bit signed long being - * used in alloc, and at most 3GB can be malloced, and - * 2/3 of that makes for a total of 2GB to be split - * into thirds. - */ - control.usable_ram = MAX(control.ramsize - 900000000ll, 900000000ll); - control.maxram = MIN(control.maxram, control.usable_ram); - control.maxram = MIN(control.maxram, one_g * 2 / 3); - } else - control.usable_ram = control.maxram; - round_to_page(&control.maxram); - + setup_ram(control); show_summary(); gettimeofday(&start_time, NULL); @@ -823,11 +467,11 @@ int main(int argc, char *argv[]) failure("Unable to work from STDIN while reading password\n"); if (DECOMPRESS || TEST_ONLY) - decompress_file(&control); + decompress_file(control); else if (INFO) - get_fileinfo(&control); + get_fileinfo(control); else - compress_file(&control); + compress_file(control); /* compute total time */ gettimeofday(&end_time, NULL); diff --git a/regression_test.sh b/regression_test.sh index 086905ec..54c6c38f 100755 --- a/regression_test.sh +++ b/regression_test.sh @@ -33,34 +33,37 @@ trap 'echo "ABORTING";end;exit' 1 2 15 echo testing compression from stdin ./lrzip -vvlfo lrztest.lrz < $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression from stdin end exit 1 fi +rm lrztest.lrz echo testing compression to stdout ./lrzip -vvlo - $infile > lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression to stdout end exit 1 fi +rm lrztest.lrz echo testing compression from stdin to stdout ./lrzip -vvl < $infile > lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing compression from stdin to stdout end exit 1 fi +rm lrztest.lrz echo testing standard compression ./lrzip -vvlfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing standard compression end exit 1 @@ -69,52 +72,57 @@ fi echo testing standard decompression ./lrzip -vvdo lrztest lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing standard decompression end exit 1 fi +rm lrztest echo testing standard decompression with file checking ./lrzip -vvdfco lrztest lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing standard decompression with file checking end exit 1 fi +rm lrztest echo testing decompression from stdin ./lrzip -vvfo lrztest -d < lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression from stdin end exit 1 fi +rm lrztest echo testing decompression to stdout ./lrzip -vvdo - lrztest.lrz > lrztest -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression to stdout end exit 1 fi +rm lrztest echo testing decompression from stdin to stdout ./lrzip -vvd < lrztest.lrz > lrztest -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest ];then echo FAILED testing decompression from stdin to stdout end exit 1 fi +rm lrztest echo testing testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing testing end exit 1 @@ -123,16 +131,17 @@ fi echo testing testing from stdin ./lrzip -vvt < lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing testing from stdin end exit 1 fi +rm lrztest.lrz echo testing rzip only compression ./lrzip -vvnfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing rzip only compression end exit 1 @@ -141,16 +150,17 @@ fi echo testing rzip only testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing rzip only testing end exit 1 fi +rm lrztest.lrz echo testing lzma compression ./lrzip -vvfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing lzma compression end exit 1 @@ -159,16 +169,17 @@ fi echo testing lzma testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing lzma testing end exit 1 fi +rm lrztest.lrz echo testing gzip compression ./lrzip -vvgfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing gzip compression end exit 1 @@ -177,16 +188,17 @@ fi echo testing gzip testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing gzip testing end exit 1 fi +rm lrztest.lrz echo testing bzip2 compression ./lrzip -vvbfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing bzip2 compression end exit 1 @@ -195,16 +207,17 @@ fi echo testing bzip2 testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing bzip2 testing end exit 1 fi +rm lrztest.lrz echo testing zpaq compression ./lrzip -vvzfo lrztest.lrz $infile -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing zpaq compression end exit 1 @@ -213,7 +226,7 @@ fi echo testing zpaq testing ./lrzip -vvt lrztest.lrz -if [ $? -ne 0 ];then +if [ $? -ne 0 ] || [ ! -f lrztest.lrz ];then echo FAILED testing zpaq testing end exit 1 diff --git a/runzip.c b/runzip.c index 6be36ab1..452077ac 100644 --- a/runzip.c +++ b/runzip.c @@ -45,25 +45,28 @@ #include "stream.h" #include "util.h" #include "lrzip.h" -#include "liblrzip.h" /* needed for CRC routines */ #include "lzma/C/7zCrc.h" -static inline uchar read_u8(rzip_control *control, void *ss, int stream) +static inline uchar read_u8(rzip_control *control, void *ss, int stream, bool *err) { uchar b; - if (unlikely(read_stream(control, ss, stream, &b, 1) != 1)) - fatal("Stream read u8 failed\n"); + if (unlikely(read_stream(control, ss, stream, &b, 1) != 1)) { + *err = true; + fatal_return(("Stream read u8 failed\n"), 0); + } return b; } -static inline u32 read_u32(rzip_control *control, void *ss, int stream) +static inline u32 read_u32(rzip_control *control, void *ss, int stream, bool *err) { u32 ret; - if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4)) - fatal("Stream read u32 failed\n"); + if (unlikely(read_stream(control, ss, stream, (uchar *)&ret, 4) != 4)) { + *err = true; + fatal_return(("Stream read u32 failed\n"), 0); + } ret = le32toh(ret); return ret; } @@ -74,7 +77,7 @@ static inline i64 read_vchars(rzip_control *control, void *ss, int stream, int l i64 s = 0; if (unlikely(read_stream(control, ss, stream, (uchar *)&s, length) != length)) - fatal("Stream read of %d bytes failed\n", length); + fatal_return(("Stream read of %d bytes failed\n", length), -1); s = le64toh(s); return s; } @@ -128,7 +131,7 @@ static i64 seekto_fdinend(rzip_control *control) while ((tmpchar = getchar()) != EOF) { control->tmp_inbuf[control->in_len++] = (char)tmpchar; if (unlikely(control->in_len > control->in_maxlen)) - failure("Trying to read greater than max_len\n"); + failure_return(("Trying to read greater than max_len\n"), -1); } control->in_ofs = control->in_len; return control->in_ofs; @@ -137,11 +140,13 @@ static i64 seekto_fdinend(rzip_control *control) static i64 read_header(rzip_control *control, void *ss, uchar *head) { int chunk_bytes = 2; + bool err = false; /* All chunks were unnecessarily encoded 8 bytes wide version 0.4x */ if (control->major_version == 0 && control->minor_version == 4) chunk_bytes = 8; - *head = read_u8(control, ss, 0); + *head = read_u8(control, ss, 0, &err); + if (err) return -1; return read_vchars(control, ss, 0, chunk_bytes); } @@ -151,18 +156,22 @@ static i64 unzip_literal(rzip_control *control, void *ss, i64 len, uint32 *cksum uchar *buf; if (unlikely(len < 0)) - failure("len %lld is negative in unzip_literal!\n",len); + failure_return(("len %lld is negative in unzip_literal!\n",len), -1); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal("Failed to malloc literal buffer of size %lld\n", len); + fatal_return(("Failed to malloc literal buffer of size %lld\n", len), -1); stream_read = read_stream(control, ss, 1, buf, len); - if (unlikely(stream_read == -1 )) - fatal("Failed to read_stream in unzip_literal\n"); + if (unlikely(stream_read == -1 )) { + free(buf); + fatal_return(("Failed to read_stream in unzip_literal\n"), -1); + } - if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) - fatal("Failed to write literal buffer of size %lld\n", stream_read); + if (unlikely(write_1g(control, buf, (size_t)stream_read) != (ssize_t)stream_read)) { + free(buf); + fatal_return(("Failed to write literal buffer of size %lld\n", stream_read), -1); + } if (!HAS_MD5) *cksum = CrcUpdate(*cksum, buf, stream_read); @@ -191,32 +200,36 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, uchar *buf, *off_buf; if (unlikely(len < 0)) - failure("len %lld is negative in unzip_match!\n",len); + failure_return(("len %lld is negative in unzip_match!\n",len), -1); total = 0; cur_pos = seekcur_fdout(control); if (unlikely(cur_pos == -1)) - fatal("Seek failed on out file in unzip_match.\n"); + fatal_return(("Seek failed on out file in unzip_match.\n"), -1); /* Note the offset is in a different format v0.40+ */ offset = read_vchars(control, ss, 0, chunk_bytes); + if (unlikely(offset == -1)) return -1; if (unlikely(seekto_fdhist(control, cur_pos - offset) == -1)) - fatal("Seek failed by %d from %d on history file in unzip_match\n", - offset, cur_pos); + fatal_return(("Seek failed by %d from %d on history file in unzip_match\n", + offset, cur_pos), -1); buf = (uchar *)malloc(len); if (unlikely(!buf)) - fatal("Failed to malloc match buffer of size %lld\n", len); + fatal_return(("Failed to malloc match buffer of size %lld\n", len), -1); off_buf = buf; while (len) { n = MIN(len, offset); - if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal("Failed to read %d bytes in unzip_match\n", n); - - if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) - fatal("Failed to write %d bytes in unzip_match\n", n); + if (unlikely(read_fdhist(control, off_buf, (size_t)n) != (ssize_t)n)) { + free(buf); + fatal_return(("Failed to read %d bytes in unzip_match\n", n), -1); + } + if (unlikely(write_1g(control, off_buf, (size_t)n) != (ssize_t)n)) { + free(buf); + fatal_return(("Failed to write %d bytes in unzip_match\n", n), -1); + } if (!HAS_MD5) *cksum = CrcUpdate(*cksum, off_buf, n); @@ -233,7 +246,7 @@ static i64 unzip_match(rzip_control *control, void *ss, i64 len, uint32 *cksum, return total; } -/* decompress a section of an open file. Call fatal() on error +/* decompress a section of an open file. Call fatal_return(() on error return the number of bytes that have been retrieved */ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 tally) @@ -245,6 +258,7 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 struct stat st; uchar head; void *ss; + bool err = false; /* for display of progress */ unsigned long divisor[] = {1,1024,1048576,1073741824U}; @@ -274,9 +288,9 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 print_maxverbose("Reading chunk_bytes at %lld\n", get_readseek(control, fd_in)); /* Read in the stored chunk byte width from the file */ if (unlikely(read_1g(control, fd_in, &chunk_bytes, 1) != 1)) - fatal("Failed to read chunk_bytes size in runzip_chunk\n"); + fatal_return(("Failed to read chunk_bytes size in runzip_chunk\n"), -1); if (unlikely(chunk_bytes < 1 || chunk_bytes > 8)) - failure("chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes); + failure_return(("chunk_bytes %d is invalid in runzip_chunk\n", chunk_bytes), -1); } if (!tally && expected_size) print_maxverbose("Expected size: %lld\n", expected_size); @@ -284,23 +298,35 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 ofs = seekcur_fdin(control); if (unlikely(ofs == -1)) - fatal("Failed to seek input file in runzip_fd\n"); + fatal_return(("Failed to seek input file in runzip_fd\n"), -1); if (fstat(fd_in, &st) || st.st_size - ofs == 0) return 0; ss = open_stream_in(control, fd_in, NUM_STREAMS, chunk_bytes); if (unlikely(!ss)) - failure("Failed to open_stream_in in runzip_chunk\n"); + failure_return(("Failed to open_stream_in in runzip_chunk\n"), -1); while ((len = read_header(control, ss, &head)) || head) { + i64 u; + if (unlikely(len == -1)) return -1; switch (head) { case 0: - total += unzip_literal(control, ss, len, &cksum); + u = unzip_literal(control, ss, len, &cksum); + if (unlikely(u == -1)) { + close_stream_in(control, ss); + return -1; + } + total += u; break; default: - total += unzip_match(control, ss, len, &cksum, chunk_bytes); + u = unzip_match(control, ss, len, &cksum, chunk_bytes); + if (unlikely(u == -1)) { + close_stream_in(control, ss); + return -1; + } + total += u; break; } if (expected_size) { @@ -315,9 +341,15 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 } if (!HAS_MD5) { - good_cksum = read_u32(control, ss, 0); - if (unlikely(good_cksum != cksum)) - failure("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum); + good_cksum = read_u32(control, ss, 0, &err); + if (unlikely(err)) { + close_stream_in(control, ss); + return -1; + } + if (unlikely(good_cksum != cksum)) { + close_stream_in(control, ss); + failure_return(("Bad checksum: 0x%08x - expected: 0x%08x\n", cksum, good_cksum), -1); + } print_maxverbose("Checksum for block: 0x%08x\n", cksum); } @@ -327,30 +359,31 @@ static i64 runzip_chunk(rzip_control *control, int fd_in, i64 expected_size, i64 return total; } -/* Decompress an open file. Call fatal() on error +/* Decompress an open file. Call fatal_return(() on error return the number of bytes that have been retrieved */ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 expected_size) { - uchar md5_resblock[MD5_DIGEST_SIZE]; uchar md5_stored[MD5_DIGEST_SIZE]; struct timeval start,end; - i64 total = 0; + i64 total = 0, u; if (!NO_MD5) md5_init_ctx (&control->ctx); gettimeofday(&start,NULL); do { - total += runzip_chunk(control, fd_in, expected_size, total); + u = runzip_chunk(control, fd_in, expected_size, total); + if (unlikely(u == -1)) return -1; + total += u; if (TMP_OUTBUF) - flush_tmpoutbuf(control); + { if (unlikely(!flush_tmpoutbuf(control))) return -1; } else if (STDOUT) - dump_tmpoutfile(control, fd_out); + { if (unlikely(!dump_tmpoutfile(control, fd_out))) return -1; } if (TMP_INBUF) clear_tmpinbuf(control); else if (STDIN) - clear_tmpinfile(control); + if (unlikely(!clear_tmpinfile(control))) return -1; } while (total < expected_size || (!expected_size && !control->eof)); gettimeofday(&end,NULL); @@ -361,35 +394,35 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (!NO_MD5) { int i,j; - md5_finish_ctx (&control->ctx, md5_resblock); + md5_finish_ctx (&control->ctx, control->md5_resblock); if (HAS_MD5) { i64 fdinend = seekto_fdinend(control); if (unlikely(fdinend == -1)) - failure("Failed to seekto_fdinend in rzip_fd\n"); + failure_return(("Failed to seekto_fdinend in rzip_fd\n"), -1); if (unlikely(seekto_fdin(control, fdinend - MD5_DIGEST_SIZE) == -1)) - failure("Failed to seekto_fdin in rzip_fd\n"); + failure_return(("Failed to seekto_fdin in rzip_fd\n"), -1); if (unlikely(read_1g(control, fd_in, md5_stored, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to read md5 data in runzip_fd\n"); + fatal_return(("Failed to read md5 data in runzip_fd\n"), -1); if (ENCRYPT) - lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass); + if (unlikely(!lrz_decrypt(control, md5_stored, MD5_DIGEST_SIZE, control->salt_pass))) return -1; for (i = 0; i < MD5_DIGEST_SIZE; i++) - if (md5_stored[i] != md5_resblock[i]) { + if (md5_stored[i] != control->md5_resblock[i]) { print_output("MD5 CHECK FAILED.\nStored:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_stored[j] & 0xFF); print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); - failure("\n"); + print_output("%02x", control->md5_resblock[j] & 0xFF); + failure_return(("\n"), -1); } } if (HASH_CHECK || MAX_VERBOSE) { print_output("MD5: "); for (i = 0; i < MD5_DIGEST_SIZE; i++) - print_output("%02x", md5_resblock[i] & 0xFF); + print_output("%02x", control->md5_resblock[i] & 0xFF); print_output("\n"); } @@ -399,23 +432,23 @@ i64 runzip_fd(rzip_control *control, int fd_in, int fd_out, int fd_hist, i64 exp if (TMP_OUTBUF) close_tmpoutbuf(control); - memcpy(md5_stored, md5_resblock, MD5_DIGEST_SIZE); + memcpy(md5_stored, control->md5_resblock, MD5_DIGEST_SIZE); if (unlikely(seekto_fdhist(control, 0) == -1)) - fatal("Failed to seekto_fdhist in runzip_fd\n"); + fatal_return(("Failed to seekto_fdhist in runzip_fd\n"), -1); if (unlikely((md5_fstream = fdopen(fd_hist, "r")) == NULL)) - fatal("Failed to fdopen fd_hist in runzip_fd\n"); - if (unlikely(md5_stream(md5_fstream, md5_resblock))) - fatal("Failed to md5_stream in runzip_fd\n"); + fatal_return(("Failed to fdopen fd_hist in runzip_fd\n"), -1); + if (unlikely(md5_stream(md5_fstream, control->md5_resblock))) + fatal_return(("Failed to md5_stream in runzip_fd\n"), -1); /* We don't close the file here as it's closed in main */ for (i = 0; i < MD5_DIGEST_SIZE; i++) - if (md5_stored[i] != md5_resblock[i]) { + if (md5_stored[i] != control->md5_resblock[i]) { print_output("MD5 CHECK FAILED.\nStored:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) print_output("%02x", md5_stored[j] & 0xFF); print_output("\nOutput file:"); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); - failure("\n"); + print_output("%02x", control->md5_resblock[j] & 0xFF); + failure_return(("\n"), -1); } print_output("MD5 integrity of written file matches archive\n"); if (!HAS_MD5) diff --git a/rzip.c b/rzip.c index 86625b0f..9e1d84ae 100644 --- a/rzip.c +++ b/rzip.c @@ -54,7 +54,6 @@ #include "md5.h" #include "stream.h" #include "util.h" -#include "liblrzip.h" #include "lrzip.h" /* needed for CRC routines */ #include "lzma/C/7zCrc.h" @@ -142,7 +141,7 @@ struct sliding_buffer { int fd; /* The fd of the mmap */ } sb; /* Sliding buffer */ -static void remap_low_sb(rzip_control *control) +static bool remap_low_sb(rzip_control *control) { i64 new_offset; @@ -150,19 +149,20 @@ static void remap_low_sb(rzip_control *control) round_to_page(&new_offset); print_maxverbose("Sliding main buffer to offset %lld\n", new_offset); if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal("Failed to munmap in remap_low_sb\n"); + fatal_return(("Failed to munmap in remap_low_sb\n"), false); if (new_offset + sb.size_low > sb.orig_size) sb.size_low = sb.orig_size - new_offset; sb.offset_low = new_offset; sb.buf_low = (uchar *)mmap(sb.buf_low, sb.size_low, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_low); if (unlikely(sb.buf_low == MAP_FAILED)) - fatal("Failed to re mmap in remap_low_sb\n"); + fatal_return(("Failed to re mmap in remap_low_sb\n"), false); + return true; } -static inline void remap_high_sb(rzip_control *control, i64 p) +static inline bool remap_high_sb(rzip_control *control, i64 p) { if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal("Failed to munmap in remap_high_sb\n"); + fatal_return(("Failed to munmap in remap_high_sb\n"), false); sb.size_high = sb.high_length; /* In case we shrunk it when we hit the end of the file */ sb.offset_high = p; /* Make sure offset is rounded to page size of total offset */ @@ -171,7 +171,8 @@ static inline void remap_high_sb(rzip_control *control, i64 p) sb.size_high = sb.orig_size - sb.offset_high; sb.buf_high = (uchar *)mmap(sb.buf_high, sb.size_high, PROT_READ, MAP_SHARED, sb.fd, sb.orig_offset + sb.offset_high); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal("Failed to re mmap in remap_high_sb\n"); + fatal_return(("Failed to re mmap in remap_high_sb\n"), false); + return true; } /* We use a "sliding mmap" to effectively read more than we can fit into the @@ -187,7 +188,7 @@ static uchar *sliding_get_sb(rzip_control *control, i64 p) if (p >= sb.offset_high && p < (sb.offset_high + sb.size_high)) return (sb.buf_high + (p - sb.offset_high)); /* p is not within the low or high buffer range */ - remap_high_sb(control, p); + if (unlikely(!remap_high_sb(control, p))) return NULL; return (sb.buf_high + (p - sb.offset_high)); } @@ -220,34 +221,38 @@ static void single_mcpy(rzip_control *control, unsigned char *buf, i64 offset, i static void (*do_mcpy)(rzip_control *control, unsigned char *buf, i64 offset, i64 len); /* All put_u8/u32/vchars go to stream 0 */ -static inline void put_u8(rzip_control *control, void *ss, uchar b) +static inline bool put_u8(rzip_control *control, void *ss, uchar b) { if (unlikely(write_stream(control, ss, 0, &b, 1))) - fatal("Failed to put_u8\n"); + fatal_return(("Failed to put_u8\n"), false); + return true; } -static inline void put_u32(rzip_control *control, void *ss, uint32_t s) +static inline bool put_u32(rzip_control *control, void *ss, uint32_t s) { s = htole32(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, 4))) - fatal("Failed to put_u32\n"); + fatal_return(("Failed to put_u32\n"), false); + return true; } /* Put a variable length of bytes dependant on how big the chunk is */ -static inline void put_vchars(rzip_control *control, void *ss, i64 s, int length) +static inline bool put_vchars(rzip_control *control, void *ss, i64 s, int length) { s = htole64(s); if (unlikely(write_stream(control, ss, 0, (uchar *)&s, length))) - fatal("Failed to put_vchars\n"); + fatal_return(("Failed to put_vchars\n"), false); + return true; } -static void put_header(rzip_control *control, void *ss, uchar head, i64 len) +static bool put_header(rzip_control *control, void *ss, uchar head, i64 len) { - put_u8(control, ss, head); - put_vchars(control, ss, len, 2); + if (unlikely(!put_u8(control, ss, head))) return false; + if (unlikely(!put_vchars(control, ss, len, 2))) return false; + return true; } -static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 offset, i64 len) +static bool put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 offset, i64 len) { do { i64 ofs; @@ -256,8 +261,8 @@ static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 o n = 0xFFFF; ofs = (p - offset); - put_header(control, st->ss, 1, n); - put_vchars(control, st->ss, ofs, st->chunk_bytes); + if (unlikely(!put_header(control, st->ss, 1, n))) return false; + if (unlikely(!put_vchars(control, st->ss, ofs, st->chunk_bytes))) return false; st->stats.matches++; st->stats.match_bytes += n; @@ -265,6 +270,7 @@ static void put_match(rzip_control *control, struct rzip_state *st, i64 p, i64 o p += n; offset += n; } while (len); + return true; } /* write some data to a stream mmap encoded. Return -1 on failure */ @@ -282,12 +288,12 @@ static int write_sbstream(rzip_control *control, void *ss, int stream, i64 p, i6 len -= n; if (sinfo->s[stream].buflen == sinfo->bufsize) - flush_buffer(control, sinfo, stream); + if (unlikely(!flush_buffer(control, sinfo, stream))) return -1; } return 0; } -static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, i64 p) +static bool put_literal(rzip_control *control, struct rzip_state *st, i64 last, i64 p) { do { i64 len = p - last; @@ -297,12 +303,13 @@ static void put_literal(rzip_control *control, struct rzip_state *st, i64 last, st->stats.literals++; st->stats.literal_bytes += len; - put_header(control, st->ss, 0, len); + if (unlikely(!put_header(control, st->ss, 0, len))) return false; if (unlikely(len && write_sbstream(control, st->ss, 1, last, len))) - fatal("Failed to write_stream in put_literal\n"); + fatal_return(("Failed to write_stream in put_literal\n"), false); last += len; } while (p > last); + return true; } /* Could give false positive on offset 0. Who cares. */ @@ -424,8 +431,14 @@ static tag clean_one_from_hash(rzip_control *control, struct rzip_state *st) static inline tag next_tag(rzip_control *control, struct rzip_state *st, i64 p, tag t) { - t ^= st->hash_index[*get_sb(control, p - 1)]; - t ^= st->hash_index[*get_sb(control, p + MINIMUM_MATCH - 1)]; + uchar *u; + + u = get_sb(control, p - 1); + if (unlikely(!u)) return -1; + t ^= st->hash_index[*u]; + u = get_sb(control, p + MINIMUM_MATCH - 1); + if (unlikely(!u)) return -1; + t ^= st->hash_index[*u]; return t; } @@ -433,9 +446,13 @@ static inline tag full_tag(rzip_control *control, struct rzip_state *st, i64 p) { tag ret = 0; int i; + uchar *u; - for (i = 0; i < MINIMUM_MATCH; i++) - ret ^= st->hash_index[*get_sb(control, p + i)]; + for (i = 0; i < MINIMUM_MATCH; i++) { + u = get_sb(control, p + i); + if (unlikely(!u)) return -1; + ret ^= st->hash_index[*u]; + } return ret; } @@ -535,7 +552,7 @@ static void show_distrib(rzip_control *control, struct rzip_state *st) primary * 100.0 / (total ? : 1)); } -static void hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple) +static bool hash_search(rzip_control *control, struct rzip_state *st, double pct_base, double pct_multiple) { int lastpct = 0, last_chunkpct = 0; i64 cksum_limit = 0, p, end; @@ -564,7 +581,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct } if (unlikely(!st->hash_table)) - fatal("Failed to allocate hash table in hash_search\n"); + fatal_return(("Failed to allocate hash table in hash_search\n"), false); st->minimum_tag_mask = tag_mask; st->tag_clean_ptr = 0; @@ -578,8 +595,10 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct current.p = p; current.ofs = 0; - if (likely(end > 0)) + if (likely(end > 0)) { t = full_tag(control, st, p); + if (unlikely(t == -1)) return false; + } while (p < end) { i64 reverse, mlen, offset = 0; @@ -589,6 +608,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct if (unlikely(sb.offset_search > sb.offset_low + sb.size_low)) remap_low_sb(control); t = next_tag(control, st, p, t); + if (unlikely(t == -1)) return false; /* Don't look for a match if there are no tags with this number of bits in the hash table. */ @@ -615,12 +635,13 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct if ((current.len >= GREAT_MATCH || p >= current.p + MINIMUM_MATCH) && current.len >= MINIMUM_MATCH) { if (st->last_match < current.p) - put_literal(control, st, st->last_match, current.p); - put_match(control, st, current.p, current.ofs, current.len); + if (unlikely(!put_literal(control, st, st->last_match, current.p))) return false; + if (unlikely(!put_match(control, st, current.p, current.ofs, current.len))) return false; st->last_match = current.p + current.len; current.p = p = st->last_match; current.len = 0; t = full_tag(control, st, p); + if (unlikely(t == -1)) return false; } if (unlikely(p % 128 == 0)) { @@ -633,6 +654,9 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct if (!STDIN || st->stdin_eof) print_progress("Total: %2d%% ", pct); print_progress("Chunk: %2d%%\r", chunk_pct); + if (control->info_cb) + control->info_cb(control->info_data, + (!STDIN || st->stdin_eof) ? pct : -1, chunk_pct); lastpct = pct; last_chunkpct = chunk_pct; } @@ -643,7 +667,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal("Failed to malloc ckbuf in hash_search\n"); + fatal_return(("Failed to malloc ckbuf in hash_search\n"), false); do_mcpy(control, ckbuf, cksum_limit, n); st->cksum = CrcUpdate(st->cksum, ckbuf, n); if (!NO_MD5) @@ -664,7 +688,7 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct uchar *ckbuf = malloc(n); if (unlikely(!ckbuf)) - fatal("Failed to malloc ckbuf in hash_search\n"); + fatal_return(("Failed to malloc ckbuf in hash_search\n"), false); do_mcpy(control, ckbuf, cksum_limit, n); st->cksum = CrcUpdate(st->cksum, ckbuf, n); if (!NO_MD5) @@ -673,8 +697,9 @@ static void hash_search(rzip_control *control, struct rzip_state *st, double pct free(ckbuf); } - put_literal(control, st, 0, 0); - put_u32(control, st->ss, st->cksum); + if (unlikely(!put_literal(control, st, 0, 0))) return false; + if (unlikely(!put_u32(control, st->ss, st->cksum))) return false; + return true; } @@ -696,7 +721,7 @@ static inline void *fake_mremap(void *old_address, size_t old_size, size_t new_s * anonymous ram and reading stdin into it. It means the maximum ram * we can use will be less but we will already have determined this in * rzip_chunk */ -static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) +static bool mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) { i64 len = st->chunk_size; uchar *offset_buf = buf; @@ -706,9 +731,9 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) total = 0; while (len > 0) { ret = MIN(len, one_g); - ret = read(0, offset_buf, (size_t)ret); + ret = read(fileno(control->inFILE), offset_buf, (size_t)ret); if (unlikely(ret < 0)) - fatal("Failed to read in mmap_stdin\n"); + fatal_return(("Failed to read in mmap_stdin\n"), false); total += ret; if (ret == 0) { /* Should be EOF */ @@ -717,13 +742,13 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) buf = (uchar *)mremap(buf, st->chunk_size, total, 0); st->mmap_size = st->chunk_size = total; } else { - /* Empty file */ + /* Empty file */ buf = (uchar *)mremap(buf, st->chunk_size, control->page_size, 0); st->mmap_size = control->page_size; st->chunk_size = 0; } if (unlikely(buf == MAP_FAILED)) - fatal("Failed to remap to smaller buf in mmap_stdin\n"); + fatal_return(("Failed to remap to smaller buf in mmap_stdin\n"), false); control->eof = st->stdin_eof = 1; break; } @@ -731,9 +756,10 @@ static void mmap_stdin(rzip_control *control, uchar *buf, struct rzip_state *st) len -= ret; } control->st_size += total; + return true; } -static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in, i64 offset) +static bool init_sliding_mmap(rzip_control *control, struct rzip_state *st, int fd_in, i64 offset) { /* Initialise the high buffer */ if (!STDIN) { @@ -743,7 +769,7 @@ static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int sb.high_length += control->page_size - (sb.high_length % control->page_size); sb.buf_high = (uchar *)mmap(NULL, sb.high_length, PROT_READ, MAP_SHARED, fd_in, offset); if (unlikely(sb.buf_high == MAP_FAILED)) - fatal("Unable to mmap buf_high in init_sliding_mmap\n"); + fatal_return(("Unable to mmap buf_high in init_sliding_mmap\n"), false); sb.size_high = sb.high_length; sb.offset_high = 0; } @@ -752,36 +778,45 @@ static void init_sliding_mmap(rzip_control *control, struct rzip_state *st, int sb.size_low = st->mmap_size; sb.orig_size = st->chunk_size; sb.fd = fd_in; + return true; } /* compress a chunk of an open file. Assumes that the file is able to be mmap'd and is seekable */ -static void rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out, i64 offset, +static bool rzip_chunk(rzip_control *control, struct rzip_state *st, int fd_in, int fd_out, i64 offset, double pct_base, double pct_multiple) { - init_sliding_mmap(control, st, fd_in, offset); + if (unlikely(!init_sliding_mmap(control, st, fd_in, offset))) return false; st->ss = open_stream_out(control, fd_out, NUM_STREAMS, st->chunk_size, st->chunk_bytes); if (unlikely(!st->ss)) - fatal("Failed to open streams in rzip_chunk\n"); + fatal_return(("Failed to open streams in rzip_chunk\n"), false); print_verbose("Beginning rzip pre-processing phase\n"); - hash_search(control, st, pct_base, pct_multiple); + if (unlikely(!hash_search(control, st, pct_base, pct_multiple))) { + close_stream_out(control, st->ss); + return false; + } /* unmap buffer before closing and reallocating streams */ - if (unlikely(munmap(sb.buf_low, sb.size_low))) - fatal("Failed to munmap in rzip_chunk\n"); + if (unlikely(munmap(sb.buf_low, sb.size_low))) { + close_stream_out(control, st->ss); + fatal_return(("Failed to munmap in rzip_chunk\n"), false); + } if (!STDIN) { - if (unlikely(munmap(sb.buf_high, sb.size_high))) - fatal("Failed to munmap in rzip_chunk\n"); + if (unlikely(munmap(sb.buf_high, sb.size_high))) { + close_stream_out(control, st->ss); + fatal_return(("Failed to munmap in rzip_chunk\n"), false); + } } if (unlikely(close_stream_out(control, st->ss))) - fatal("Failed to flush/close streams in rzip_chunk\n"); + fatal_return(("Failed to flush/close streams in rzip_chunk\n"), false); + return true; } /* compress a whole file chunks at a time */ -void rzip_fd(rzip_control *control, int fd_in, int fd_out) +bool rzip_fd(rzip_control *control, int fd_in, int fd_out) { /* add timers for ETA estimates * Base it off the file size and number of iterations required @@ -790,7 +825,6 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) * If file size < compression window, can't do */ struct timeval current, start, last; - uchar md5_resblock[MD5_DIGEST_SIZE]; i64 len = 0, last_chunk = 0; int pass = 0, passes, j; struct rzip_state *st; @@ -804,15 +838,19 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) st = calloc(sizeof(*st), 1); if (unlikely(!st)) - fatal("Failed to allocate control state in rzip_fd\n"); + fatal_return(("Failed to allocate control state in rzip_fd\n"), false); if (LZO_COMPRESS) { - if (unlikely(lzo_init() != LZO_E_OK)) - fatal("lzo_init() failed\n"); + if (unlikely(lzo_init() != LZO_E_OK)) { + free(st); + fatal_return(("lzo_init() failed\n"), false); + } } - if (unlikely(fstat(fd_in, &s))) - fatal("Failed to stat fd_in in rzip_fd\n"); + if (unlikely(fstat(fd_in, &s))) { + free(st); + fatal_return(("Failed to stat fd_in in rzip_fd\n"), false); + } if (!STDIN) { len = control->st_size = s.st_size; @@ -824,14 +862,18 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) /* Check if there's enough free space on the device chosen to fit the * compressed file, based on the compressed file being as large as the * uncompressed file. */ - if (unlikely(fstatvfs(fd_out, &fbuf))) - fatal("Failed to fstatvfs in compress_file\n"); + if (unlikely(fstatvfs(fd_out, &fbuf))) { + free(st); + fatal_return(("Failed to fstatvfs in compress_file\n"), false); + } free_space = (i64)fbuf.f_bsize * (i64)fbuf.f_bavail; if (free_space < control->st_size) { if (FORCE_REPLACE) print_err("Warning, possibly inadequate free space detected, but attempting to compress due to -f option being used.\n"); - else - failure("Possibly inadequate free space to compress file, use -f to override.\n"); + else { + free(st); + failure_return(("Possibly inadequate free space to compress file, use -f to override.\n"), false); + } } } @@ -900,26 +942,47 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) sb.buf_low = mmap(NULL, st->mmap_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); /* Better to shrink the window to the largest size that works than fail */ if (sb.buf_low == MAP_FAILED) { - if (unlikely(errno != ENOMEM)) - fatal("Failed to mmap %s\n", control->infile); + if (unlikely(errno != ENOMEM)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Failed to mmap %s\n", control->infile), false); + } st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); - if (unlikely(!st->mmap_size)) - fatal("Unable to mmap any ram\n"); + if (unlikely(!st->mmap_size)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Unable to mmap any ram\n"), false); + } goto retry; } st->chunk_size = st->mmap_size; - mmap_stdin(control, sb.buf_low, st); + if (unlikely(!mmap_stdin(control, sb.buf_low, st))) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + return false; + } } else { /* NOTE The buf is saved here for !STDIN mode */ sb.buf_low = (uchar *)mmap(sb.buf_low, st->mmap_size, PROT_READ, MAP_SHARED, fd_in, offset); if (sb.buf_low == MAP_FAILED) { - if (unlikely(errno != ENOMEM)) - fatal("Failed to mmap %s\n", control->infile); + if (unlikely(errno != ENOMEM)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Failed to mmap %s\n", control->infile), false); + } st->mmap_size = st->mmap_size / 10 * 9; round_to_page(&st->mmap_size); - if (unlikely(!st->mmap_size)) - fatal("Unable to mmap any ram\n"); + if (unlikely(!st->mmap_size)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + fatal_return(("Unable to mmap any ram\n"), false); + } goto retry; } if (st->mmap_size < st->chunk_size) { @@ -995,37 +1058,57 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) if (st->chunk_size == len) control->eof = 1; - rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple); + if (unlikely(!rzip_chunk(control, st, fd_in, fd_out, offset, pct_base, pct_multiple))) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + return false; + } /* st->chunk_size may be shrunk in rzip_chunk */ last_chunk = st->chunk_size; len -= st->chunk_size; - if (unlikely(len > 0 && control->eof)) - failure("Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"); + if (unlikely(len > 0 && control->eof)) { + close_streamout_threads(control); + free(st->hash_table); + free(st); + failure_return(("Wrote EOF to file yet chunk_size was shrunk, corrupting archive.\n"), false); + } } - close_streamout_threads(control); if (likely(st->hash_table)) free(st->hash_table); + if (unlikely(!close_streamout_threads(control))) { + free(st); + return false; + } if (!NO_MD5) { /* Temporary workaround till someone fixes apple md5 */ - md5_finish_ctx(&control->ctx, md5_resblock); + md5_finish_ctx(&control->ctx, control->md5_resblock); if (HASH_CHECK || MAX_VERBOSE) { print_output("MD5: "); for (j = 0; j < MD5_DIGEST_SIZE; j++) - print_output("%02x", md5_resblock[j] & 0xFF); + print_output("%02x", control->md5_resblock[j] & 0xFF); print_output("\n"); } /* When encrypting data, we encrypt the MD5 value as well */ if (ENCRYPT) - lrz_encrypt(control, md5_resblock, MD5_DIGEST_SIZE, control->salt_pass); - if (unlikely(write_1g(control, md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) - fatal("Failed to write md5 in rzip_fd\n"); + if (unlikely(!lrz_encrypt(control, control->md5_resblock, MD5_DIGEST_SIZE, control->salt_pass))) { + free(st); + return false; + } + if (unlikely(write_1g(control, control->md5_resblock, MD5_DIGEST_SIZE) != MD5_DIGEST_SIZE)) { + free(st); + fatal_return(("Failed to write md5 in rzip_fd\n"), false); + } } if (TMP_OUTBUF) - flush_tmpoutbuf(control); + if (unlikely(!flush_tmpoutbuf(control))) { + free(st); + return false; + } gettimeofday(¤t, NULL); if (STDIN) @@ -1050,4 +1133,23 @@ void rzip_fd(rzip_control *control, int fd_in, int fd_out) 1.0 * s.st_size / s2.st_size, chunkmbs); free(st); + return true; +} + +void rzip_control_free(rzip_control *control) +{ + size_t x; + if (!control) return; + + free(control->tmpdir); + free(control->outname); + free(control->outdir); + if (control->suffix && control->suffix[0]) free(control->suffix); + + for (x = 0; x < control->sinfo_idx; x++) { + free(control->sinfo_queue[x]->s); + free(control->sinfo_queue[x]); + } + free(control->sinfo_queue); + free(control); } diff --git a/rzip.h b/rzip.h index 9c10a656..116198dd 100644 --- a/rzip.h +++ b/rzip.h @@ -21,6 +21,7 @@ #define RZIP_H #include "lrzip_private.h" -void rzip_fd(rzip_control *control, int fd_in, int fd_out); +bool rzip_fd(rzip_control *control, int fd_in, int fd_out); +void rzip_control_free(rzip_control *control); #endif diff --git a/stream.c b/stream.c index f3218903..824156b6 100644 --- a/stream.c +++ b/stream.c @@ -60,13 +60,12 @@ #include "util.h" #include "zpipe.h" -#include "liblrzip.h" #include "lrzip.h" #if defined(__APPLE__) || defined(__FreeBSD__) -# define fmemopen fake_fmemopen -# define open_memstream fake_open_memstream +# define fmemopen(s, len, modes) fake_fmemopen(control, (s), (len), (modes)) +# define open_memstream(bufloc, sizeloc) fake_open_memstream(control, (bufloc), (sizeloc)) # define memstream_update_buffer fake_open_memstream_update_buffer # define mremap fake_mremap #else @@ -105,53 +104,61 @@ static pthread_mutex_t output_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t output_cond = PTHREAD_COND_INITIALIZER; static pthread_t *threads; -static void init_mutex(pthread_mutex_t *mutex) +static bool init_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_init(mutex, NULL))) - fatal("pthread_mutex_init failed"); + fatal_return(("pthread_mutex_init failed"), false); + return true; } -static void unlock_mutex(pthread_mutex_t *mutex) +static bool unlock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_unlock(mutex))) - fatal("pthread_mutex_unlock failed"); + fatal_return(("pthread_mutex_unlock failed"), false); + return true; } -static void lock_mutex(pthread_mutex_t *mutex) +static bool lock_mutex(rzip_control *control, pthread_mutex_t *mutex) { if (unlikely(pthread_mutex_lock(mutex))) - fatal("pthread_mutex_lock failed"); + fatal_return(("pthread_mutex_lock failed"), false); + return true; } -static void cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) +static bool cond_wait(rzip_control *control, pthread_cond_t *cond, pthread_mutex_t *mutex) { if (unlikely(pthread_cond_wait(cond, mutex))) - fatal("pthread_cond_wait failed"); + fatal_return(("pthread_cond_wait failed"), false); + return true; } -static void cond_broadcast(pthread_cond_t *cond) +static bool cond_broadcast(rzip_control *control, pthread_cond_t *cond) { if (unlikely(pthread_cond_broadcast(cond))) - fatal("pthread_cond_broadcast failed"); + fatal_return(("pthread_cond_broadcast failed"), false); + return true; } -void create_pthread(pthread_t *thread, pthread_attr_t * attr, +bool create_pthread(rzip_control *control, pthread_t *thread, pthread_attr_t * attr, void * (*start_routine)(void *), void *arg) { if (unlikely(pthread_create(thread, attr, start_routine, arg))) - fatal("pthread_create"); + fatal_return(("pthread_create"), false); + return true; } -void detach_pthread(pthread_t *thread) +bool detach_pthread(rzip_control *control, pthread_t *thread) { if (unlikely(pthread_detach(*thread))) - fatal("pthread_detach"); + fatal_return(("pthread_detach"), false); + return true; } -void join_pthread(pthread_t th, void **thread_return) +bool join_pthread(rzip_control *control, pthread_t th, void **thread_return) { if (pthread_join(th, thread_return)) - fatal("pthread_join"); + fatal_return(("pthread_join"), false); + return true; } /* just to keep things clean, declare function here @@ -159,27 +166,29 @@ void join_pthread(pthread_t th, void **thread_return) */ static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len); -static inline FILE *fake_fmemopen(void *buf, size_t buflen, char *mode) +static inline FILE *fake_fmemopen(rzip_control *control, void *buf, size_t buflen, char *mode) { FILE *in; if (unlikely(strcmp(mode, "r"))) - failure("fake_fmemopen only supports mode \"r\"."); + failure_return(("fake_fmemopen only supports mode \"r\"."), NULL); in = tmpfile(); if (unlikely(!in)) return NULL; - if (unlikely(fwrite(buf, buflen, 1, in) != 1)) + if (unlikely(fwrite(buf, buflen, 1, in) != 1)) { + fclose(in); return NULL; + } rewind(in); return in; } -static inline FILE *fake_open_memstream(char **buf, size_t *length) +static inline FILE *fake_open_memstream(rzip_control *control, char **buf, size_t *length) { FILE *out; if (unlikely(buf == NULL || length == NULL)) - failure("NULL parameter to fake_open_memstream"); + failure_return(("NULL parameter to fake_open_memstream"), NULL); out = tmpfile(); if (unlikely(!out)) return NULL; @@ -197,10 +206,14 @@ static inline int fake_open_memstream_update_buffer(FILE *fp, uchar **buf, size_ *buf = (uchar *)malloc(*length); if (unlikely(!*buf)) return -1; - if (unlikely(fread(*buf, *length, 1, fp) != 1)) + if (unlikely(fread(*buf, *length, 1, fp) != 1)) { + free(*buf); return -1; - if (unlikely(fseek(fp, original_pos, SEEK_SET))) + } + if (unlikely(fseek(fp, original_pos, SEEK_SET))) { + free(*buf); return -1; + } return 0; } @@ -238,8 +251,11 @@ static int zpaq_compress_buf(rzip_control *control, struct compress_thread *cthr zpipe_compress(in, out, control->msgout, cthread->s_len, (int)(SHOW_PROGRESS), thread); - if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal("Failed to memstream_update_buffer in zpaq_compress_buf"); + if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) { + fclose(in); + fclose(out); + fatal_return(("Failed to memstream_update_buffer in zpaq_compress_buf"), -1); + } fclose(in); fclose(out); @@ -495,13 +511,17 @@ static int zpaq_decompress_buf(rzip_control *control, struct uncomp_thread *ucth out = open_memstream((char **)&c_buf, &dlen); if (unlikely(!out)) { print_err("Failed to open_memstream in zpaq_decompress_buf\n"); + fclose(in); return -1; } zpipe_decompress(in, out, control->msgout, ucthread->u_len, (int)(SHOW_PROGRESS), thread); - if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) - fatal("Failed to memstream_update_buffer in zpaq_decompress_buf"); + if (unlikely(memstream_update_buffer(out, &c_buf, &dlen))) { + fclose(in); + fclose(out); + fatal_return(("Failed to memstream_update_buffer in zpaq_decompress_buf"), -1); + } fclose(in); fclose(out); @@ -662,8 +682,6 @@ static int lzo_decompress_buf(rzip_control *control __UNUSED__, struct uncomp_th /* WORK FUNCTIONS */ -i64 one_g = 1000 * 1024 * 1024; - /* Look at whether we're writing to a ram location or physical files and write * the data accordingly. */ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret) @@ -675,9 +693,9 @@ ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret) /* The data won't fit in a temporary output buffer so we have * to fall back to temporary files. */ print_verbose("Unable to decompress entirely in ram, will use physical files\n"); - write_fdout(control, control->tmp_outbuf, control->out_len); + if (unlikely(!write_fdout(control, control->tmp_outbuf, control->out_len))) return -1; close_tmpoutbuf(control); - write_fdout(control, offset_buf, ret); + if (unlikely(!write_fdout(control, offset_buf, ret))) return -1; return ret; } memcpy(control->tmp_outbuf + control->out_ofs, offset_buf, ret); @@ -709,7 +727,7 @@ ssize_t write_1g(rzip_control *control, void *buf, i64 len) return total; } -static void read_fdin(struct rzip_control *control, i64 len) +static bool read_fdin(struct rzip_control *control, i64 len) { int tmpchar; i64 i; @@ -717,11 +735,12 @@ static void read_fdin(struct rzip_control *control, i64 len) for (i = 0; i < len; i++) { tmpchar = getchar(); if (unlikely(tmpchar == EOF)) - failure("Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", - len, i); + failure_return(("Reached end of file on STDIN prematurely on read_fdin, asked for %lld got %lld\n", + len, i), false); control->tmp_inbuf[control->in_ofs + i] = (char)tmpchar; } control->in_len = control->in_ofs + len; + return true; } /* Ditto for read */ @@ -735,13 +754,13 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) /* We're decompressing from STDIN */ if (unlikely(control->in_ofs + len > control->in_maxlen)) { /* We're unable to fit it all into the temp buffer */ - write_fdin(control); - read_tmpinfile(control, control->fd_in); + if (unlikely(!write_fdin(control))) return -1; + if (unlikely(!read_tmpinfile(control, control->fd_in))) return -1; close_tmpinbuf(control); goto read_fd; } if (control->in_ofs + len > control->in_len) - read_fdin(control, control->in_ofs + len - control->in_len); + if (unlikely(!read_fdin(control, control->in_ofs + len - control->in_len))) return false; memcpy(buf, control->tmp_inbuf + control->in_ofs, len); control->in_ofs += len; return len; @@ -749,7 +768,7 @@ ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len) if (TMP_OUTBUF && fd == control->fd_out) { if (unlikely(control->out_ofs + len > control->out_maxlen)) - failure("Trying to read beyond out_ofs in tmpoutbuf\n"); + failure_return(("Trying to read beyond out_ofs in tmpoutbuf\n"), -1); memcpy(buf, control->tmp_outbuf + control->out_ofs, len); control->out_ofs += len; return len; @@ -845,7 +864,7 @@ static inline int read_val(rzip_control *control, int f, i64 *v, int len) return ret; } -static int fd_seekto(struct stream_info *sinfo, i64 spos, i64 pos) +static int fd_seekto(rzip_control *control, struct stream_info *sinfo, i64 spos, i64 pos) { if (unlikely(lseek(sinfo->fd, spos, SEEK_SET) != spos)) { print_err("Failed to seek to %lld in stream\n", pos); @@ -869,7 +888,7 @@ static int seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) return 0; } - return fd_seekto(sinfo, spos, pos); + return fd_seekto(control, sinfo, spos, pos); } static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos) @@ -878,7 +897,7 @@ static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos if (TMP_INBUF) { if (spos > control->in_len) - read_fdin(control, spos - control->in_len); + if (unlikely(!read_fdin(control, spos - control->in_len))) return -1; control->in_ofs = spos; if (unlikely(spos < 0)) { print_err("Trying to seek to %lld outside tmp inbuf in read_seekto\n", spos); @@ -887,7 +906,7 @@ static int read_seekto(rzip_control *control, struct stream_info *sinfo, i64 pos return 0; } - return fd_seekto(sinfo, spos, pos); + return fd_seekto(control, sinfo, spos, pos); } static i64 get_seek(rzip_control *control, int fd) @@ -898,7 +917,7 @@ static i64 get_seek(rzip_control *control, int fd) return control->out_relofs + control->out_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal("Failed to lseek in get_seek\n"); + fatal_return(("Failed to lseek in get_seek\n"), -1); return ret; } @@ -910,11 +929,11 @@ i64 get_readseek(rzip_control *control, int fd) return control->in_ofs; ret = lseek(fd, 0, SEEK_CUR); if (unlikely(ret == -1)) - fatal("Failed to lseek in get_seek\n"); + fatal_return(("Failed to lseek in get_seek\n"), -1); return ret; } -void prepare_streamout_threads(rzip_control *control) +bool prepare_streamout_threads(rzip_control *control) { int i; @@ -928,30 +947,47 @@ void prepare_streamout_threads(rzip_control *control) control->threads = 1; threads = calloc(sizeof(pthread_t), control->threads); if (unlikely(!threads)) - fatal("Unable to calloc threads in prepare_streamout_threads\n"); + fatal_return(("Unable to calloc threads in prepare_streamout_threads\n"), false); cthread = calloc(sizeof(struct compress_thread), control->threads); - if (unlikely(!cthread)) - fatal("Unable to calloc cthread in prepare_streamout_threads\n"); + if (unlikely(!cthread)) { + free(threads); + fatal_return(("Unable to calloc cthread in prepare_streamout_threads\n"), false); + } for (i = 0; i < control->threads; i++) - init_mutex(&cthread[i].mutex); + if (unlikely(!init_mutex(control, &cthread[i].mutex))) { + int x; + for (x = 0; x < i; x++) pthread_mutex_destroy(&cthread[x].mutex); + free(threads); + free(cthread); + return false; + } + return true; } -void close_streamout_threads(rzip_control *control) +bool close_streamout_threads(rzip_control *control) { int i, close_thread = output_thread; /* Wait for the threads in the correct order in case they end up * serialised */ for (i = 0; i < control->threads; i++) { - lock_mutex(&cthread[close_thread].mutex); + if (unlikely(!lock_mutex(control, &cthread[close_thread].mutex))) { + int x; + for (x = 0; x < i; x++) unlock_mutex(control, &cthread[close_thread].mutex); + free(cthread); + free(threads); + return false; + } + if (++close_thread == control->threads) close_thread = 0; } free(cthread); free(threads); + return true; } /* open a set of output streams, compressing with the given @@ -1046,8 +1082,12 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li for (i = 0; i < n; i++) { sinfo->s[i].buf = calloc(sinfo->bufsize , 1); - if (unlikely(!sinfo->s[i].buf)) + if (unlikely(!sinfo->s[i].buf)) { fatal("Unable to malloc buffer of size %lld in open_stream_out\n", sinfo->bufsize); + free(sinfo->s); + free(sinfo); + return NULL; + } } return (void *)sinfo; @@ -1056,7 +1096,7 @@ void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_li /* The block headers are all encrypted so we read the data and salt associated * with them, decrypt the data, then return the decrypted version of the * values */ -static void decrypt_header(rzip_control *control, uchar *head, uchar *c_type, +static bool decrypt_header(rzip_control *control, uchar *head, uchar *c_type, i64 *c_len, i64 *u_len, i64 *last_head) { uchar *buf = head + SALT_LEN; @@ -1066,12 +1106,13 @@ static void decrypt_header(rzip_control *control, uchar *head, uchar *c_type, memcpy(buf + 9, u_len, 8); memcpy(buf + 17, last_head, 8); - lrz_decrypt(control, buf, 25, head); + if (unlikely(!lrz_decrypt(control, buf, 25, head))) return false; memcpy(c_type, buf, 1); memcpy(c_len, buf + 1, 8); memcpy(u_len, buf + 9, 8); memcpy(last_head, buf + 17, 8); + return true; } /* prepare a set of n streams for reading on file descriptor f */ @@ -1096,8 +1137,11 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) return NULL; ucthread = calloc(sizeof(struct uncomp_thread), total_threads); - if (unlikely(!ucthread)) - fatal("Unable to calloc cthread in open_stream_in\n"); + if (unlikely(!ucthread)) { + free(sinfo); + free(threads); + fatal_return(("Unable to calloc cthread in open_stream_in\n"), NULL); + } sinfo->num_streams = n; sinfo->fd = f; @@ -1135,6 +1179,7 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) } } sinfo->initial_pos = get_readseek(control, f); + if (unlikely(sinfo->initial_pos == -1)) goto failed; for (i = 0; i < n; i++) { uchar c, enc_head[25 + SALT_LEN]; @@ -1185,7 +1230,7 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) sinfo->total_read += header_length; if (ENCRYPT) - decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head); + if (unlikely(!decrypt_header(control, enc_head, &c, &v1, &v2, &sinfo->s[i].last_head))) goto failed; v1 = le64toh(v1); v2 = le64toh(v2); @@ -1228,32 +1273,38 @@ void *open_stream_in(rzip_control *control, int f, int n, char chunk_bytes) * by reading what has been written, encrypting it, and writing back over it. * This is very convoluted depending on whether a last_head value is written * to this block or not. See the callers of this function */ -static void rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, i64 ofs) +static bool rewrite_encrypted(rzip_control *control, struct stream_info *sinfo, i64 ofs) { uchar *buf, *head; i64 cur_ofs; cur_ofs = get_seek(control, sinfo->fd) - sinfo->initial_pos; + if (unlikely(cur_ofs == -1)) return false; head = malloc(25 + SALT_LEN); if (unlikely(!head)) - fatal("Failed to malloc head in rewrite_encrypted\n"); + fatal_return(("Failed to malloc head in rewrite_encrypted\n"), false); buf = head + SALT_LEN; - get_rand(head, SALT_LEN); + if (unlikely(!get_rand(control, head, SALT_LEN))) goto error; if (unlikely(seekto(control, sinfo, ofs - SALT_LEN))) - failure("Failed to seekto buf ofs in rewrite_encrypted\n"); + failure_goto(("Failed to seekto buf ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, head, SALT_LEN))) - failure("Failed to write_buf head in rewrite_encrypted\n"); + failure_goto(("Failed to write_buf head in rewrite_encrypted\n"), error); if (unlikely(read_buf(control, sinfo->fd, buf, 25))) - failure("Failed to read_buf buf in rewrite_encrypted\n"); - lrz_encrypt(control, buf, 25, head); + failure_goto(("Failed to read_buf buf in rewrite_encrypted\n"), error); + + if (unlikely(!lrz_encrypt(control, buf, 25, head))) goto error; if (unlikely(seekto(control, sinfo, ofs))) - failure("Failed to seek back to ofs in rewrite_encrypted\n"); + failure_goto(("Failed to seek back to ofs in rewrite_encrypted\n"), error); if (unlikely(write_buf(control, buf, 25))) - failure("Failed to write_buf encrypted buf in rewrite_encrypted\n"); + failure_goto(("Failed to write_buf encrypted buf in rewrite_encrypted\n"), error); free(head); seekto(control, sinfo, cur_ofs); + return true; +error: + free(head); + return false; } /* Enter with s_buf allocated,s_buf points to the compressed data after the @@ -1270,7 +1321,7 @@ static void *compthread(void *data) int write_len; /* Make sure this thread doesn't already exist */ - + free(data); cti = &cthread[i]; ctis = cti->sinfo; @@ -1296,7 +1347,7 @@ static void *compthread(void *data) ret = gzip_compress_buf(control, cti); else if (ZPAQ_COMPRESS) ret = zpaq_compress_buf(control, cti, i); - else failure("Dunno wtf compression to use!\n"); + else failure_goto(("Dunno wtf compression to use!\n"), error); } padded_len = cti->c_len; @@ -1307,21 +1358,21 @@ static void *compthread(void *data) padded_len = MIN_SIZE; cti->s_buf = realloc(cti->s_buf, MIN_SIZE); if (unlikely(!cti->s_buf)) - fatal("Failed to realloc s_buf in compthread\n"); - get_rand(cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len); + fatal_goto(("Failed to realloc s_buf in compthread\n"), error); + if (unlikely(!get_rand(control, cti->s_buf + cti->c_len, MIN_SIZE - cti->c_len))) goto error; } /* If compression fails for whatever reason multithreaded, then wait * for the previous thread to finish, serialising the work to decrease * the memory requirements, increasing the chance of success */ if (unlikely(ret && waited)) - failure("Failed to compress in compthread\n"); + failure_goto(("Failed to compress in compthread\n"), error); if (!waited) { - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); while (output_thread != i) - cond_wait(&output_cond, &output_lock); - unlock_mutex(&output_lock); + cond_wait(control, &output_cond, &output_lock); + unlock_mutex(control, &output_lock); waited = 1; } if (unlikely(ret)) { @@ -1341,7 +1392,7 @@ static void *compthread(void *data) if (TMP_OUTBUF) { if (!control->magic_written) write_magic(control); - flush_tmpoutbuf(control); + if (unlikely(!flush_tmpoutbuf(control))) goto error; } print_maxverbose("Writing initial chunk bytes value %d at %lld\n", @@ -1358,6 +1409,7 @@ static void *compthread(void *data) /* First chunk of this stream, write headers */ ctis->initial_pos = get_seek(control, ctis->fd); + if (unlikely(ctis->initial_pos == -1)) goto error; print_maxverbose("Writing initial header at %lld\n", ctis->initial_pos); for (j = 0; j < ctis->num_streams; j++) { @@ -1365,7 +1417,7 @@ static void *compthread(void *data) * later */ if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal("Failed to write_buf blank salt in compthread %d\n", i); + fatal_goto(("Failed to write_buf blank salt in compthread %d\n", i), error); ctis->cur_pos += SALT_LEN; } ctis->s[j].last_head = ctis->cur_pos + 1 + (write_len * 2); @@ -1380,10 +1432,10 @@ static void *compthread(void *data) print_maxverbose("Compthread %ld seeking to %lld to store length %d\n", i, ctis->s[cti->streamno].last_head, write_len); if (unlikely(seekto(control, ctis, ctis->s[cti->streamno].last_head))) - fatal("Failed to seekto in compthread %d\n", i); + fatal_goto(("Failed to seekto in compthread %d\n", i), error); if (unlikely(write_val(control, ctis->cur_pos, write_len))) - fatal("Failed to write_val cur_pos in compthread %d\n", i); + fatal_goto(("Failed to write_val cur_pos in compthread %d\n", i), error); if (ENCRYPT) rewrite_encrypted(control, ctis, ctis->s[cti->streamno].last_head - 17); @@ -1393,13 +1445,13 @@ static void *compthread(void *data) print_maxverbose("Compthread %ld seeking to %lld to write header\n", i, ctis->cur_pos); if (unlikely(seekto(control, ctis, ctis->cur_pos))) - fatal("Failed to seekto cur_pos in compthread %d\n", i); + fatal_goto(("Failed to seekto cur_pos in compthread %d\n", i), error); print_maxverbose("Thread %ld writing %lld compressed bytes from stream %d\n", i, padded_len, cti->streamno); if (ENCRYPT) { if (unlikely(write_val(control, 0, SALT_LEN))) - fatal("Failed to write_buf header salt in compthread %d\n", i); + fatal_goto(("Failed to write_buf header salt in compthread %d\n", i), error); ctis->cur_pos += SALT_LEN; ctis->s[cti->streamno].last_headofs = ctis->cur_pos; } @@ -1408,44 +1460,45 @@ static void *compthread(void *data) write_val(control, cti->c_len, write_len) || write_val(control, cti->s_len, write_len) || write_val(control, 0, write_len))) { - fatal("Failed write in compthread %d\n", i); + fatal_goto(("Failed write in compthread %d\n", i), error); } ctis->cur_pos += 1 + (write_len * 3); if (ENCRYPT) { - get_rand(cti->salt, SALT_LEN); + if (unlikely(!get_rand(control, cti->salt, SALT_LEN))) goto error; if (unlikely(write_buf(control, cti->salt, SALT_LEN))) - fatal("Failed to write_buf block salt in compthread %d\n", i); - lrz_encrypt(control, cti->s_buf, padded_len, cti->salt); + fatal_goto(("Failed to write_buf block salt in compthread %d\n", i), error); + if (unlikely(!lrz_encrypt(control, cti->s_buf, padded_len, cti->salt))) goto error; ctis->cur_pos += SALT_LEN; } print_maxverbose("Compthread %ld writing data at %lld\n", i, ctis->cur_pos); if (unlikely(write_buf(control, cti->s_buf, padded_len))) - fatal("Failed to write_buf s_buf in compthread %d\n", i); + fatal_goto(("Failed to write_buf s_buf in compthread %d\n", i), error); ctis->cur_pos += padded_len; free(cti->s_buf); - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); if (++output_thread == control->threads) output_thread = 0; - cond_broadcast(&output_cond); - unlock_mutex(&output_lock); + cond_broadcast(control, &output_cond); + unlock_mutex(control, &output_lock); - unlock_mutex(&cti->mutex); +error: + unlock_mutex(control, &cti->mutex); - return 0; + return NULL; } -static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int streamno, int newbuf) +static bool clear_buffer(rzip_control *control, struct stream_info *sinfo, int streamno, int newbuf) { static long i = 0; stream_thread_struct *s; /* Make sure this thread doesn't already exist */ - lock_mutex(&cthread[i].mutex); + lock_mutex(control, &cthread[i].mutex); cthread[i].sinfo = sinfo; cthread[i].streamno = streamno; @@ -1456,30 +1509,39 @@ static void clear_buffer(rzip_control *control, struct stream_info *sinfo, int s i, cthread[i].s_len, streamno); s = malloc(sizeof(stream_thread_struct)); - if (unlikely(!s)) - fatal("Unable to malloc in clear_buffer"); + if (unlikely(!s)) { + unlock_mutex(control, &cthread[i].mutex); + fatal_return(("Unable to malloc in clear_buffer"), false); + } s->i = i; s->control = control; - create_pthread(&threads[i], NULL, compthread, s); - detach_pthread(&threads[i]); + if (unlikely((!create_pthread(control, &threads[i], NULL, compthread, s)) || + (!detach_pthread(control, &threads[i])))) { + unlock_mutex(control, &cthread[i].mutex); + return false; + } + if (newbuf) { /* The stream buffer has been given to the thread, allocate a * new one. */ sinfo->s[streamno].buf = malloc(sinfo->bufsize); - if (unlikely(!sinfo->s[streamno].buf)) - fatal("Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize); + if (unlikely(!sinfo->s[streamno].buf)) { + unlock_mutex(control, &cthread[i].mutex); + fatal_return(("Unable to malloc buffer of size %lld in flush_buffer\n", sinfo->bufsize), false); + } sinfo->s[streamno].buflen = 0; } if (++i == control->threads) i = 0; + return true; } /* flush out any data in a stream buffer */ -void flush_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) +bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int streamno) { - clear_buffer(control, sinfo, streamno, 1); + return clear_buffer(control, sinfo, streamno, 1); } static void *ucompthread(void *data) @@ -1515,7 +1577,7 @@ static void *ucompthread(void *data) ret = zpaq_decompress_buf(control, uci, i); break; default: - failure("Dunno wtf decompression type to use!\n"); + failure_return(("Dunno wtf decompression type to use!\n"), NULL); break; } } @@ -1524,15 +1586,15 @@ static void *ucompthread(void *data) * parallel */ if (unlikely(ret)) { if (unlikely(waited)) - failure("Failed to decompress in ucompthread\n"); + failure_return(("Failed to decompress in ucompthread\n"), NULL); print_maxverbose("Unable to decompress in parallel, waiting for previous thread to complete before trying again\n"); /* We do not strictly need to wait for this, so it's used when * decompression fails due to inadequate memory to try again * serialised. */ - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); while (output_thread != i) - cond_wait(&output_cond, &output_lock); - unlock_mutex(&output_lock); + cond_wait(control, &output_cond, &output_lock); + unlock_mutex(control, &output_lock); waited = 1; goto retry; } @@ -1557,7 +1619,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str goto out; fill_another: if (unlikely(ucthread[s->uthread_no].busy)) - failure("Trying to start a busy thread, this shouldn't happen!\n"); + failure_return(("Trying to start a busy thread, this shouldn't happen!\n"), -1); if (unlikely(read_seekto(control, sinfo, s->last_head))) return -1; @@ -1601,7 +1663,7 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str sinfo->total_read += header_length; if (ENCRYPT) { - decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head); + if (unlikely(!decrypt_header(control, enc_head, &c_type, &c_len, &u_len, &last_head))) return -1; if (unlikely(read_buf(control, sinfo->fd, blocksalt, SALT_LEN))) return -1; } @@ -1615,14 +1677,14 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str s_buf = malloc(MAX(u_len, MIN_SIZE)); if (unlikely(u_len && !s_buf)) - fatal("Unable to malloc buffer of size %lld in fill_buffer\n", u_len); + fatal_return(("Unable to malloc buffer of size %lld in fill_buffer\n", u_len), -1); sinfo->ram_alloced += u_len; if (unlikely(read_buf(control, sinfo->fd, s_buf, padded_len))) return -1; if (ENCRYPT) - lrz_decrypt(control, s_buf, padded_len, blocksalt); + if (unlikely(!lrz_decrypt(control, s_buf, padded_len, blocksalt))) return -1; ucthread[s->uthread_no].s_buf = s_buf; ucthread[s->uthread_no].c_len = c_len; @@ -1638,10 +1700,13 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str st = malloc(sizeof(stream_thread_struct)); if (unlikely(!st)) - fatal("Unable to malloc in fill_buffer"); + fatal_return(("Unable to malloc in fill_buffer"), -1); st->i = s->uthread_no; st->control = control; - create_pthread(&threads[s->uthread_no], NULL, ucompthread, st); + if (unlikely(!create_pthread(control, &threads[s->uthread_no], NULL, ucompthread, st))) { + free(st); + return -1; + } if (++s->uthread_no == s->base_thread + s->total_threads) s->uthread_no = s->base_thread; @@ -1656,13 +1721,13 @@ static int fill_buffer(rzip_control *control, struct stream_info *sinfo, int str sinfo->ram_alloced < control->maxram) goto fill_another; out: - lock_mutex(&output_lock); + lock_mutex(control, &output_lock); output_thread = s->unext_thread; - cond_broadcast(&output_cond); - unlock_mutex(&output_lock); + cond_broadcast(control, &output_cond); + unlock_mutex(control, &output_lock); /* join_pthread here will make it wait till the data is ready */ - join_pthread(threads[s->unext_thread], NULL); + if (unlikely(!join_pthread(control, threads[s->unext_thread], NULL))) return -1; ucthread[s->unext_thread].busy = 0; print_maxverbose("Taking decompressed data from thread %ld\n", s->unext_thread); @@ -1694,7 +1759,7 @@ int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 le /* Flush the buffer every sinfo->bufsize into one thread */ if (sinfo->s[streamno].buflen == sinfo->bufsize) - flush_buffer(control, sinfo, streamno); + if (unlikely(!flush_buffer(control, sinfo, streamno))) return -1; } return 0; } @@ -1738,7 +1803,7 @@ int close_stream_out(rzip_control *control, void *ss) for (i = 0; i < sinfo->num_streams; i++) { if (sinfo->s[i].buflen) - clear_buffer(control, sinfo, i, 0); + if (unlikely(!clear_buffer(control, sinfo, i, 0))) return -1; } if (ENCRYPT) { @@ -1748,19 +1813,36 @@ int close_stream_out(rzip_control *control, void *ss) int close_thread = output_thread; for (i = 0; i < control->threads; i++) { - lock_mutex(&cthread[close_thread].mutex); - unlock_mutex(&cthread[close_thread].mutex); + lock_mutex(control, &cthread[close_thread].mutex); + unlock_mutex(control, &cthread[close_thread].mutex); if (++close_thread == control->threads) close_thread = 0; } for (i = 0; i < sinfo->num_streams; i++) rewrite_encrypted(control, sinfo, sinfo->s[i].last_headofs); } - + if (control->library_mode) { + if (!control->sinfo_buckets) { + /* no streams added */ + control->sinfo_queue = calloc(STREAM_BUCKET_SIZE + 1, sizeof(void*)); + if (!control->sinfo_queue) return -1; + control->sinfo_buckets++; + } else if (control->sinfo_idx == STREAM_BUCKET_SIZE * control->sinfo_buckets + 1) { + /* all buckets full, create new bucket */ + void *tmp; + + tmp = realloc(control->sinfo_queue, (++control->sinfo_buckets * STREAM_BUCKET_SIZE + 1) * sizeof(void*)); + if (!tmp) return -1; + control->sinfo_queue = tmp; + memset(control->sinfo_queue + control->sinfo_idx, 0, ((control->sinfo_buckets * STREAM_BUCKET_SIZE + 1) - control->sinfo_idx) * sizeof(void*)); + } + control->sinfo_queue[control->sinfo_idx++] = sinfo; + } #if 0 - /* These cannot be freed because their values are read after the next - * stream has started so they're not properly freed and just dropped on - * program exit! FIXME */ + /* These cannot be freed immediately because their values are read after the next + * stream has started. Instead (in library mode), they are stored and only freed + * after the entire operation has completed. + */ free(sinfo->s); free(sinfo); #endif @@ -1811,14 +1893,16 @@ static int lzo_compresses(rzip_control *control, uchar *s_buf, i64 s_len) return 1; wrkmem = (lzo_bytep) malloc(LZO1X_1_MEM_COMPRESS); if (unlikely(wrkmem == NULL)) - fatal("Unable to allocate wrkmem in lzo_compresses\n"); + fatal_return(("Unable to allocate wrkmem in lzo_compresses\n"), 0); in_len = MIN(test_len, buftest_size); dlen = STREAM_BUFSIZE + STREAM_BUFSIZE / 16 + 64 + 3; c_buf = malloc(dlen); - if (unlikely(!c_buf)) - fatal("Unable to allocate c_buf in lzo_compresses\n"); + if (unlikely(!c_buf)) { + free(wrkmem); + fatal_return(("Unable to allocate c_buf in lzo_compresses\n"), 0); + } /* Test progressively larger blocks at a time and as soon as anything compressible is found, jump out as a success */ diff --git a/stream.h b/stream.h index c9fbc7f7..abf00c52 100644 --- a/stream.h +++ b/stream.h @@ -23,22 +23,21 @@ #include "lrzip_private.h" #include -void create_pthread(pthread_t *thread, pthread_attr_t *attr, +bool create_pthread(pthread_t *thread, pthread_attr_t *attr, void * (*start_routine)(void *), void *arg); -void join_pthread(pthread_t th, void **thread_return); +bool join_pthread(pthread_t th, void **thread_return); ssize_t write_1g(rzip_control *control, void *buf, i64 len); ssize_t read_1g(rzip_control *control, int fd, void *buf, i64 len); i64 get_readseek(rzip_control *control, int fd); -void prepare_streamout_threads(rzip_control *control); -void close_streamout_threads(rzip_control *control); +bool prepare_streamout_threads(rzip_control *control); +bool close_streamout_threads(rzip_control *control); void *open_stream_out(rzip_control *control, int f, unsigned int n, i64 chunk_limit, char cbytes); void *open_stream_in(rzip_control *control, int f, int n, char cbytes); -void flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); +bool flush_buffer(rzip_control *control, struct stream_info *sinfo, int stream); int write_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); i64 read_stream(rzip_control *control, void *ss, int streamno, uchar *p, i64 len); int close_stream_out(rzip_control *control, void *ss); int close_stream_in(rzip_control *control, void *ss); ssize_t put_fdout(rzip_control *control, void *offset_buf, ssize_t ret); -i64 one_g; #endif diff --git a/util.c b/util.c index 2179abc8..ea3dfb19 100644 --- a/util.c +++ b/util.c @@ -52,45 +52,45 @@ #include #include #include "lrzip_private.h" -#include "liblrzip.h" #include "util.h" #include "sha4.h" #include "aes.h" +#ifdef HAVE_CTYPE_H +# include +#endif -static const char *infile = NULL; -static char delete_infile = 0; -static const char *outfile = NULL; -static char delete_outfile = 0; -static FILE *outputfile = NULL; +/* Macros for testing parameters */ +#define isparameter( parmstring, value ) (!strcasecmp( parmstring, value )) +#define iscaseparameter( parmvalue, value ) (!strcmp( parmvalue, value )) -void register_infile(const char *name, char delete) +void register_infile(rzip_control *control, const char *name, char delete) { - infile = name; - delete_infile = delete; + control->util_infile = name; + control->delete_infile = delete; } -void register_outfile(const char *name, char delete) +void register_outfile(rzip_control *control, const char *name, char delete) { - outfile = name; - delete_outfile = delete; + control->util_outfile = name; + control->delete_outfile = delete; } -void register_outputfile(FILE *f) +void register_outputfile(rzip_control *control, FILE *f) { - outputfile = f; + control->outputfile = f; } -void unlink_files(void) +void unlink_files(rzip_control *control) { /* Delete temporary files generated for testing or faking stdio */ - if (outfile && delete_outfile) - unlink(outfile); + if (control->util_outfile && control->delete_outfile) + unlink(control->util_outfile); - if (infile && delete_infile) - unlink(infile); + if (control->util_infile && control->delete_infile) + unlink(control->util_infile); } -static void fatal_exit(void) +void fatal_exit(rzip_control *control) { struct termios termios_p; @@ -99,38 +99,47 @@ static void fatal_exit(void) termios_p.c_lflag |= ECHO; tcsetattr(fileno(stdin), 0, &termios_p); - unlink_files(); - fprintf(outputfile, "Fatal error - exiting\n"); - fflush(outputfile); + unlink_files(control); + fprintf(control->outputfile, "Fatal error - exiting\n"); + fflush(control->outputfile); exit(1); } -/* Failure when there is likely to be a meaningful error in perror */ -void fatal(const char *format, ...) +void setup_overhead(rzip_control *control) { - va_list ap; - - if (format) { - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); - } - - perror(NULL); - fatal_exit(); + /* Work out the compression overhead per compression thread for the + * compression back-ends that need a lot of ram */ + if (LZMA_COMPRESS) { + int level = control->compression_level * 7 / 9 ? : 1; + i64 dictsize = (level <= 5 ? (1 << (level * 2 + 14)) : + (level == 6 ? (1 << 25) : (1 << 26))); + + control->overhead = (dictsize * 23 / 2) + (4 * 1024 * 1024); + } else if (ZPAQ_COMPRESS) + control->overhead = 112 * 1024 * 1024; } -void failure(const char *format, ...) +void setup_ram(rzip_control *control) { - va_list ap; - - if (format) { - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); - } - - fatal_exit(); + /* Use less ram when using STDOUT to store the temporary output file. */ + if (STDOUT && ((STDIN && DECOMPRESS) || !(DECOMPRESS || TEST_ONLY))) + control->maxram = control->ramsize * 2 / 9; + else + control->maxram = control->ramsize / 3; + if (BITS32) { + /* Decrease usable ram size on 32 bits due to kernel / + * userspace split. Cannot allocate larger than a 1 + * gigabyte chunk due to 32 bit signed long being + * used in alloc, and at most 3GB can be malloced, and + * 2/3 of that makes for a total of 2GB to be split + * into thirds. + */ + control->usable_ram = MAX(control->ramsize - 900000000ll, 900000000ll); + control->maxram = MIN(control->maxram, control->usable_ram); + control->maxram = MIN(control->maxram, one_g * 2 / 3); + } else + control->usable_ram = control->maxram; + round_to_page(&control->maxram); } void round_to_page(i64 *size) @@ -140,7 +149,7 @@ void round_to_page(i64 *size) *size = PAGE_SIZE; } -void get_rand(uchar *buf, int len) +bool get_rand(rzip_control *control, uchar *buf, int len) { int fd, i; @@ -150,10 +159,157 @@ void get_rand(uchar *buf, int len) buf[i] = (uchar)random(); } else { if (unlikely(read(fd, buf, len) != len)) - fatal("Failed to read fd in get_rand\n"); + fatal_return(("Failed to read fd in get_rand\n"), false); if (unlikely(close(fd))) - fatal("Failed to close fd in get_rand\n"); + fatal_return(("Failed to close fd in get_rand\n"), false); + } + return true; +}bool read_config(rzip_control *control) +{ + /* check for lrzip.conf in ., $HOME/.lrzip and /etc/lrzip */ + char *HOME, homeconf[255]; + char *parametervalue; + char *parameter; + char line[255]; + FILE *fp; + + fp = fopen("lrzip.conf", "r"); + if (fp) + fprintf(control->msgout, "Using configuration file ./lrzip.conf\n"); + if (fp == NULL) { + fp = fopen("/etc/lrzip/lrzip.conf", "r"); + if (fp) + fprintf(control->msgout, "Using configuration file /etc/lrzip/lrzip.conf\n"); } + if (fp == NULL) { + HOME=getenv("HOME"); + if (HOME) { + snprintf(homeconf, sizeof(homeconf), "%s/.lrzip/lrzip.conf", HOME); + fp = fopen(homeconf, "r"); + if (fp) + fprintf(control->msgout, "Using configuration file %s\n", homeconf); + } + } + if (fp == NULL) return true; + + /* if we get here, we have a file. read until no more. */ + + while ((fgets(line, 255, fp)) != NULL) { + if (strlen(line)) + line[strlen(line) - 1] = '\0'; + parameter = strtok(line, " ="); + if (parameter == NULL) + continue; + /* skip if whitespace or # */ + if (isspace(*parameter)) + continue; + if (*parameter == '#') + continue; + + parametervalue = strtok(NULL, " ="); + if (parametervalue == NULL) + continue; + + /* have valid parameter line, now assign to control */ + + if (isparameter(parameter, "window")) + control->window = atoi(parametervalue); + else if (isparameter(parameter, "unlimited")) { + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_UNLIMITED; + } else if (isparameter(parameter, "compressionlevel")) { + control->compression_level = atoi(parametervalue); + if ( control->compression_level < 1 || control->compression_level > 9 ) + failure_return(("CONF.FILE error. Compression Level must between 1 and 9"), false); + } else if (isparameter(parameter, "compressionmethod")) { + /* valid are rzip, gzip, bzip2, lzo, lzma (default), and zpaq */ + if (control->flags & FLAG_NOT_LZMA) + failure_return(("CONF.FILE error. Can only specify one compression method"), false); + if (isparameter(parametervalue, "bzip2")) + control->flags |= FLAG_BZIP2_COMPRESS; + else if (isparameter(parametervalue, "gzip")) + control->flags |= FLAG_ZLIB_COMPRESS; + else if (isparameter(parametervalue, "lzo")) + control->flags |= FLAG_LZO_COMPRESS; + else if (isparameter(parametervalue, "rzip")) + control->flags |= FLAG_NO_COMPRESS; + else if (isparameter(parametervalue, "zpaq")) + control->flags |= FLAG_ZPAQ_COMPRESS; + else if (!isparameter(parametervalue, "lzma")) /* oops, not lzma! */ + failure_return(("CONF.FILE error. Invalid compression method %s specified\n",parametervalue), false); + } else if (isparameter(parameter, "lzotest")) { + /* default is yes */ + if (isparameter(parametervalue, "no")) + control->flags &= ~FLAG_THRESHOLD; + } else if (isparameter(parameter, "hashcheck")) { + if (isparameter(parametervalue, "yes")) { + control->flags |= FLAG_CHECK; + control->flags |= FLAG_HASH; + } + } else if (isparameter(parameter, "showhash")) { + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_HASH; + } else if (isparameter(parameter, "outputdirectory")) { + control->outdir = malloc(strlen(parametervalue) + 2); + if (!control->outdir) + fatal_return(("Fatal Memory Error in read_config"), false); + strcpy(control->outdir, parametervalue); + if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) + strcat(control->outdir, "/"); + } else if (isparameter(parameter,"verbosity")) { + if (control->flags & FLAG_VERBOSE) + failure_return(("CONF.FILE error. Verbosity already defined."), false); + if (isparameter(parametervalue, "yes")) + control->flags |= FLAG_VERBOSITY; + else if (isparameter(parametervalue,"max")) + control->flags |= FLAG_VERBOSITY_MAX; + else /* oops, unrecognized value */ + print_err("lrzip.conf: Unrecognized verbosity value %s. Ignored.\n", parametervalue); + } else if (isparameter(parameter, "showprogress")) { + /* Yes by default */ + if (isparameter(parametervalue, "NO")) + control->flags &= ~FLAG_SHOW_PROGRESS; + } else if (isparameter(parameter,"nice")) { + control->nice_val = atoi(parametervalue); + if (control->nice_val < -20 || control->nice_val > 19) + failure_return(("CONF.FILE error. Nice must be between -20 and 19"), false); + } else if (isparameter(parameter, "keepbroken")) { + if (isparameter(parametervalue, "yes" )) + control->flags |= FLAG_KEEP_BROKEN; + } else if (iscaseparameter(parameter, "DELETEFILES")) { + /* delete files must be case sensitive */ + if (iscaseparameter(parametervalue, "YES")) + control->flags &= ~FLAG_KEEP_FILES; + } else if (iscaseparameter(parameter, "REPLACEFILE")) { + /* replace lrzip file must be case sensitive */ + if (iscaseparameter(parametervalue, "YES")) + control->flags |= FLAG_FORCE_REPLACE; + } else if (isparameter(parameter, "tmpdir")) { + control->tmpdir = realloc(NULL, strlen(parametervalue) + 2); + if (!control->tmpdir) + fatal_return(("Fatal Memory Error in read_config"), false); + strcpy(control->tmpdir, parametervalue); + if (strcmp(parametervalue + strlen(parametervalue) - 1, "/")) + strcat(control->tmpdir, "/"); + } else if (isparameter(parameter, "encrypt")) { + if (isparameter(parameter, "YES")) + control->flags |= FLAG_ENCRYPT; + } else + /* oops, we have an invalid parameter, display */ + print_err("lrzip.conf: Unrecognized parameter value, %s = %s. Continuing.\n",\ + parameter, parametervalue); + } + + if (unlikely(fclose(fp))) + fatal_return(("Failed to fclose fp in read_config\n"), false); + +/* fprintf(stderr, "\nWindow = %d \ + \nCompression Level = %d \ + \nThreshold = %1.2f \ + \nOutput Directory = %s \ + \nFlags = %d\n", control->window,control->compression_level, control->threshold, control->outdir, control->flags); +*/ + return true; } static void xor128 (void *pa, const void *pb) @@ -184,7 +340,7 @@ static void lrz_keygen(const rzip_control *control, const uchar *salt, uchar *ke munlock(buf, sizeof(buf)); } -void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt) +bool lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt) { /* Encryption requires CBC_LEN blocks so we can use ciphertext * stealing to not have to pad the block */ @@ -192,6 +348,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa uchar tmp0[CBC_LEN], tmp1[CBC_LEN]; aes_context aes_ctx; i64 N, M; + bool ret = false; /* Generate unique key and IV for each block of data based on salt */ mlock(&aes_ctx, sizeof(aes_ctx)); @@ -206,9 +363,9 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa if (encrypt == LRZ_ENCRYPT) { print_maxverbose("Encrypting data \n"); if (unlikely(aes_setkey_enc(&aes_ctx, key, 128))) - failure("Failed to aes_setkey_enc in lrz_crypt\n"); + failure_goto(("Failed to aes_setkey_enc in lrz_crypt\n"), error); aes_crypt_cbc(&aes_ctx, AES_ENCRYPT, N, iv, buf, buf); - + if (M) { memset(tmp0, 0, CBC_LEN); memcpy(tmp0, buf + N, M); @@ -219,7 +376,7 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa } } else { if (unlikely(aes_setkey_dec(&aes_ctx, key, 128))) - failure("Failed to aes_setkey_dec in lrz_crypt\n"); + failure_goto(("Failed to aes_setkey_dec in lrz_crypt\n"), error); print_maxverbose("Decrypting data \n"); if (M) { aes_crypt_cbc(&aes_ctx, AES_DECRYPT, N - CBC_LEN, @@ -239,12 +396,15 @@ void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *sa iv, buf, buf); } + ret = true; +error: memset(&aes_ctx, 0, sizeof(aes_ctx)); memset(iv, 0, HASH_LEN); memset(key, 0, HASH_LEN); munlock(&aes_ctx, sizeof(aes_ctx)); munlock(iv, HASH_LEN); munlock(key, HASH_LEN); + return ret; } void lrz_stretch(rzip_control *control) diff --git a/util.h b/util.h index 74fe8199..91932405 100644 --- a/util.h +++ b/util.h @@ -20,30 +20,85 @@ #define LRZIP_UTIL_H #include "lrzip_private.h" +#include -void register_infile(const char *name, char delete); -void register_outfile(const char *name, char delete); -void unlink_files(void); -void register_outputfile(FILE *f); -void fatal(const char *format, ...); -void failure(const char *format, ...); +void register_infile(rzip_control *control, const char *name, char delete); +void register_outfile(rzip_control *control, const char *name, char delete); +void unlink_files(rzip_control *control); +void register_outputfile(rzip_control *control, FILE *f); +void fatal_exit(rzip_control *control); +/* Failure when there is likely to be a meaningful error in perror */ +static inline void fatal(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + if (!control->log_cb) { + vfprintf(stderr, format, ap); + perror(NULL); + } else + control->log_cb(control->log_data, 0, line, file, func, format, ap); + va_end(ap); + if (!control->library_mode) + fatal_exit((rzip_control*)control); +} +#ifdef fatal +# undef fatal +#endif +#define fatal(stuff...) fatal(control, __LINE__, __FILE__, __func__, stuff) +#define fatal_return(stuff, ...) do { \ + fatal stuff; \ + return __VA_ARGS__; \ +} while (0) +#define fatal_goto(stuff, label) do { \ + fatal stuff; \ + goto label; \ +} while (0) +static inline void failure(const rzip_control *control, unsigned int line, const char *file, const char *func, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + if (!control->log_cb) + vfprintf(stderr, format, ap); + else + control->log_cb(control->log_data, 0, line, file, func, format, ap); + va_end(ap); + if (!control->library_mode) + fatal_exit((rzip_control*)control); +} +#ifdef failure +# undef failure +#endif +#define failure(stuff...) failure(control, __LINE__, __FILE__, __func__, stuff) +#define failure_return(stuff, ...) do { \ + failure stuff; \ + return __VA_ARGS__; \ +} while (0) +#define failure_goto(stuff, label) do { \ + failure stuff; \ + goto label; \ +} while (0) +void setup_overhead(rzip_control *control); +void setup_ram(rzip_control *control); void round_to_page(i64 *size); -void get_rand(uchar *buf, int len); +bool get_rand(rzip_control *control, uchar *buf, int len); +bool read_config(rzip_control *control); void lrz_stretch(rzip_control *control); void lrz_stretch2(rzip_control *control); -void lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); +bool lrz_crypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt, int encrypt); #define LRZ_DECRYPT (0) #define LRZ_ENCRYPT (1) -static inline void lrz_encrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) +static inline bool lrz_encrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) { - lrz_crypt(control, buf, len, salt, LRZ_ENCRYPT); + return lrz_crypt(control, buf, len, salt, LRZ_ENCRYPT); } -static inline void lrz_decrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) +static inline bool lrz_decrypt(const rzip_control *control, uchar *buf, i64 len, const uchar *salt) { - lrz_crypt(control, buf, len, salt, LRZ_DECRYPT); + return lrz_crypt(control, buf, len, salt, LRZ_DECRYPT); } #endif