yank ppmd_sh

.gitmodules

@@ -9,7 +9,4 @@
 	url = https://github.com/facebook/zstd
 [submodule "vendor/bzip3"]
 	path = vendor/bzip3
-	url = https://github.com/kspalaiologos/bzip3
-[submodule "vendor/ppmd_sh"]
-	path = vendor/ppmd_sh
-	url = https://github.com/Shelwien/ppmd_sh
+	url = https://github.com/kspalaiologos/bzip3

AUTHORS

@@ -8,7 +8,6 @@ Conor McCarthy for Fast LZMA2
 Serge Belyshev for encryption help and code
 Ulrich Drepper for MD5 implementation
 PolarSSL authors for sha512 + aes128 implementation
-Dymitry Shkarin and Eugene Shelwien for PPMD_sh varJr1
 Kamila Szewczyk for bzip3
 Yann Collet for lz4 and zstandard
 Matt Mahoney for zpaq

Makefile.in

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 CONFIG=-Ivendor/zpaq -Ivendor/lz4/lib -Ivendor/zstd/lib -Ivendor/fast-lzma2 \
-       -Ivendor/bzip3/include -Ivendor/ppmd_sh -Ivendor/ppmd_sh/libpmd -Iinclude
+       -Ivendor/bzip3/include -Iinclude
 PROGRAM=mrzip
 
 all:

README.md

@@ -1,7 +1,7 @@
 modern-rzip.
 ======================
 
-A backup suite. Supports FLZMA2, PPMD, bzip3, LZ4, Zstandard, LSH i-node ordering deduplicating archiver, long range deduplication, encryption and recovery records. Also a refurbished partial rewrite of lrzip in active development.
+A backup suite. Supports FLZMA2, bzip3, LZ4, Zstandard, LSH i-node ordering deduplicating archiver, long range deduplication, encryption and recovery records. Also a refurbished partial rewrite of lrzip in active development.
 
 ### Download, build and install
 ```
@@ -28,7 +28,6 @@ significantly faster compression than standalone compressors and much faster dec
 **modern-rzip**'s compressors are:
 * Fast LZMA2 (default)
 * Zstandard
-* PPMd_sh based on vJr1
 * LZ4
 * zpaq
 * bzip3

include/mrzip_core.h

@@ -48,7 +48,5 @@ extern void zpaq_compress(uchar * c_buf, i64 * c_len, uchar * s_buf, i64 s_len,
                           bool progress, int thread);
 extern void zpaq_decompress(uchar * s_buf, i64 * d_len, uchar * c_buf, i64 c_len, FILE * msgout, bool progress,
                             int thread);
-int ppmdsh_varjr1_compress(char * input, int input_len, char * output, int * output_size, unsigned level);
-int ppmdsh_varjr1_decompress(char * input, int input_len, char * output, int * output_size);
 
 #endif

include/mrzip_private.h

@@ -250,7 +250,7 @@ static inline unsigned char bzip3_prop_from_block_size(u32 bs) {
 #define FLAG_DECOMPRESS (1 << 4)
 #define FLAG_NO_COMPRESS (1 << 5)
 #define FLAG_LZ4_COMPRESS (1 << 6)
-#define FLAG_PPM_COMPRESS (1 << 7)
+#define FLAG_UNUSED_COMPRESS (1 << 7)
 #define FLAG_ZSTD_COMPRESS (1 << 8)
 #define FLAG_ZPAQ_COMPRESS (1 << 9)
 #define FLAG_VERBOSITY (1 << 10)
@@ -272,7 +272,7 @@ static inline unsigned char bzip3_prop_from_block_size(u32 bs) {
 #define NO_HASH (!(HASH_CHECK) && !(HAS_HASH))
 
 #define CTYPE_NONE 3
-#define CTYPE_PPM 4
+#define CTYPE_UNUSED 4
 #define CTYPE_LZ4 5
 #define CTYPE_LZMA 6
 #define CTYPE_ZSTD 7
@@ -321,7 +321,7 @@ static inline unsigned char bzip3_prop_from_block_size(u32 bs) {
 
 #define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
 #define FLAG_NOT_LZMA                                                                                     \
-    (FLAG_NO_COMPRESS | FLAG_LZ4_COMPRESS | FLAG_PPM_COMPRESS | FLAG_ZSTD_COMPRESS | FLAG_ZPAQ_COMPRESS | \
+    (FLAG_NO_COMPRESS | FLAG_LZ4_COMPRESS | FLAG_UNUSED_COMPRESS | FLAG_ZSTD_COMPRESS | FLAG_ZPAQ_COMPRESS | \
      FLAG_BZIP3_COMPRESS)
 #define LZMA_COMPRESS (!(control->flags & FLAG_NOT_LZMA))
 
@@ -332,7 +332,7 @@ static inline unsigned char bzip3_prop_from_block_size(u32 bs) {
 #define DECOMPRESS (control->flags & FLAG_DECOMPRESS)
 #define NO_COMPRESS (control->flags & FLAG_NO_COMPRESS)
 #define LZ4_COMPRESS (control->flags & FLAG_LZ4_COMPRESS)
-#define PPM_COMPRESS (control->flags & FLAG_PPM_COMPRESS)
+#define UNUSED_COMPRESS (control->flags & FLAG_UNUSED_COMPRESS)
 #define ZSTD_COMPRESS (control->flags & FLAG_ZSTD_COMPRESS)
 #define ZPAQ_COMPRESS (control->flags & FLAG_ZPAQ_COMPRESS)
 #define BZIP3_COMPRESS (control->flags & FLAG_BZIP3_COMPRESS)

src/main.c

@@ -100,7 +100,6 @@ static void usage(void) {
                  "Compression Options:\n"
                  "--------------------\n"
                  "	--lzma			lzma compression (default)\n"
-                 "	-s, --ppmdsh		ppmd_sh varJr1 compression\n"
                  "	-Z, --zstd		zstd compression using zlib\n"
                  "	-l, --lz4		lz4 compression (ultra fast)\n"
                  "	-n, --no-compress	no backend compression - prepare for other compressor\n"
@@ -181,8 +180,6 @@ static char * compression_type(void) {
         return "LZMA";
     else if (LZ4_COMPRESS)
         return "LZ4";
-    else if (PPM_COMPRESS)
-        return "PPM";
     else if (ZSTD_COMPRESS)
         return "ZSTD";
     else if (ZPAQ_COMPRESS)
@@ -253,7 +250,6 @@ static void show_summary(void) {
 }
 
 static struct option long_options[] = {
-    { "ppmdsh", no_argument, 0, 's' }, /* 0 */
     { "check", no_argument, 0, 'c' },
     { "comment", required_argument, 0, 'C' },
     { "decompress", no_argument, 0, 'd' },
@@ -372,9 +368,7 @@ int main(int argc, char * argv[]) {
                     fatal("Can only use one of -l, -b, -g, -z, -B or -n\n");
                 /* Select Compression Mode */
                 control->flags &= ~FLAG_NOT_LZMA; /* must clear all compressions first */
-                if (c == 's')
-                    control->flags |= FLAG_PPM_COMPRESS;
-                else if (c == 'Z')
+                if (c == 'Z')
                     control->flags |= FLAG_ZSTD_COMPRESS;
                 else if (c == 'l')
                     control->flags |= FLAG_LZ4_COMPRESS;

src/mrzip.c

@@ -883,8 +883,6 @@ bool get_fileinfo(rzip_control * control) {
             if (INFO) print_verbose("%'d\t", block);
             if (ctype == CTYPE_NONE) {
                 if (INFO) print_verbose("none");
-            } else if (ctype == CTYPE_PPM) {
-                if (INFO) print_verbose("ppmdsh");
             } else if (ctype == CTYPE_LZ4) {
                 if (INFO) print_verbose("lz4");
             } else if (ctype == CTYPE_LZMA) {
@@ -962,8 +960,6 @@ bool get_fileinfo(rzip_control * control) {
         print_output("Compression Method: ");
         if (save_ctype == CTYPE_NONE)
             print_output("rzip alone\n");
-        else if (save_ctype == CTYPE_PPM)
-            print_output("rzip + ppmdsh\n");
         else if (save_ctype == CTYPE_LZ4)
             print_output("rzip + lz4\n");
         else if (save_ctype == CTYPE_LZMA) {

src/stream.c

@@ -211,44 +211,6 @@ static int zpaq_compress_buf(rzip_control * control, struct compress_thread * ct
     return 0;
 }
 
-static int ppmdsh_compress_buf(rzip_control * control, struct compress_thread * cthread) {
-    u32 dlen = round_up_page(control, cthread->s_len + 10000);
-    int ppmdsh_ret;
-    uchar * c_buf;
-
-    if (LZ4_TEST) {
-        if (!lz4_compresses(control, cthread->s_buf, cthread->s_len)) return 0;
-    }
-
-    c_buf = malloc(dlen);
-    if (!c_buf) {
-        print_err("Unable to allocate c_buf in ppmdsh_compress_buf\n");
-        return -1;
-    }
-
-    // TODO: mapping between mem and compression lvl.
-    int res = ppmdsh_varjr1_compress(cthread->s_buf, cthread->s_len, c_buf, &dlen, control->compression_level);
-
-    if (unlikely(res != 0)) {
-        dealloc(c_buf);
-        print_maxverbose("PPMD_sh varJr1 compress failed\n");
-        return -1;
-    }
-
-    if (unlikely(dlen >= cthread->c_len)) {
-        print_maxverbose("Incompressible block\n");
-        /* Incompressible, leave as CTYPE_NONE */
-        dealloc(c_buf);
-        return 0;
-    }
-
-    cthread->c_len = dlen;
-    dealloc(cthread->s_buf);
-    cthread->s_buf = c_buf;
-    cthread->c_type = CTYPE_PPM;
-    return 0;
-}
-
 static int zstd_compress_buf(rzip_control * control, struct compress_thread * cthread) {
     unsigned long dlen = round_up_page(control, ZSTD_COMPRESSBOUND(cthread->s_len));
     uchar * c_buf;
@@ -416,41 +378,6 @@ static int zpaq_decompress_buf(rzip_control * control, struct uncomp_thread * uc
     return ret;
 }
 
-static int ppmdsh_decompress_buf(rzip_control * control, struct uncomp_thread * ucthread) {
-    u32 dlen = ucthread->u_len + 10;
-    int ret = 0, bzerr;
-    uchar * c_buf;
-
-    c_buf = ucthread->s_buf;
-    ucthread->s_buf = malloc(round_up_page(control, dlen));
-    if (unlikely(!ucthread->s_buf)) {
-        print_err("Failed to allocate %'d bytes for decompression\n", dlen);
-        ret = -1;
-        goto out;
-    }
-
-    bzerr = ppmdsh_varjr1_decompress(c_buf, ucthread->c_len, ucthread->s_buf, &dlen);
-
-    if (unlikely(bzerr != 0)) {
-        print_err("Failed to decompress buffer - bzerr=%'d\n", bzerr);
-        ret = -1;
-        goto out;
-    }
-
-    if (unlikely(dlen != ucthread->u_len)) {
-        print_err("Inconsistent length after decompression. Got %'d bytes, expected %'" PRId64 "\n", dlen,
-                  ucthread->u_len);
-        ret = -1;
-    } else
-        dealloc(c_buf);
-out:
-    if (ret == -1) {
-        dealloc(ucthread->s_buf);
-        ucthread->s_buf = c_buf;
-    }
-    return ret;
-}
-
 static int zstd_decompress_buf(rzip_control * control, struct uncomp_thread * ucthread) {
     unsigned long dlen = ucthread->u_len;
     int ret = 0;
@@ -977,8 +904,6 @@ void * open_stream_out(rzip_control * control, int f, unsigned int n, i64 chunk_
             stream_bufsize = round_up_page(control, (0x100000 << control->zpaq_bs) - 0x1000);
         else if (BZIP3_COMPRESS && (limit / control->threads > control->bzip3_block_size))
             stream_bufsize = round_up_page(control, control->bzip3_block_size - 0x1000);
-        else if (PPM_COMPRESS && (limit / control->threads > 0x100000 << control->bzip3_bs))
-            stream_bufsize = round_up_page(control, (0x100000 << control->bzip3_bs) - 0x1000);
         else if (LZMA_COMPRESS && limit / control->threads > STREAM_BUFSIZE)
             // for smaller dictionary sizes, need MAX test to bring in larger buffer from limit
             // limit = usable ram / 2
@@ -1225,8 +1150,6 @@ static void * compthread(void * data) {
             ret = lzma_compress_buf(control, cti, current_thread);
         else if (LZ4_COMPRESS)
             ret = lz4_compress_buf(control, cti);
-        else if (PPM_COMPRESS)
-            ret = ppmdsh_compress_buf(control, cti);
         else if (ZSTD_COMPRESS)
             ret = zstd_compress_buf(control, cti);
         else if (ZPAQ_COMPRESS)
@@ -1448,9 +1371,6 @@ static void * ucompthread(void * data) {
             case CTYPE_LZ4:
                 ret = lz4_decompress_buf(control, uci);
                 break;
-            case CTYPE_PPM:
-                ret = ppmdsh_decompress_buf(control, uci);
-                break;
             case CTYPE_ZSTD:
                 ret = zstd_decompress_buf(control, uci);
                 break;

src/util.c

@@ -148,8 +148,6 @@ void setup_overhead(rzip_control * control) {
         }
         control->bzip3_block_size = BZIP3_BLOCK_SIZE_FROM_PROP(control->bzip3_bs);
         control->overhead = (i64)control->bzip3_block_size * 6;
-    } else if (PPM_COMPRESS) {
-        control->overhead = (i64)(1 << control->compression_level) * ONE_MB;
     }
 
     /* no need for zpaq computation here. do in open_stream_out() */
@@ -229,11 +227,9 @@ bool read_config(rzip_control * control) {
             if (control->rzip_compression_level < 1 || control->rzip_compression_level > 9)
                 fatal("CONF.FILE error. RZIP Compression Level must between 1 and 9\n");
         } else if (isparameter(parameter, "compressionmethod")) {
-            /* valid are rzip, zstd, ppmdsh, lz4, lzma (default), and zpaq */
+            /* valid are rzip, zstd, lz4, lzma (default), and zpaq */
             if (control->flags & FLAG_NOT_LZMA) fatal("CONF.FILE error. Can only specify one compression method\n");
-            if (isparameter(parametervalue, "ppmdsh"))
-                control->flags |= FLAG_PPM_COMPRESS;
-            else if (isparameter(parametervalue, "zstd"))
+            if (isparameter(parametervalue, "zstd"))
                 control->flags |= FLAG_ZSTD_COMPRESS;
             else if (isparameter(parametervalue, "lz4"))
                 control->flags |= FLAG_LZ4_COMPRESS;

vendor/cxx_glue.cpp

@@ -106,46 +106,3 @@ void libzpaq::error(const char * msg) {  // print message and exit
     fprintf(stderr, "ZPAQ Error: %s\n", msg);
     exit(1);
 }
-
-#include <gcrypt.h>
-
-#include "common.inc"
-#include "coro3b_fake.inc"
-#include "libpmd.inc"
-
-int g_getc(FILE * f, FILE * g) { return fgetc(f); }
-void g_putc(int c, FILE * f, FILE * g) { fputc(c, g); }
-
-extern "C" {
-int ppmdsh_varjr1_compress(char * input, int input_len, char * output, int * output_size, unsigned level) {
-    ALIGN(4096) pmd_codec C;
-    uint pmd_args[] = { 12 /* Order */, static_cast<uint>(1 << level) /* Memory */, 1 /* Restore */,
-                        static_cast<uint>(input_len) };
-    if (C.Init(0, pmd_args)) return -1;
-    C.f = fmemopen(input, input_len, "rb");
-    C.g = fmemopen(output, *output_size, "wb");
-    fputc(level, C.g);
-    C.coro_call(&C);
-    C.Quit();
-    fflush(C.g);
-    *output_size = ftell(C.g);
-    fclose(C.f);
-    fclose(C.g);
-    return 0;
-}
-
-int ppmdsh_varjr1_decompress(char * input, int input_len, char * output, int * output_size) {
-    ALIGN(4096) pmd_codec C;
-    uint pmd_args[] = { 12 /* Order */, static_cast<uint>((*input) << 1) /* Memory */, 1 /* Restore */, -1U };
-    if (C.Init(1, pmd_args)) return -1;
-    C.f = fmemopen(input + 1, input_len - 1, "rb");
-    C.g = fmemopen(output, *output_size, "wb");
-    C.coro_call(&C);
-    C.Quit();
-    fflush(C.g);
-    *output_size = ftell(C.g);
-    fclose(C.f);
-    fclose(C.g);
-    return 0;
-}
-}