From 71382a1d14dad58219e7f6634d2381fa30dab175 Mon Sep 17 00:00:00 2001 From: vurtun Date: Sun, 27 Mar 2022 18:56:37 +0200 Subject: [PATCH] fix off by one bug --- sdefl.h | 36 +++++++++---------- sinfl.h | 66 +++++++++++++++++----------------- tests/defl_test.c | 92 +++++++++++++++++++++++------------------------ 3 files changed, 97 insertions(+), 97 deletions(-) diff --git a/sdefl.h b/sdefl.h index 5db7676..f7adc4d 100644 --- a/sdefl.h +++ b/sdefl.h @@ -38,10 +38,10 @@ this file implementation in *one* C or C++ file to prevent collisions. | zlib 1.2.11 -1 | 72 MB/s | 307 MB/s | 42298774 | 42.30 | | zlib 1.2.11 -6 | 24 MB/s | 313 MB/s | 36548921 | 36.55 | | zlib 1.2.11 -9 | 20 MB/s | 314 MB/s | 36475792 | 36.48 | -| sdefl 1.0 -0 | 127 MB/s | 371 MB/s | 40004116 | 39.88 | -| sdefl 1.0 -1 | 111 MB/s | 398 MB/s | 38940674 | 38.82 | -| sdefl 1.0 -5 | 45 MB/s | 420 MB/s | 36577183 | 36.46 | -| sdefl 1.0 -7 | 38 MB/s | 423 MB/s | 36523781 | 36.41 | +| sdefl 1.0 -0 | 127 MB/s | 355 MB/s | 40004116 | 39.88 | +| sdefl 1.0 -1 | 111 MB/s | 413 MB/s | 38940674 | 38.82 | +| sdefl 1.0 -5 | 45 MB/s | 436 MB/s | 36577183 | 36.46 | +| sdefl 1.0 -7 | 38 MB/s | 432 MB/s | 36523781 | 36.41 | | libdeflate 1.3 -1 | 147 MB/s | 667 MB/s | 39597378 | 39.60 | | libdeflate 1.3 -6 | 69 MB/s | 689 MB/s | 36648318 | 36.65 | | libdeflate 1.3 -9 | 13 MB/s | 672 MB/s | 35197141 | 35.20 | @@ -50,20 +50,20 @@ this file implementation in *one* C or C++ file to prevent collisions. ### Compression Results on the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia): -| File | Original | `sdefl 0` | `sdefl 5` | `sdefl 7` | -| :------ | ---------: | -----------------: | ---------: | ----------: | -| dickens | 10.192.446 | 4,260,187| 3,845,261| 3,833,657 | -| mozilla | 51.220.480 | 20,774,706 | 19,607,009 | 19,565,867 | -| mr | 9.970.564 | 3,860,531 | 3,673,460 | 3,665,627 | -| nci | 33.553.445 | 4,030,283 | 3,094,526 | 3,006,075 | -| ooffice | 6.152.192 | 3,320,063 | 3,186,373 | 3,183,815 | -| osdb | 10.085.684 | 3,919,646 | 3,649,510 | 3,649,477 | -| reymont | 6.627.202 | 2,263,378 | 1,857,588 | 1,827,237 | -| samba | 21.606.400 | 6,121,797 | 5,462,670 | 5,450,762 | -| sao | 7.251.944 | 5,612,421 | 5,485,380 | 5,481,765 | -| webster | 41.458.703 | 13,972,648 | 12,059,432 | 11,991,421 | -| xml | 5.345.280 | 886,620| 674,009 | 662,141 | -| x-ray | 8.474.240 | 6,304,655 | 6,244,779 | 6,244,779 | +| File | Original | `sdefl 0` | `sdefl 5` | `sdefl 7` | +| --------| -----------| -------------| ---------- | ------------| +| dickens | 10.192.446 | 4,260,187 | 3,845,261 | 3,833,657 | +| mozilla | 51.220.480 | 20,774,706 | 19,607,009 | 19,565,867 | +| mr | 9.970.564 | 3,860,531 | 3,673,460 | 3,665,627 | +| nci | 33.553.445 | 4,030,283 | 3,094,526 | 3,006,075 | +| ooffice | 6.152.192 | 3,320,063 | 3,186,373 | 3,183,815 | +| osdb | 10.085.684 | 3,919,646 | 3,649,510 | 3,649,477 | +| reymont | 6.627.202 | 2,263,378 | 1,857,588 | 1,827,237 | +| samba | 21.606.400 | 6,121,797 | 5,462,670 | 5,450,762 | +| sao | 7.251.944 | 5,612,421 | 5,485,380 | 5,481,765 | +| webster | 41.458.703 | 13,972,648 | 12,059,432 | 11,991,421 | +| xml | 5.345.280 | 886,620 | 674,009 | 662,141 | +| x-ray | 8.474.240 | 6,304,655 | 6,244,779 | 6,244,779 | ## License ``` diff --git a/sinfl.h b/sinfl.h index 01f50f1..70d572e 100644 --- a/sinfl.h +++ b/sinfl.h @@ -221,7 +221,7 @@ sinfl_peek(struct sinfl *s, int cnt) { return s->bitbuf & ((1ull << cnt) - 1); } static void -sinfl_consume(struct sinfl *s, int cnt) { +sinfl_eat(struct sinfl *s, int cnt) { assert(cnt <= s->bitcnt); s->bitbuf >>= cnt; s->bitcnt -= cnt; @@ -229,7 +229,7 @@ sinfl_consume(struct sinfl *s, int cnt) { static int sinfl__get(struct sinfl *s, int cnt) { int res = sinfl_peek(s, cnt); - sinfl_consume(s, cnt); + sinfl_eat(s, cnt); return res; } static int @@ -284,7 +284,7 @@ sinfl_build_subtbl(struct sinfl_gen *gen, unsigned *tbl, int tbl_bits, while (1) { unsigned entry; int bit, stride, i; - /* start new subtable */ + /* start new sub-table */ if ((gen->word & ((1 << tbl_bits)-1)) != sub_prefix) { int used = 0; sub_prefix = gen->word & ((1 << tbl_bits)-1); @@ -298,7 +298,7 @@ sinfl_build_subtbl(struct sinfl_gen *gen, unsigned *tbl, int tbl_bits, tbl_end = sub_start + (1 << sub_bits); tbl[sub_prefix] = (sub_start << 16) | 0x10 | (sub_bits & 0xf); } - /* fill subtable */ + /* fill sub-table */ entry = (*gen->sorted << 16) | ((gen->len - tbl_bits) & 0xf); gen->sorted++; i = sub_start + (gen->word >> tbl_bits); @@ -352,17 +352,17 @@ sinfl_build(unsigned *tbl, unsigned char *lens, int tbl_bits, int maxlen, } static int sinfl_decode(struct sinfl *s, const unsigned *tbl, int bit_len) { - {int idx = sinfl_peek(s, bit_len); + int idx = sinfl_peek(s, bit_len); unsigned key = tbl[idx]; if (key & 0x10) { /* sub-table lookup */ int len = key & 0x0f; - sinfl_consume(s, bit_len); + sinfl_eat(s, bit_len); idx = sinfl_peek(s, len); key = tbl[((key >> 16) & 0xffff) + (unsigned)idx]; } - sinfl_consume(s, key & 0x0f); - return (key >> 16) & 0x0fff;} + sinfl_eat(s, key & 0x0f); + return (key >> 16) & 0x0fff; } static int sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size) { @@ -428,32 +428,32 @@ sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size) state = blk; } break; case dyn: { - /* dynamic huffman codes */ - int n, i; - unsigned hlens[SINFL_PRE_TBL_SIZE]; - unsigned char nlens[19] = {0}, lens[288+32]; + /* dynamic huffman codes */ + int n, i; + unsigned hlens[SINFL_PRE_TBL_SIZE]; + unsigned char nlens[19] = {0}, lens[288+32]; + sinfl_refill(&s); + {int nlit = 257 + sinfl__get(&s,5); + int ndist = 1 + sinfl__get(&s,5); + int nlen = 4 + sinfl__get(&s,4); + for (n = 0; n < nlen; n++) + nlens[order[n]] = (unsigned char)sinfl_get(&s,3); + sinfl_build(hlens, nlens, 7, 7, 19); + + /* decode code lengths */ + for (n = 0; n < nlit + ndist;) { sinfl_refill(&s); - {int nlit = 257 + sinfl__get(&s,5); - int ndist = 1 + sinfl__get(&s,5); - int nlen = 4 + sinfl__get(&s,4); - for (n = 0; n < nlen; n++) - nlens[order[n]] = (unsigned char)sinfl_get(&s,3); - sinfl_build(hlens, nlens, 7, 7, 19); - - /* decode code lengths */ - for (n = 0; n < nlit + ndist;) { - sinfl_refill(&s); - int sym = sinfl_decode(&s, hlens, 7); - switch (sym) {default: lens[n++] = (unsigned char)sym; break; - case 16: for (i=3+sinfl_get(&s,2);i;i--,n++) lens[n]=lens[n-1]; break; - case 17: for (i=3+sinfl_get(&s,3);i;i--,n++) lens[n]=0; break; - case 18: for (i=11+sinfl_get(&s,7);i;i--,n++) lens[n]=0; break;} - } - /* build lit/dist tables */ - sinfl_build(s.lits, lens, 10, 15, nlit); - sinfl_build(s.dsts, lens + nlit, 8, 15, ndist); - state = blk;} + int sym = sinfl_decode(&s, hlens, 7); + switch (sym) {default: lens[n++] = (unsigned char)sym; break; + case 16: for (i=3+sinfl_get(&s,2);i;i--,n++) lens[n]=lens[n-1]; break; + case 17: for (i=3+sinfl_get(&s,3);i;i--,n++) lens[n]=0; break; + case 18: for (i=11+sinfl_get(&s,7);i;i--,n++) lens[n]=0; break;} + } + /* build lit/dist tables */ + sinfl_build(s.lits, lens, 10, 15, nlit); + sinfl_build(s.dsts, lens + nlit, 8, 15, ndist); + state = blk;} } break; case blk: { /* decompress block */ @@ -462,7 +462,7 @@ sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size) int sym = sinfl_decode(&s, s.lits, 10); if (sym < 256) { /* literal */ - if (sinfl_unlikely(out + 1 >= oe)) { + if (sinfl_unlikely(out >= oe)) { return (int)(out-o); } *out++ = (unsigned char)sym; diff --git a/tests/defl_test.c b/tests/defl_test.c index a25dd0c..7d992e4 100644 --- a/tests/defl_test.c +++ b/tests/defl_test.c @@ -39,59 +39,59 @@ static struct sdefl sdefl; int main(int argc, char **argv) { - void *comp = 0; - size_t size = 0; - struct timespec compr_started; - struct timespec compr_ended; - struct timespec decompr_started; - struct timespec decompr_ended; - double compr_mbs = 0; - double decompr_mbs = 0; - void *decomp; - double ratio = 0.0f; - int len = 0, n, lvl; + void *comp = 0; + size_t size = 0; + struct timespec compr_started; + struct timespec compr_ended; + struct timespec decompr_started; + struct timespec decompr_ended; + double compr_mbs = 0; + double decompr_mbs = 0; + void *decomp; + double ratio = 0.0f; + int len = 0, n, lvl; - void *data = 0; - { - FILE* fp = fopen(argv[1], "rb"); - if (!fp) exit(2); - fseek(fp, 0, SEEK_END); - size = (size_t)ftell(fp); - fseek(fp, 0, SEEK_SET); - data = calloc(size, 1); - fread(data, 1, (size_t)size, fp); - fclose(fp); - } + void *data = 0; + { + FILE* fp = fopen(argv[1], "rb"); + if (!fp) exit(2); + fseek(fp, 0, SEEK_END); + size = (size_t)ftell(fp); + fseek(fp, 0, SEEK_SET); + data = calloc(size, 1); + fread(data, 1, (size_t)size, fp); + fclose(fp); + } - comp = calloc(size*2, 1); - decomp = calloc(size, 1); - for (lvl = SDEFL_LVL_MIN; lvl < SDEFL_LVL_MAX; ++lvl) { - clock_gettime(CLOCK_REALTIME, &compr_started); - len = sdeflate(&sdefl, comp, data, (int)size, lvl); - clock_gettime(CLOCK_REALTIME, &compr_ended); + comp = calloc(size*2, 1); + decomp = calloc(size, 1); + for (lvl = SDEFL_LVL_MIN; lvl < SDEFL_LVL_MAX; ++lvl) { + clock_gettime(CLOCK_REALTIME, &compr_started); + len = sdeflate(&sdefl, comp, data, (int)size, lvl); + clock_gettime(CLOCK_REALTIME, &compr_ended); - clock_gettime(CLOCK_REALTIME, &decompr_started); - n = sinflate(decomp, (int)size, comp, len); - clock_gettime(CLOCK_REALTIME, &decompr_ended); + clock_gettime(CLOCK_REALTIME, &decompr_started); + n = sinflate(decomp, (int)size, comp, len); + clock_gettime(CLOCK_REALTIME, &decompr_ended); - double compr_time = profiler_time(compr_started, compr_ended); - double decompr_time = profiler_time(decompr_started, decompr_ended); + double compr_time = profiler_time(compr_started, compr_ended); + double decompr_time = profiler_time(decompr_started, decompr_ended); - ratio = (double)len / (double)size * 100.0; - compr_mbs = ((double)size / 1000000.0f) / (compr_time / 1000.0f); - decompr_mbs = ((double)size / 1000000.0f) / (decompr_time / 1000.0f); + ratio = (double)len / (double)size * 100.0; + compr_mbs = ((double)size / 1000000.0f) / (compr_time / 1000.0f); + decompr_mbs = ((double)size / 1000000.0f) / (decompr_time / 1000.0f); - printf("lvl: %d size: %d, compr: %.2fms (%.2fMB/s) decompr: %.2fms (%.2fMB/s) ratio: %f%% ", lvl, len, compr_time, compr_mbs, decompr_time, decompr_mbs, ratio); - if ((size_t)n != size || memcmp(data, decomp, (size_t)size)) { - FILE *fd = fopen("error.bin", "wb"); - fwrite(decomp, 1, (size_t)n, fd); - fclose(fd); + printf("lvl: %d size: %d, compr: %.2fms (%.2fMB/s) decompr: %.2fms (%.2fMB/s) ratio: %f%% ", lvl, len, compr_time, compr_mbs, decompr_time, decompr_mbs, ratio); + if ((size_t)n != size || memcmp(data, decomp, (size_t)size)) { + FILE *fd = fopen("error.bin", "wb"); + fwrite(decomp, 1, (size_t)n, fd); + fclose(fd); - printf("error: %d:%d !\n", n, (int)size); - break; - } - else printf("\n"); + printf("error: %d:%d !\n", n, (int)size); + break; } - return 0; + else printf("\n"); + } + return 0; }