Skip to content

Commit

Permalink
fix off by one bug
Browse files Browse the repository at this point in the history
  • Loading branch information
@vurtun
vurtun committed Mar 27, 2022
1 parent 8252ab9 commit 71382a1
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 97 deletions.
36 changes: 18 additions & 18 deletions sdefl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ this file implementation in *one* C or C++ file to prevent collisions.
| zlib 1.2.11 -1 | 72 MB/s | 307 MB/s | 42298774 | 42.30 |
| zlib 1.2.11 -6 | 24 MB/s | 313 MB/s | 36548921 | 36.55 |
| zlib 1.2.11 -9 | 20 MB/s | 314 MB/s | 36475792 | 36.48 |
| sdefl 1.0 -0 | 127 MB/s | 371 MB/s | 40004116 | 39.88 |
| sdefl 1.0 -1 | 111 MB/s | 398 MB/s | 38940674 | 38.82 |
| sdefl 1.0 -5 | 45 MB/s | 420 MB/s | 36577183 | 36.46 |
| sdefl 1.0 -7 | 38 MB/s | 423 MB/s | 36523781 | 36.41 |
| sdefl 1.0 -0 | 127 MB/s | 355 MB/s | 40004116 | 39.88 |
| sdefl 1.0 -1 | 111 MB/s | 413 MB/s | 38940674 | 38.82 |
| sdefl 1.0 -5 | 45 MB/s | 436 MB/s | 36577183 | 36.46 |
| sdefl 1.0 -7 | 38 MB/s | 432 MB/s | 36523781 | 36.41 |
| libdeflate 1.3 -1 | 147 MB/s | 667 MB/s | 39597378 | 39.60 |
| libdeflate 1.3 -6 | 69 MB/s | 689 MB/s | 36648318 | 36.65 |
| libdeflate 1.3 -9 | 13 MB/s | 672 MB/s | 35197141 | 35.20 |
Expand All @@ -50,20 +50,20 @@ this file implementation in *one* C or C++ file to prevent collisions.
### Compression
Results on the [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia):
| File | Original | `sdefl 0` | `sdefl 5` | `sdefl 7` |
| :------ | ---------: | -----------------: | ---------: | ----------: |
| dickens | 10.192.446 | 4,260,187| 3,845,261| 3,833,657 |
| mozilla | 51.220.480 | 20,774,706 | 19,607,009 | 19,565,867 |
| mr | 9.970.564 | 3,860,531 | 3,673,460 | 3,665,627 |
| nci | 33.553.445 | 4,030,283 | 3,094,526 | 3,006,075 |
| ooffice | 6.152.192 | 3,320,063 | 3,186,373 | 3,183,815 |
| osdb | 10.085.684 | 3,919,646 | 3,649,510 | 3,649,477 |
| reymont | 6.627.202 | 2,263,378 | 1,857,588 | 1,827,237 |
| samba | 21.606.400 | 6,121,797 | 5,462,670 | 5,450,762 |
| sao | 7.251.944 | 5,612,421 | 5,485,380 | 5,481,765 |
| webster | 41.458.703 | 13,972,648 | 12,059,432 | 11,991,421 |
| xml | 5.345.280 | 886,620| 674,009 | 662,141 |
| x-ray | 8.474.240 | 6,304,655 | 6,244,779 | 6,244,779 |
| File | Original | `sdefl 0` | `sdefl 5` | `sdefl 7` |
| --------| -----------| -------------| ---------- | ------------|
| dickens | 10.192.446 | 4,260,187 | 3,845,261 | 3,833,657 |
| mozilla | 51.220.480 | 20,774,706 | 19,607,009 | 19,565,867 |
| mr | 9.970.564 | 3,860,531 | 3,673,460 | 3,665,627 |
| nci | 33.553.445 | 4,030,283 | 3,094,526 | 3,006,075 |
| ooffice | 6.152.192 | 3,320,063 | 3,186,373 | 3,183,815 |
| osdb | 10.085.684 | 3,919,646 | 3,649,510 | 3,649,477 |
| reymont | 6.627.202 | 2,263,378 | 1,857,588 | 1,827,237 |
| samba | 21.606.400 | 6,121,797 | 5,462,670 | 5,450,762 |
| sao | 7.251.944 | 5,612,421 | 5,485,380 | 5,481,765 |
| webster | 41.458.703 | 13,972,648 | 12,059,432 | 11,991,421 |
| xml | 5.345.280 | 886,620 | 674,009 | 662,141 |
| x-ray | 8.474.240 | 6,304,655 | 6,244,779 | 6,244,779 |
## License
```
Expand Down
66 changes: 33 additions & 33 deletions sinfl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,15 +221,15 @@ sinfl_peek(struct sinfl *s, int cnt) {
return s->bitbuf & ((1ull << cnt) - 1);
}
static void
sinfl_consume(struct sinfl *s, int cnt) {
sinfl_eat(struct sinfl *s, int cnt) {
assert(cnt <= s->bitcnt);
s->bitbuf >>= cnt;
s->bitcnt -= cnt;
}
static int
sinfl__get(struct sinfl *s, int cnt) {
int res = sinfl_peek(s, cnt);
sinfl_consume(s, cnt);
sinfl_eat(s, cnt);
return res;
}
static int
Expand Down Expand Up @@ -284,7 +284,7 @@ sinfl_build_subtbl(struct sinfl_gen *gen, unsigned *tbl, int tbl_bits,
while (1) {
unsigned entry;
int bit, stride, i;
/* start new subtable */
/* start new sub-table */
if ((gen->word & ((1 << tbl_bits)-1)) != sub_prefix) {
int used = 0;
sub_prefix = gen->word & ((1 << tbl_bits)-1);
Expand All @@ -298,7 +298,7 @@ sinfl_build_subtbl(struct sinfl_gen *gen, unsigned *tbl, int tbl_bits,
tbl_end = sub_start + (1 << sub_bits);
tbl[sub_prefix] = (sub_start << 16) | 0x10 | (sub_bits & 0xf);
}
/* fill subtable */
/* fill sub-table */
entry = (*gen->sorted << 16) | ((gen->len - tbl_bits) & 0xf);
gen->sorted++;
i = sub_start + (gen->word >> tbl_bits);
Expand Down Expand Up @@ -352,17 +352,17 @@ sinfl_build(unsigned *tbl, unsigned char *lens, int tbl_bits, int maxlen,
}
static int
sinfl_decode(struct sinfl *s, const unsigned *tbl, int bit_len) {
{int idx = sinfl_peek(s, bit_len);
int idx = sinfl_peek(s, bit_len);
unsigned key = tbl[idx];
if (key & 0x10) {
/* sub-table lookup */
int len = key & 0x0f;
sinfl_consume(s, bit_len);
sinfl_eat(s, bit_len);
idx = sinfl_peek(s, len);
key = tbl[((key >> 16) & 0xffff) + (unsigned)idx];
}
sinfl_consume(s, key & 0x0f);
return (key >> 16) & 0x0fff;}
sinfl_eat(s, key & 0x0f);
return (key >> 16) & 0x0fff;
}
static int
sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size) {
Expand Down Expand Up @@ -428,32 +428,32 @@ sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size)
state = blk;
} break;
case dyn: {
/* dynamic huffman codes */
int n, i;
unsigned hlens[SINFL_PRE_TBL_SIZE];
unsigned char nlens[19] = {0}, lens[288+32];
/* dynamic huffman codes */
int n, i;
unsigned hlens[SINFL_PRE_TBL_SIZE];
unsigned char nlens[19] = {0}, lens[288+32];

sinfl_refill(&s);
{int nlit = 257 + sinfl__get(&s,5);
int ndist = 1 + sinfl__get(&s,5);
int nlen = 4 + sinfl__get(&s,4);
for (n = 0; n < nlen; n++)
nlens[order[n]] = (unsigned char)sinfl_get(&s,3);
sinfl_build(hlens, nlens, 7, 7, 19);

/* decode code lengths */
for (n = 0; n < nlit + ndist;) {
sinfl_refill(&s);
{int nlit = 257 + sinfl__get(&s,5);
int ndist = 1 + sinfl__get(&s,5);
int nlen = 4 + sinfl__get(&s,4);
for (n = 0; n < nlen; n++)
nlens[order[n]] = (unsigned char)sinfl_get(&s,3);
sinfl_build(hlens, nlens, 7, 7, 19);

/* decode code lengths */
for (n = 0; n < nlit + ndist;) {
sinfl_refill(&s);
int sym = sinfl_decode(&s, hlens, 7);
switch (sym) {default: lens[n++] = (unsigned char)sym; break;
case 16: for (i=3+sinfl_get(&s,2);i;i--,n++) lens[n]=lens[n-1]; break;
case 17: for (i=3+sinfl_get(&s,3);i;i--,n++) lens[n]=0; break;
case 18: for (i=11+sinfl_get(&s,7);i;i--,n++) lens[n]=0; break;}
}
/* build lit/dist tables */
sinfl_build(s.lits, lens, 10, 15, nlit);
sinfl_build(s.dsts, lens + nlit, 8, 15, ndist);
state = blk;}
int sym = sinfl_decode(&s, hlens, 7);
switch (sym) {default: lens[n++] = (unsigned char)sym; break;
case 16: for (i=3+sinfl_get(&s,2);i;i--,n++) lens[n]=lens[n-1]; break;
case 17: for (i=3+sinfl_get(&s,3);i;i--,n++) lens[n]=0; break;
case 18: for (i=11+sinfl_get(&s,7);i;i--,n++) lens[n]=0; break;}
}
/* build lit/dist tables */
sinfl_build(s.lits, lens, 10, 15, nlit);
sinfl_build(s.dsts, lens + nlit, 8, 15, ndist);
state = blk;}
} break;
case blk: {
/* decompress block */
Expand All @@ -462,7 +462,7 @@ sinfl_decompress(unsigned char *out, int cap, const unsigned char *in, int size)
int sym = sinfl_decode(&s, s.lits, 10);
if (sym < 256) {
/* literal */
if (sinfl_unlikely(out + 1 >= oe)) {
if (sinfl_unlikely(out >= oe)) {
return (int)(out-o);
}
*out++ = (unsigned char)sym;
Expand Down
92 changes: 46 additions & 46 deletions tests/defl_test.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,59 +39,59 @@ static struct sdefl sdefl;

int main(int argc, char **argv)
{
void *comp = 0;
size_t size = 0;
struct timespec compr_started;
struct timespec compr_ended;
struct timespec decompr_started;
struct timespec decompr_ended;
double compr_mbs = 0;
double decompr_mbs = 0;
void *decomp;
double ratio = 0.0f;
int len = 0, n, lvl;
void *comp = 0;
size_t size = 0;
struct timespec compr_started;
struct timespec compr_ended;
struct timespec decompr_started;
struct timespec decompr_ended;
double compr_mbs = 0;
double decompr_mbs = 0;
void *decomp;
double ratio = 0.0f;
int len = 0, n, lvl;

void *data = 0;
{
FILE* fp = fopen(argv[1], "rb");
if (!fp) exit(2);
fseek(fp, 0, SEEK_END);
size = (size_t)ftell(fp);
fseek(fp, 0, SEEK_SET);
data = calloc(size, 1);
fread(data, 1, (size_t)size, fp);
fclose(fp);
}
void *data = 0;
{
FILE* fp = fopen(argv[1], "rb");
if (!fp) exit(2);
fseek(fp, 0, SEEK_END);
size = (size_t)ftell(fp);
fseek(fp, 0, SEEK_SET);
data = calloc(size, 1);
fread(data, 1, (size_t)size, fp);
fclose(fp);
}

comp = calloc(size*2, 1);
decomp = calloc(size, 1);
for (lvl = SDEFL_LVL_MIN; lvl < SDEFL_LVL_MAX; ++lvl) {
clock_gettime(CLOCK_REALTIME, &compr_started);
len = sdeflate(&sdefl, comp, data, (int)size, lvl);
clock_gettime(CLOCK_REALTIME, &compr_ended);
comp = calloc(size*2, 1);
decomp = calloc(size, 1);
for (lvl = SDEFL_LVL_MIN; lvl < SDEFL_LVL_MAX; ++lvl) {
clock_gettime(CLOCK_REALTIME, &compr_started);
len = sdeflate(&sdefl, comp, data, (int)size, lvl);
clock_gettime(CLOCK_REALTIME, &compr_ended);

clock_gettime(CLOCK_REALTIME, &decompr_started);
n = sinflate(decomp, (int)size, comp, len);
clock_gettime(CLOCK_REALTIME, &decompr_ended);
clock_gettime(CLOCK_REALTIME, &decompr_started);
n = sinflate(decomp, (int)size, comp, len);
clock_gettime(CLOCK_REALTIME, &decompr_ended);

double compr_time = profiler_time(compr_started, compr_ended);
double decompr_time = profiler_time(decompr_started, decompr_ended);
double compr_time = profiler_time(compr_started, compr_ended);
double decompr_time = profiler_time(decompr_started, decompr_ended);

ratio = (double)len / (double)size * 100.0;
compr_mbs = ((double)size / 1000000.0f) / (compr_time / 1000.0f);
decompr_mbs = ((double)size / 1000000.0f) / (decompr_time / 1000.0f);
ratio = (double)len / (double)size * 100.0;
compr_mbs = ((double)size / 1000000.0f) / (compr_time / 1000.0f);
decompr_mbs = ((double)size / 1000000.0f) / (decompr_time / 1000.0f);

printf("lvl: %d size: %d, compr: %.2fms (%.2fMB/s) decompr: %.2fms (%.2fMB/s) ratio: %f%% ", lvl, len, compr_time, compr_mbs, decompr_time, decompr_mbs, ratio);
if ((size_t)n != size || memcmp(data, decomp, (size_t)size)) {
FILE *fd = fopen("error.bin", "wb");
fwrite(decomp, 1, (size_t)n, fd);
fclose(fd);
printf("lvl: %d size: %d, compr: %.2fms (%.2fMB/s) decompr: %.2fms (%.2fMB/s) ratio: %f%% ", lvl, len, compr_time, compr_mbs, decompr_time, decompr_mbs, ratio);
if ((size_t)n != size || memcmp(data, decomp, (size_t)size)) {
FILE *fd = fopen("error.bin", "wb");
fwrite(decomp, 1, (size_t)n, fd);
fclose(fd);

printf("error: %d:%d !\n", n, (int)size);
break;
}
else printf("\n");
printf("error: %d:%d !\n", n, (int)size);
break;
}
return 0;
else printf("\n");
}
return 0;
}

0 comments on commit 71382a1

Please sign in to comment.