Skip to content
This repository has been archived by the owner on Mar 1, 2024. It is now read-only.

Commit

Permalink
Reorganize inflate window layout
Browse files Browse the repository at this point in the history 
This commit significantly improves inflate performance by reorganizing
the window buffer into a contiguous window and pending output buffer.
The goal of this layout is to reduce branching, improve cache locality,
and enable for the use of crc folding with gzip input.

The window buffer is allocated as a multiple of the user-selected window
size. In this commit, a factor of 4 is utilized.

The layout of the window buffer is divided into two sections. The first
section, window offset [0, wsize), is reserved for history that has
already been output. The second section, window offset [wsize, 4 * wsize),
is reserved for buffering pending output that hasn't been flushed to the
user's output buffer yet.

The history section grows downwards, towards the window offset of 0. The
pending output section grows upwards, towards the end of the buffer. As
a result, all of the possible distance/length data that may need to be
copied is contiguous. This removes the need to stitch together output
from 2 separate buffers.

In the case of gzip input, crc folding is used to copy the pending
output to the user's buffers.
  • Loading branch information
@jtkukunas
jtkukunas committed Jun 21, 2018
1 parent 3e77468 commit 62b7616
Show file tree
Hide file tree
Showing 4 changed files with 230 additions and 176 deletions.
10 changes: 0 additions & 10 deletions infback.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,16 +479,6 @@ void FAR *out_desc;
state->mode = LEN;

case LEN:
/* use inflate_fast() if we have enough input and output */
if (have >= 6 && left >= 258) {
RESTORE();
if (state->whave < state->wsize)
state->whave = state->wsize - left;
inflate_fast(strm, state->wsize);
LOAD();
break;
}

/* get a literal, length, or end-of-block code */
for (;;) {
here = state->lencode[BITS(state->lenbits)];
Expand Down
131 changes: 31 additions & 100 deletions inffast.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
z_const unsigned char FAR *in; /* local strm->next_in */
z_const unsigned char FAR *last; /* have enough input while in < last */
unsigned char FAR *out; /* local strm->next_out */
unsigned char FAR *beg; /* inflate()'s initial strm->next_out */
unsigned char FAR *end; /* while out < end, enough space available */
#ifdef INFLATE_STRICT
unsigned dmax; /* maximum distance from zlib header */
#endif
unsigned wsize; /* window size or zero if not using window */
unsigned whave; /* valid bytes in the window */
unsigned wnext; /* window write index */
unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */
unsigned long hold; /* local strm->hold */
unsigned bits; /* local strm->bits */
code const FAR *lcode; /* local strm->lencode */
Expand All @@ -81,16 +76,11 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
state = (struct inflate_state FAR *)strm->state;
in = strm->next_in;
last = in + (strm->avail_in - 5);
out = strm->next_out;
beg = out - (start - strm->avail_out);
end = out + (strm->avail_out - 257);
out = state->window + state->wsize + state->wnext;
end = state->window + (state->wsize * 4) - MAX_MATCH;
#ifdef INFLATE_STRICT
dmax = state->dmax;
#endif
wsize = state->wsize;
whave = state->whave;
wnext = state->wnext;
window = state->window;
hold = state->hold;
bits = state->bits;
lcode = state->lencode;
Expand All @@ -107,6 +97,15 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
hold += (unsigned long)(*in++) << bits;
bits += 8;
}

if (out >= end) {
state->wnext = out - (state->window + state->wsize);
window_output_flush(strm);
out = state->window + state->wsize + state->wnext;
if (strm->avail_out == 0)
break;
}

here = lcode + (hold & lmask);
dolen:
op = (unsigned)(here->bits);
Expand Down Expand Up @@ -166,76 +165,19 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
hold >>= op;
bits -= op;
Tracevv((stderr, "inflate: distance %u\n", dist));
op = (unsigned)(out - beg); /* max distance in output */
if (dist > op) { /* see if copy from window */
op = dist - op; /* distance back in window */
if (op > whave) {
if (state->sane) {
strm->msg =
(char *)"invalid distance too far back";
state->mode = BAD;
break;
}
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
if (len <= op - whave) {
do {
*out++ = 0;
} while (--len);
continue;
}
len -= op - whave;
do {
*out++ = 0;
} while (--op > whave);
if (op == 0) {
from = out - dist;
do {
*out++ = *from++;
} while (--len);
continue;
}
#endif
}
from = window;
if (wnext == 0) { /* very common case */
from += wsize - op;
if (op < len) { /* some from window */
len -= op;
do {
*out++ = *from++;
} while (--op);
from = out - dist; /* rest from output */
}
}
else if (wnext < op) { /* wrap around window */
from += wsize + wnext - op;
op -= wnext;
if (op < len) { /* some from end of window */
len -= op;
do {
*out++ = *from++;
} while (--op);
from = window;
if (wnext < len) { /* some from start of window */
op = wnext;
len -= op;
do {
*out++ = *from++;
} while (--op);
from = out - dist; /* rest from output */
}
}
}
else { /* contiguous in window */
from += wnext - op;
if (op < len) { /* some from window */
len -= op;
do {
*out++ = *from++;
} while (--op);
from = out - dist; /* rest from output */
}

if (out - dist < ((state->window + state->wsize) - state->whave)) {
if (state->sane) {
strm->msg =
(char *)"invalid distance too far back";
state->mode = BAD;
break;
}
}

from = out - dist;

if (len > dist) {
while (len > 2) {
*out++ = *from++;
*out++ = *from++;
Expand All @@ -245,22 +187,11 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
if (len) {
*out++ = *from++;
if (len > 1)
*out++ = *from++;
}
}
else {
from = out - dist; /* copy direct from output */
do { /* minimum length is three */
*out++ = *from++;
*out++ = *from++;
*out++ = *from++;
len -= 3;
} while (len > 2);
if (len) {
*out++ = *from++;
if (len > 1)
*out++ = *from++;
*out++ = *from;
}
} else {
zmemcpy(out, from, len);
out += len;
}
}
else if ((op & 64) == 0) { /* 2nd level distance code */
Expand All @@ -287,7 +218,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */
state->mode = BAD;
break;
}
} while (in < last && out < end);
} while (in < last);

/* return unused bytes (on entry, bits < 8, so in won't go too far back) */
len = bits >> 3;
Expand All @@ -297,10 +228,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */

/* update state and return */
strm->next_in = in;
strm->next_out = out;
strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last));
strm->avail_out = (unsigned)(out < end ?
257 + (end - out) : 257 - (out - end));

state->wnext = out - (state->window + state->wsize);

state->hold = hold;
state->bits = bits;
return;
Expand Down
Loading

0 comments on commit 62b7616

Please sign in to comment.