Move dict reset logic to dict_buffer.c; fix gcc warnings

dict_buffer.c

@@ -32,7 +32,7 @@ int DICT_construct(DICT_buffer * const buf, int const async)
     return 0;
 }
 
-int DICT_init(DICT_buffer * const buf, size_t const dict_size, int const do_hash)
+int DICT_init(DICT_buffer * const buf, size_t const dict_size, size_t const overlap, unsigned const reset_multiplier, int const do_hash)
 {
     /* Allocate if not yet allocated or existing dict too small */
     if (buf->data[0] == NULL || dict_size > buf->size) {
@@ -51,9 +51,12 @@ int DICT_init(DICT_buffer * const buf, size_t const dict_size, int const do_hash
         }
     }
     buf->index = 0;
+    buf->overlap = overlap;
     buf->start = 0;
     buf->end = 0;
     buf->size = dict_size;
+    buf->total = 0;
+    buf->reset_interval = (reset_multiplier != 0) ? dict_size * reset_multiplier : ((size_t)1 << 31);
 
 #ifndef NO_XXHASH
     if (do_hash) {
@@ -96,9 +99,9 @@ size_t DICT_size(const DICT_buffer * const buf)
 }
 
 /* Get the dictionary buffer for adding input */
-size_t DICT_get(DICT_buffer * const buf, size_t const overlap, FL2_outBuffer * const dict)
+size_t DICT_get(DICT_buffer * const buf, FL2_outBuffer * const dict)
 {
-    DICT_shift(buf, overlap);
+    DICT_shift(buf);
 
     dict->dst = buf->data[buf->index] + buf->end;
     dict->pos = 0;
@@ -151,12 +154,17 @@ void DICT_getBlock(DICT_buffer * const buf, FL2_dataBlock * const block)
         XXH32_update(buf->xxh, buf->data[buf->index] + buf->start, buf->end - buf->start);
 #endif
 
+    buf->total += buf->end - buf->start;
     buf->start = buf->end;
 }
 
 /* Shift occurs when all is processed and end is beyond the overlap size */
-int DICT_needShift(DICT_buffer * const buf, size_t const overlap)
+int DICT_needShift(DICT_buffer * const buf)
 {
+    if (buf->start < buf->end)
+        return 0;
+    /* Reset the dict if the next compression cycle would exceed the reset interval */
+    size_t overlap = (buf->total + buf->size - buf->overlap > buf->reset_interval) ? 0 : buf->overlap;
     return buf->start == buf->end && (overlap == 0 || buf->end >= overlap + ALIGNMENT_SIZE);
 }
 
@@ -167,16 +175,24 @@ int DICT_async(const DICT_buffer * const buf)
 
 /* Shift the overlap amount to the start of either the only dict buffer or the alternate one
  * if it exists */
-void DICT_shift(DICT_buffer * const buf, size_t overlap)
+void DICT_shift(DICT_buffer * const buf)
 {
     if (buf->start < buf->end)
         return;
 
+    size_t overlap = buf->overlap;
+    /* Reset the dict if the next compression cycle would exceed the reset interval */
+    if (buf->total + buf->size - buf->overlap > buf->reset_interval) {
+        DEBUGLOG(4, "Resetting dictionary after %u bytes", (unsigned)buf->total);
+        overlap = 0;
+    }
+
     if (overlap == 0) {
         /* No overlap means a simple buffer switch */
         buf->start = 0;
         buf->end = 0;
         buf->index ^= buf->async;
+        buf->total = 0;
     }
     else if (buf->end >= overlap + ALIGNMENT_SIZE) {
         size_t const from = (buf->end - overlap) & ALIGNMENT_MASK;
@@ -187,11 +203,11 @@ void DICT_shift(DICT_buffer * const buf, size_t overlap)
         overlap = buf->end - from;
 
         if (overlap <= from || dst != src) {
-            DEBUGLOG(5, "Copy overlap data : %u bytes from %u", (U32)overlap, (U32)from);
+            DEBUGLOG(5, "Copy overlap data : %u bytes from %u", (unsigned)overlap, (unsigned)from);
             memcpy(dst, src + from, overlap);
         }
         else if (from != 0) {
-            DEBUGLOG(5, "Move overlap data : %u bytes from %u", (U32)overlap, (U32)from);
+            DEBUGLOG(5, "Move overlap data : %u bytes from %u", (unsigned)overlap, (unsigned)from);
             memmove(dst, src + from, overlap);
         }
         /* New data will be written after the overlap */

dict_buffer.h

@@ -17,23 +17,26 @@ typedef struct {
     BYTE* data[2];
     size_t index;
     size_t async;
+    size_t overlap;
     size_t start;  /* start = 0 (first block) or overlap */
     size_t end;    /* never < overlap */
     size_t size;   /* allocation size */
+    size_t total;  /* total size compressed after last dict reset */
+    size_t reset_interval;
 #ifndef NO_XXHASH
     XXH32_state_t *xxh;
 #endif
 } DICT_buffer;
 
 int DICT_construct(DICT_buffer *const buf, int const async);
 
-int DICT_init(DICT_buffer *const buf, size_t const dict_size, int const do_hash);
+int DICT_init(DICT_buffer *const buf, size_t const dict_size, size_t const overlap, unsigned const reset_multiplier, int const do_hash);
 
 void DICT_destruct(DICT_buffer *const buf);
 
 size_t DICT_size(const DICT_buffer *const buf);
 
-size_t DICT_get(DICT_buffer *const buf, size_t const overlap, FL2_outBuffer* const dict);
+size_t DICT_get(DICT_buffer *const buf, FL2_outBuffer* const dict);
 
 int DICT_update(DICT_buffer *const buf, size_t const added_size);
 
@@ -45,11 +48,11 @@ int DICT_hasUnprocessed(const DICT_buffer *const buf);
 
 void DICT_getBlock(DICT_buffer *const buf, FL2_dataBlock *const block);
 
-int DICT_needShift(DICT_buffer *const buf, size_t const overlap);
+int DICT_needShift(DICT_buffer *const buf);
 
 int DICT_async(const DICT_buffer *const buf);
 
-void DICT_shift(DICT_buffer *const buf, size_t overlap);
+void DICT_shift(DICT_buffer *const buf);
 
 #ifndef NO_XXHASH
 XXH32_hash_t DICT_getDigest(const DICT_buffer *const buf);

fl2_compress.c

@@ -379,8 +379,6 @@ static size_t FL2_compressCurBlock(FL2_CCtx* const cctx, int const streamProp)
     /* update largest dict size used */
     cctx->dictMax = MAX(cctx->dictMax, cctx->curBlock.end);
 
-    cctx->blockTotal += cctx->curBlock.end - cctx->curBlock.start;
-
     cctx->outThread = 0;
     cctx->threadCount = 0;
     cctx->outPos = 0;
@@ -456,7 +454,6 @@ static size_t FL2_beginFrame(FL2_CCtx* const cctx, size_t const dictReduce)
     }
 
     cctx->dictMax = 0;
-    cctx->blockTotal = 0;
     cctx->streamTotal = 0;
     cctx->streamCsize = 0;
     cctx->progressIn = 0;
@@ -499,8 +496,11 @@ static size_t FL2_compressBuffer(FL2_CCtx* const cctx,
     cctx->curBlock.data = src;
     cctx->curBlock.start = 0;
 
+    size_t blockTotal = 0;
+
     do {
         cctx->curBlock.end = cctx->curBlock.start + MIN(srcSize, dictionarySize - cctx->curBlock.start);
+        blockTotal += cctx->curBlock.end - cctx->curBlock.start;
 
         CHECK_F(FL2_compressCurBlock(cctx, streamProp));
 
@@ -520,10 +520,11 @@ static size_t FL2_compressBuffer(FL2_CCtx* const cctx,
         }
         srcSize -= cctx->curBlock.end - cctx->curBlock.start;
         if (cctx->params.cParams.reset_interval
-            && cctx->blockTotal + MIN(cctx->curBlock.end - blockOverlap, srcSize) > dictionarySize * cctx->params.cParams.reset_interval) {
+            && blockTotal + MIN(dictionarySize - blockOverlap, srcSize) > dictionarySize * cctx->params.cParams.reset_interval) {
             /* periodically reset the dictionary for mt decompression */
+            DEBUGLOG(4, "Resetting dictionary after %u bytes", (unsigned)blockTotal);
             cctx->curBlock.start = 0;
-            cctx->blockTotal = 0;
+            blockTotal = 0;
         }
         else {
             cctx->curBlock.start = blockOverlap;
@@ -871,7 +872,8 @@ FL2LIB_API size_t FL2LIB_CALL FL2_initCStream(FL2_CStream* fcs, int compressionL
 #else
     int const doHash = (fcs->params.doXXH && !fcs->params.omitProp);
 #endif
-    if (DICT_init(buf, dictSize, doHash) != 0)
+    size_t dictOverlap = OVERLAP_FROM_DICT_SIZE(fcs->params.rParams.dictionary_size, fcs->params.rParams.overlap_fraction);
+    if (DICT_init(buf, dictSize, dictOverlap, fcs->params.cParams.reset_interval, doHash) != 0)
         return FL2_ERROR(memory_allocation);
 
     CHECK_F(FL2_beginFrame(fcs, 0));
@@ -961,25 +963,14 @@ static size_t FL2_compressStream_input(FL2_CStream* fcs, FL2_inBuffer* input)
     CHECK_F(fcs->asyncRes);
 
     DICT_buffer * const buf = &fcs->buf;
-    size_t const blockOverlap = OVERLAP_FROM_DICT_SIZE(fcs->params.rParams.dictionary_size, fcs->params.rParams.overlap_fraction);
 
     while (input->pos < input->size) {
         /* read input until the buffer(s) are full */
-        size_t overlap = blockOverlap;
-        if (!DICT_hasUnprocessed(buf)
-            && fcs->params.cParams.reset_interval != 0
-            && fcs->blockTotal + (fcs->params.rParams.dictionary_size - overlap)
-            > (fcs->params.rParams.dictionary_size * fcs->params.cParams.reset_interval))
-        {
-            /* periodically reset the dictionary for mt decompression */
-            overlap = 0;
-            fcs->blockTotal = 0;
-        }
-        if (DICT_needShift(buf, overlap)) {
+        if (DICT_needShift(buf)) {
             /* cannot shift single dict during compression */
             if(!DICT_async(buf))
                 CHECK_F(FL2_waitStream(fcs));
-            DICT_shift(buf, overlap);
+            DICT_shift(buf);
         }
 
         CHECK_F(fcs->asyncRes);
@@ -1041,12 +1032,10 @@ FL2LIB_API size_t FL2LIB_CALL FL2_getDictionaryBuffer(FL2_CStream * fcs, FL2_out
 
     CHECK_F(fcs->asyncRes);
 
-    size_t const blockOverlap = OVERLAP_FROM_DICT_SIZE(fcs->params.rParams.dictionary_size, fcs->params.rParams.overlap_fraction);
-
-    if (DICT_needShift(&fcs->buf, blockOverlap) && !DICT_async(&fcs->buf))
+    if (DICT_needShift(&fcs->buf) && !DICT_async(&fcs->buf))
         CHECK_F(FL2_waitStream(fcs));
 
-    DICT_get(&fcs->buf, blockOverlap, dict);
+    DICT_get(&fcs->buf, dict);
 
     return FL2_error_no_error;
 }
@@ -1252,11 +1241,11 @@ FL2LIB_API size_t FL2LIB_CALL FL2_getLevelParameters(int compressionLevel, int h
 }
 
 static size_t FL2_memoryUsage_internal(size_t const dictionarySize, unsigned const bufferLog,
-    unsigned const searchDepth, unsigned const chainLog,
+    unsigned const chainLog,
     FL2_strategy const strategy,
     unsigned const nbThreads)
 {
-    return RMF_memoryUsage(dictionarySize, bufferLog, searchDepth, nbThreads)
+    return RMF_memoryUsage(dictionarySize, bufferLog, nbThreads)
         + LZMA2_encMemoryUsage(chainLog, strategy, nbThreads);
 }
 
@@ -1274,7 +1263,6 @@ FL2LIB_API size_t FL2LIB_CALL FL2_estimateCCtxSize_byParams(const FL2_compressio
 {
     return FL2_memoryUsage_internal(params->dictionarySize,
         params->bufferLog,
-        params->searchDepth,
         params->chainLog,
         params->strategy,
         nbThreads);
@@ -1284,7 +1272,6 @@ FL2LIB_API size_t FL2LIB_CALL FL2_estimateCCtxSize_usingCCtx(const FL2_CCtx * cc
 {
     return FL2_memoryUsage_internal(cctx->params.rParams.dictionary_size,
         cctx->params.rParams.match_buffer_log,
-        cctx->params.rParams.depth,
         cctx->params.cParams.second_dict_bits,
         cctx->params.cParams.strategy,
         cctx->jobCount) + DICT_memUsage(&cctx->buf);

fl2_compress_internal.h

@@ -66,7 +66,6 @@ struct FL2_CCtx_s {
     size_t outThread;
     size_t outPos;
     size_t dictMax;
-    U64 blockTotal;
     U64 streamTotal;
     U64 streamCsize;
     FL2_matchTable* matchTable;

fl2_decompress.c

@@ -170,6 +170,8 @@ static void FL2_decompressCtxBlock(void* const jobDescription, ptrdiff_t const n
     BlockDecMt* const blocks = (BlockDecMt*)jobDescription;
     size_t srcLen = blocks[n].packSize;
 
+    DEBUGLOG(4, "Thread %u: decoding block of input size %u, output size %u", (unsigned)n, (unsigned)srcLen, (unsigned)blocks[n].unpackSize);
+
     blocks[n].res = LZMA2_decodeToDic(blocks[n].dec, blocks[n].unpackSize, blocks[n].src, &srcLen, blocks[n].finish);
 
     /* If no error occurred, store into res the dic_pos value, which is the end of the decompressed data in the buffer */
@@ -676,6 +678,8 @@ static size_t FL2_decompressBlockMt(FL2_DStream* const fds, size_t const thread)
     FL2_decJob *const ti = &decmt->threads[thread];
     LZMA2_DCtx *const dec = &ti->dec;
 
+    DEBUGLOG(4, "Thread %u: decoding block of size %u", (unsigned)thread, (unsigned)ti->bufSize);
+
     CHECK_F(LZMA2_initDecoder(dec, decmt->prop, ti->outBuf, ti->bufSize));
 
     /* Input buffer node containing the starting chunk. If thread > 0 this is usually

lzma2_dec.c

@@ -1025,7 +1025,7 @@ static unsigned LZMA2_nextChunkInfo(BYTE *const control,
     return LZMA2_STATE_DATA;
 }
 
-size_t LZMA2_decodeChunkToDic(LZMA2_DCtx *const p, size_t const dic_limit,
+static size_t LZMA2_decodeChunkToDic(LZMA2_DCtx *const p, size_t const dic_limit,
     const BYTE *src, size_t *const src_len, ELzmaFinishMode const finish_mode)
 {
     if (p->state2 == LZMA2_STATE_FINISHED)