Small LZSA2 ratio increase for some files

emmanuel-marty · Jun 4, 2021 · 5e404e9 · 5e404e9
1 parent 65d6972
commit 5e404e9
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 2 deletions.
diff --git a/src/shrink_block_v2.c b/src/shrink_block_v2.c
@@ -1310,6 +1310,7 @@ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigne
 
          int* first_offset_for_byte = pCompressor->first_offset_for_byte;
          int* next_offset_for_pos = pCompressor->next_offset_for_pos;
+         int* offset_cache = pCompressor->offset_cache;
          int nPosition;
 
          /* Supplement small matches */
@@ -1352,12 +1353,77 @@ int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigne
                   match[m].offset = nMatchOffset;
                   m++;
                   nInserted++;
-                  if (nInserted >= 15)
+                  if (nInserted >= 12)
                      break;
                }
             }
          }
 
+         /* Supplement matches further */
+
+         memset(offset_cache, 0xff, sizeof(int) * 2048);
+
+         for (nPosition = nPreviousBlockSize + 1; nPosition < (nEndOffset - 1); nPosition++) {
+            lzsa_match* match = pCompressor->match + ((nPosition - nPreviousBlockSize) << MATCHES_PER_INDEX_SHIFT_V2);
+
+            if (match[0].length < 5) {
+               int m = 0;
+               int nMatchPos;
+
+               while (m < 46 && match[m].length) {
+                  offset_cache[match[m].offset & 2047] = nPosition;
+                  m++;
+               }
+
+               for (nMatchPos = next_offset_for_pos[nPosition - nPreviousBlockSize]; m < 46 && nMatchPos >= 0; nMatchPos = next_offset_for_pos[nMatchPos - nPreviousBlockSize]) {
+                  int nMatchOffset = nPosition - nMatchPos;
+
+                  if (nMatchOffset <= MAX_OFFSET) {
+                     int nAlreadyExists = 0;
+
+                     if (offset_cache[nMatchOffset & 2047] == nPosition) {
+                        int nExistingMatchIdx;
+
+                        for (nExistingMatchIdx = 0; nExistingMatchIdx < m; nExistingMatchIdx++) {
+                           if (match[nExistingMatchIdx].offset == nMatchOffset) {
+                              nAlreadyExists = 1;
+                              break;
+                           }
+                        }
+                     }
+
+                     if (!nAlreadyExists) {
+                        int nForwardPos = nPosition + 2;
+                        int nGotMatch = 0;
+
+                        while (nForwardPos >= nMatchOffset && (nForwardPos + 2) < nEndOffset && nForwardPos < (nPosition + 2 + 1 + 2)) {
+                           if (!memcmp(pInWindow + nForwardPos, pInWindow + nForwardPos - nMatchOffset, 2)) {
+                              nGotMatch = 1;
+                              break;
+                           }
+                           nForwardPos++;
+                        }
+
+                        if (nGotMatch) {
+                           int nMatchLen = 2;
+                           while (nMatchLen < 16 && nPosition < (nEndOffset - nMatchLen) && pInWindow[nMatchPos + nMatchLen] == pInWindow[nPosition + nMatchLen])
+                              nMatchLen++;
+                           match[m].length = nMatchLen | 0x8000;
+                           match[m].offset = nMatchOffset;
+                           m++;
+
+                           lzsa_insert_forward_match_v2(pCompressor, pInWindow, nPosition, nMatchOffset, nPreviousBlockSize, nEndOffset, 8);
+                           break;
+                        }
+                     }
+                  }
+                  else {
+                     break;
+                  }
+               }
+            }
+         }
+
          /* Compress optimally with the extra matches */
          memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match));
          lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */, 0 /* use forward reps */, nArrivalsPerPosition);

diff --git a/src/shrink_context.c b/src/shrink_context.c
@@ -66,6 +66,7 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
    pCompressor->rep_len_handled_mask = NULL;
    pCompressor->first_offset_for_byte = NULL;
    pCompressor->next_offset_for_pos = NULL;
+   pCompressor->offset_cache = NULL;
    pCompressor->min_match_size = nMinMatchSize;
    if (pCompressor->min_match_size < nMinMatchSizeForFormat)
       pCompressor->min_match_size = nMinMatchSizeForFormat;
@@ -116,7 +117,10 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
                                     if (pCompressor->first_offset_for_byte) {
                                        pCompressor->next_offset_for_pos = (int*)malloc(BLOCK_SIZE * sizeof(int));
                                        if (pCompressor->next_offset_for_pos) {
-                                          return 0;
+                                          pCompressor->offset_cache = (int*)malloc(2048 * sizeof(int));
+                                          if (pCompressor->offset_cache) {
+                                             return 0;
+                                          }
                                        }
                                     }
                                  }
@@ -146,6 +150,11 @@ int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize,
 void lzsa_compressor_destroy(lzsa_compressor *pCompressor) {
    divsufsort_destroy(&pCompressor->divsufsort_context);
 
+   if (pCompressor->offset_cache) {
+      free(pCompressor->offset_cache);
+      pCompressor->offset_cache = NULL;
+   }
+
    if (pCompressor->next_offset_for_pos) {
       free(pCompressor->next_offset_for_pos);
       pCompressor->next_offset_for_pos = NULL;

diff --git a/src/shrink_context.h b/src/shrink_context.h
@@ -128,6 +128,7 @@ typedef struct _lzsa_compressor {
    char *rep_len_handled_mask;
    int *first_offset_for_byte;
    int *next_offset_for_pos;
+   int *offset_cache;
    int min_match_size;
    int format_version;
    int flags;