Skip to content

Commit

Permalink
fixes a bug in decompression
Browse files Browse the repository at this point in the history 
- fixes a bug in decompression
- fullbench is working now
  • Loading branch information
@inikep
inikep committed Nov 2, 2015
1 parent 47fd000 commit e93625d
Show file tree
Hide file tree
Showing 8 changed files with 159 additions and 1,318 deletions.
78 changes: 54 additions & 24 deletions lib/lz5.c
View file
Expand Up @@ -305,6 +305,7 @@ static const int LZ5_minLength = (MFLIMIT+1);

#define MAXD_LOG 22
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
#define LZ5_DICT_SIZE (1 << MAXD_LOG)

#define ML_BITS 3
#define ML_MASK ((1U<<ML_BITS)-1)
Expand Down Expand Up @@ -906,14 +907,29 @@ static int LZ5_compress_destSize_generic(
op--;
goto _last_literals;
}
if (litLength>=RUN_MASK)

if (ip-match < (1<<10))
{
unsigned len = litLength - RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
if (litLength>=RUN_MASK2)
{
int len = (int)litLength-RUN_MASK2;
*token=(RUN_MASK2<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (BYTE)(litLength<<ML_BITS);
}
else
{
if (litLength>=RUN_MASK)
{
int len = (int)litLength-RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
else *token = (BYTE)(litLength<<ML_BITS);
}
else *token = (BYTE)(litLength<<ML_BITS);

/* Copy Literals */
LZ5_wildCopy(op, anchor, op+litLength);
Expand All @@ -922,7 +938,21 @@ static int LZ5_compress_destSize_generic(

_next_match:
/* Encode Offset */
LZ5_writeLE16(op, (U16)(ip-match)); op+=2;
if (ip-match < (1<<10))
{
*token+=((4+((ip-match)>>8))<<ML_RUN_BITS2);
*op++=(ip-match);
}
else
if (ip-match < (1<<16))
{
LZ5_writeLE16(op, (U16)(ip-match)); op+=2;
}
else
{
*token+=(1<<ML_RUN_BITS);
LZ5_writeLE24(op, (U32)(ip-match)); op+=3;
}

/* Encode MatchLength */
{
Expand Down Expand Up @@ -1079,8 +1109,8 @@ int LZ5_loadDict (LZ5_stream_t* LZ5_dict, const char* dictionary, int dictSize)
return 0;
}

if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
dict->currentOffset += 64 KB;
if ((dictEnd - p) > LZ5_DICT_SIZE) p = dictEnd - LZ5_DICT_SIZE;
dict->currentOffset += LZ5_DICT_SIZE;
base = p - dict->currentOffset;
dict->dictionary = p;
dict->dictSize = (U32)(dictEnd - p);
Expand All @@ -1102,16 +1132,16 @@ static void LZ5_renormDictT(LZ5_stream_t_internal* LZ5_dict, const BYTE* src)
((size_t)LZ5_dict->currentOffset > (size_t)src)) /* address space overflow */
{
/* rescale hash table */
U32 delta = LZ5_dict->currentOffset - 64 KB;
U32 delta = LZ5_dict->currentOffset - LZ5_DICT_SIZE;
const BYTE* dictEnd = LZ5_dict->dictionary + LZ5_dict->dictSize;
int i;
for (i=0; i<HASH_SIZE_U32; i++)
{
if (LZ5_dict->hashTable[i] < delta) LZ5_dict->hashTable[i]=0;
else LZ5_dict->hashTable[i] -= delta;
}
LZ5_dict->currentOffset = 64 KB;
if (LZ5_dict->dictSize > 64 KB) LZ5_dict->dictSize = 64 KB;
LZ5_dict->currentOffset = LZ5_DICT_SIZE;
if (LZ5_dict->dictSize > LZ5_DICT_SIZE) LZ5_dict->dictSize = LZ5_DICT_SIZE;
LZ5_dict->dictionary = dictEnd - LZ5_dict->dictSize;
}
}
Expand All @@ -1134,7 +1164,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch
if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
{
streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize > LZ5_DICT_SIZE) streamPtr->dictSize = LZ5_DICT_SIZE;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
streamPtr->dictionary = dictEnd - streamPtr->dictSize;
}
Expand All @@ -1144,7 +1174,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch
if (dictEnd == (const BYTE*)source)
{
int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset))
result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
else
result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
Expand All @@ -1156,7 +1186,7 @@ int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, ch
/* external dictionary mode */
{
int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset))
result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
else
result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
Expand Down Expand Up @@ -1194,7 +1224,7 @@ int LZ5_saveDict (LZ5_stream_t* LZ5_dict, char* safeBuffer, int dictSize)
LZ5_stream_t_internal* dict = (LZ5_stream_t_internal*) LZ5_dict;
const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;

if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; /* useless to define a dictionary > LZ5_DICT_SIZE */
if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;

memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
Expand Down Expand Up @@ -1246,7 +1276,7 @@ FORCE_INLINE int LZ5_decompress_generic(
const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};

const int safeDecode = (endOnInput==endOnInputSize);
const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
const int checkOffset = ((safeDecode) && (dictSize < (int)(LZ5_DICT_SIZE)));


/* Special cases */
Expand Down Expand Up @@ -1298,7 +1328,7 @@ FORCE_INLINE int LZ5_decompress_generic(

/* copy literals */
cpy = op+length;
if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(1+1+LASTLITERALS))) )
|| ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)))
{
if (partialDecoding)
Expand Down Expand Up @@ -1440,7 +1470,7 @@ int LZ5_decompress_safe_partial(const char* source, char* dest, int compressedSi

int LZ5_decompress_fast(const char* source, char* dest, int originalSize)
{
return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - LZ5_DICT_SIZE), NULL, LZ5_DICT_SIZE);
}


Expand Down Expand Up @@ -1567,8 +1597,8 @@ FORCE_INLINE int LZ5_decompress_usingDict_generic(const char* source, char* dest
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
if (dictStart+dictSize == dest)
{
if (dictSize >= (int)(64 KB - 1))
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
if (dictSize >= (int)(LZ5_DICT_SIZE - 1))
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-LZ5_DICT_SIZE, NULL, 0);
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
}
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
Expand Down Expand Up @@ -1639,20 +1669,20 @@ void* LZ5_create (char* inputBuffer)
char* LZ5_slideInputBuffer (void* LZ5_Data)
{
LZ5_stream_t_internal* ctx = (LZ5_stream_t_internal*)LZ5_Data;
int dictSize = LZ5_saveDict((LZ5_stream_t*)LZ5_Data, (char*)ctx->bufferStart, 64 KB);
int dictSize = LZ5_saveDict((LZ5_stream_t*)LZ5_Data, (char*)ctx->bufferStart, LZ5_DICT_SIZE);
return (char*)(ctx->bufferStart + dictSize);
}

/* Obsolete streaming decompression functions */

int LZ5_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE);
}

int LZ5_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
{
return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE);
}

#endif /* LZ5_COMMONDEFS_ONLY */
Expand Down
82 changes: 51 additions & 31 deletions lib/lz5hc.c
View file
Expand Up @@ -92,7 +92,7 @@ static const int g_maxCompressionLevel = 16;
/**************************************
* Local Types
**************************************/
typedef struct
struct LZ5HC_Data_s
{
U32* hashTable;
U32* chainTable;
Expand All @@ -104,7 +104,7 @@ typedef struct
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel;
} LZ5HC_Data_Structure;
};


/**************************************
Expand Down Expand Up @@ -584,6 +584,29 @@ int LZ5_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
return LZ5HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
}

int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr)
{
statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE);
if (!statePtr->hashTable)
return 0;

statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD);
if (!statePtr->chainTable)
{
FREEMEM(statePtr->hashTable);
statePtr->hashTable = NULL;
return 0;
}

return 1;
}

void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr)
{
if (statePtr->chainTable) FREEMEM(statePtr->chainTable);
if (statePtr->hashTable) FREEMEM(statePtr->hashTable);
}

int LZ5_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
#if LZ5HC_HEAPMODE==1
Expand All @@ -594,17 +617,13 @@ int LZ5_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
#endif

int cSize = 0;
statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE);
if (statePtr->hashTable)
{
statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD);
if (statePtr->chainTable)
{
cSize = LZ5_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
FREEMEM(statePtr->chainTable);
}
FREEMEM(statePtr->hashTable);
}

if (!LZ5_alloc_mem_HC(statePtr))
return 0;

cSize = LZ5_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);

LZ5_free_mem_HC(statePtr);

#if LZ5HC_HEAPMODE==1
free(statePtr);
Expand All @@ -624,29 +643,19 @@ LZ5_streamHC_t* LZ5_createStreamHC(void)
if (!statePtr)
return NULL;

statePtr->hashTable = ALLOCATOR(1, sizeof(U32)*HASHTABLESIZE);
if (!statePtr->hashTable)
if (!LZ5_alloc_mem_HC(statePtr))
{
FREEMEM(statePtr);
return NULL;
}

statePtr->chainTable = ALLOCATOR(1, sizeof(U32)*MAXD);
if (!statePtr->chainTable)
{
FREEMEM(statePtr->hashTable);
FREEMEM(statePtr);
return NULL;
}

return (LZ5_streamHC_t*) statePtr;
}

int LZ5_freeStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr)
int LZ5_freeStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr)
{
LZ5HC_Data_Structure* statePtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr;
FREEMEM(statePtr->chainTable);
FREEMEM(statePtr->hashTable);
LZ5_free_mem_HC(statePtr);
free(LZ5_streamHCPtr);
return 0;
}
Expand All @@ -663,10 +672,10 @@ void LZ5_resetStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr, int compressionLevel)
int LZ5_loadDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, const char* dictionary, int dictSize)
{
LZ5HC_Data_Structure* ctxPtr = (LZ5HC_Data_Structure*) LZ5_streamHCPtr;
if (dictSize > 64 KB)
if (dictSize > LZ5_DICT_SIZE)
{
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
dictionary += dictSize - LZ5_DICT_SIZE;
dictSize = LZ5_DICT_SIZE;
}
LZ5HC_init (ctxPtr, (const BYTE*)dictionary);
if (dictSize >= 4) LZ5HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
Expand Down Expand Up @@ -702,7 +711,7 @@ static int LZ5_compressHC_continue_generic (LZ5HC_Data_Structure* ctxPtr,
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
{
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE;

LZ5_loadDictHC((LZ5_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
Expand Down Expand Up @@ -742,7 +751,7 @@ int LZ5_saveDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, char* safeBuffer, int dictS
{
LZ5HC_Data_Structure* streamPtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr;
int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
Expand All @@ -757,3 +766,14 @@ int LZ5_saveDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, char* safeBuffer, int dictS
return dictSize;
}

/***********************************
* Deprecated Functions
***********************************/
/* Deprecated compression functions */
/* These functions are planned to start generate warnings by r131 approximately */
int LZ5_compressHC(const char* src, char* dst, int srcSize) { return LZ5_compress_HC (src, dst, srcSize, LZ5_compressBound(srcSize), 0); }
int LZ5_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC(src, dst, srcSize, maxDstSize, 0); }
int LZ5_compressHC_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, LZ5_compressBound(srcSize)); }
int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); }
int LZ5_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, LZ5_compressBound(srcSize), 0); }
int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); }
13 changes: 6 additions & 7 deletions lib/lz5hc.h
View file
Expand Up @@ -66,6 +66,10 @@ LZ5_compress_HC :
Decompression functions are provided within LZ5 source code (see "lz5.h") (BSD license)
*/

typedef struct LZ5HC_Data_s LZ5HC_Data_Structure;

int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr);
void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr);

int LZ5_sizeofStateHC(void);
int LZ5_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
Expand Down Expand Up @@ -165,15 +169,10 @@ int LZ5_saveDictHC (LZ5_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSi
/* these functions are planned to trigger warning messages by r131 approximately */
int LZ5_compressHC (const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ5_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
int LZ5_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ5_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
int LZ5_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ5_compressHC_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);

int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);

#if defined (__cplusplus)
}
Expand Down

0 comments on commit e93625d

Please sign in to comment.