Skip to content

Commit

Permalink
uncompressed-api: allow uncompressed_update only for independent blocks
Browse files Browse the repository at this point in the history 
Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
  • Loading branch information
@alexmohr
alexmohr committed Jul 5, 2022
1 parent e595150 commit 42eb47d
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 163 deletions.
12 changes: 0 additions & 12 deletions doc/lz4_manual.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,18 +391,6 @@ <h1>1.9.4 Manual</h1>

</p></pre><BR>

<pre><b>LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);
</b><p> Get the size of the dictionary. This can be used for adding data without
compression to the LZ4 archive. If linked blocked mode is used the memory
of the dictionary is kept free.
This way uncompressed data does not influence the effectiveness of the
dictionary.
@param LZ4_dict Pointer to the dictionary to get the size of.
@param dictSize The maximum dictionary size. (Normally 64 KB).
@return The size of the dictionary.

</p></pre><BR>

<pre><b></b><p>
It's possible to have input and output sharing the same buffer,
for highly constrained memory environments.
Expand Down
24 changes: 4 additions & 20 deletions lib/lz4.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1679,25 +1679,6 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
return result;
}

/*! LZ4_getDictSize():
* Get the size of the dictionary. This can be used for adding data without
* compression to the LZ4 archive. If linked blocked mode is used the memory
* of the dictionary is kept free.
* This way uncompressed data does not influence the effectiveness of the
* dictionary.
* @param LZ4_dict Pointer to the dictionary to get the size of.
* @param dictSize The maximum dictionary size. (Normally 64 KB).
* @return The size of the dictionary.
*/
int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize)
{
const LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;

if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }

return dictSize;
}

/*! LZ4_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
Expand All @@ -1709,9 +1690,12 @@ int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize)
int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
dictSize = LZ4_getDictSize(LZ4_dict, dictSize);

DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);

if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }

if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0) {
const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
Expand Down
11 changes: 0 additions & 11 deletions lib/lz4.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,17 +509,6 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
*/
LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);

/*! LZ4_getDictSize():
* Get the size of the dictionary. This can be used for adding data without
* compression to the LZ4 archive. If linked blocked mode is used the memory
* of the dictionary is kept free.
* This way uncompressed data does not influence the effectiveness of the
* dictionary.
* @param LZ4_dict Pointer to the dictionary to get the size of.
* @param dictSize The maximum dictionary size. (Normally 64 KB).
* @return The size of the dictionary.
*/
LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);

/*! In-place compression and decompression
*
Expand Down
172 changes: 73 additions & 99 deletions lib/lz4frame.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,23 +841,12 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
return LZ4F_compressBlockHC_continue;
}

static int LZ4F_maxDictSize(void) {
return 64 KB;
}

/* Save history (up to 64KB) into @tmpBuff */
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
}

static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
return LZ4_getDictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
return LZ4_getDictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
}

typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
Expand Down Expand Up @@ -886,69 +875,64 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
size_t const blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcPtr + srcSize;
BYTE* dstStart = (BYTE*)dstBuffer;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* dstPtr = dstStart;
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
size_t bytesWritten = 0;
size_t bytesWritten;
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);

/* flush currently written block, to continue with new block compression */
if (cctxPtr->blockCompression != blockCompression) {
bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
dstStart = (BYTE*)dstBuffer + bytesWritten;
dstPtr = dstStart;
dstCapacity -= bytesWritten;
cctxPtr->blockCompression = blockCompression;
}

RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */

if (blockCompression == LZ4B_COMPRESSED &&
dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
RETURN_ERROR(dstMaxSize_tooSmall);
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
RETURN_ERROR(dstMaxSize_tooSmall);

if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
RETURN_ERROR(dstMaxSize_tooSmall);
RETURN_ERROR(dstMaxSize_tooSmall);

/* flush currently written block, to continue with new block compression */
if (cctxPtr->blockCompression != blockCompression) {
bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
dstPtr += bytesWritten;
cctxPtr->blockCompression = blockCompression;
}

if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;

/* complete tmp buffer */
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
assert(blockSize > cctxPtr->tmpInSize);
if (sizeToCopy > srcSize) {
/* add src to tmpIn buffer */
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
srcPtr = srcEnd;
cctxPtr->tmpInSize += srcSize;
/* still needs some CRC */
} else {
/* complete tmpIn block and then compress it */
lastBlockCompressed = fromTmpBuffer;
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;
size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
assert(blockSize > cctxPtr->tmpInSize);
if (sizeToCopy > srcSize) {
/* add src to tmpIn buffer */
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
srcPtr = srcEnd;
cctxPtr->tmpInSize += srcSize;
/* still needs some CRC */
} else {
/* complete tmpIn block and then compress it */
lastBlockCompressed = fromTmpBuffer;
memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
srcPtr += sizeToCopy;

dstPtr += LZ4F_makeBlock(dstPtr,
cctxPtr->tmpIn, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
} }

while ((size_t)(srcEnd - srcPtr) >= blockSize) {
/* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_makeBlock(dstPtr,
cctxPtr->tmpIn, blockSize,
srcPtr, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);

if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
cctxPtr->tmpInSize = 0;
}
}

while ((size_t)(srcEnd - srcPtr) >= blockSize) {
/* compress full blocks */
lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_makeBlock(dstPtr,
srcPtr, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
srcPtr += blockSize;
cctxPtr->prefs.frameInfo.blockChecksumFlag,
blockCompression);
srcPtr += blockSize;
}

if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
Expand All @@ -960,27 +944,20 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag,
blockCompression);
srcPtr = srcEnd;
srcPtr = srcEnd;
}

/* preserve dictionary within @tmpBuff whenever necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
if (compressOptionsPtr->stableSrc) {
cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
} else {
int realDictSize;
if (blockCompression == LZ4B_COMPRESSED) {
realDictSize = LZ4F_localSaveDict(cctxPtr);
/* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
assert(blockCompression == LZ4B_COMPRESSED);
if (compressOptionsPtr->stableSrc) {
cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
} else {
/* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate
* this is only relevant if linked block mode
* */
realDictSize = LZ4F_localDictSize(cctxPtr);
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
assert(0 <= realDictSize && realDictSize <= 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}

assert(0 <= realDictSize && realDictSize <= 64 KB);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
}
}

/* keep tmpIn within limits */
Expand All @@ -989,30 +966,24 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
{
/* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
* note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
int realDictSize;
if (blockCompression == LZ4B_COMPRESSED) {
realDictSize = LZ4F_localSaveDict(cctxPtr);
} else {
/* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/
realDictSize = LZ4F_maxDictSize();
}
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
}

/* some input data left, necessarily < blockSize */
if (srcPtr < srcEnd) {
/* fill tmp buffer */
size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
cctxPtr->tmpInSize = sizeToCopy;
/* fill tmp buffer */
size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
cctxPtr->tmpInSize = sizeToCopy;
}

if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
(void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
(void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);

cctxPtr->totalInSize += srcSize;
return bytesWritten + (size_t)(dstPtr - dstStart);
return (size_t)(dstPtr - dstStart);
}

/*! LZ4F_compressUpdate() :
Expand All @@ -1032,10 +1003,10 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
const void* srcBuffer, size_t srcSize,
const LZ4F_compressOptions_t* compressOptionsPtr)
{
return LZ4F_compressUpdateImpl(cctxPtr,
dstBuffer, dstCapacity,
srcBuffer, srcSize,
compressOptionsPtr, LZ4B_COMPRESSED);
return LZ4F_compressUpdateImpl(cctxPtr,
dstBuffer, dstCapacity,
srcBuffer, srcSize,
compressOptionsPtr, LZ4B_COMPRESSED);
}

/*! LZ4F_compressUpdate() :
Expand All @@ -1044,20 +1015,22 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
* src data is either buffered or compressed into @dstBuffer.
* If previously an uncompressed block was written, buffered data is flushed
* before appending compressed data is continued.
* This is only supported when LZ4F_blockIndependent is used
* @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
* @compressOptionsPtr is optional : provide NULL to mean "default".
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
* or an error code if it fails (which can be tested using LZ4F_isError())
* After an error, the state is left in a UB state, and must be re-initialized.
*/
size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
const LZ4F_compressOptions_t* compressOptionsPtr) {
return LZ4F_compressUpdateImpl(cctxPtr,
dstBuffer, dstCapacity,
srcBuffer, srcSize,
compressOptionsPtr, LZ4B_UNCOMPRESSED);
void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
const LZ4F_compressOptions_t* compressOptionsPtr) {
assert(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockIndependent);
return LZ4F_compressUpdateImpl(cctxPtr,
dstBuffer, dstCapacity,
srcBuffer, srcSize,
compressOptionsPtr, LZ4B_UNCOMPRESSED);
}


Expand Down Expand Up @@ -1090,7 +1063,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
cctxPtr->tmpIn, cctxPtr->tmpInSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression);
cctxPtr->prefs.frameInfo.blockChecksumFlag,
cctxPtr->blockCompression);
assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));

if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
Expand Down
1 change: 1 addition & 0 deletions lib/lz4frame.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
* After an error, the state is left in a UB state, and must be re-initialized or freed.
* If previously a compressed block was written, buffered data is flushed
* before appending uncompressed data is continued.
* This is only supported when LZ4F_blockIndependent is used
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
Expand Down
27 changes: 6 additions & 21 deletions lib/lz4hc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1154,26 +1154,6 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
}

/*! LZ4_getDictHCSize():
* Get the size of the dictionary. This can be used for adding data without
* compression to the LZ4 archive. If linked blocked mode is used the memory
* of the dictionary is kept free.
* This way uncompressed data does not influence the effectiveness of the
* dictionary.
* @param LZ4_dict Pointer to the dictionary to get the size of.
* @param dictSize The maximum dictionary size. (Normally 64 KB).
* @return The size of the dictionary.
*/
int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
const LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
assert(prefixSize >= 0);
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
return dictSize;
}


/* LZ4_saveDictHC :
Expand All @@ -1184,7 +1164,12 @@ int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
dictSize = LZ4_getDictHCSize(LZ4_streamHCPtr, dictSize);
int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
assert(prefixSize >= 0);
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0)
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
Expand Down

0 comments on commit 42eb47d

Please sign in to comment.