Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HPCC-17721 Ensure serialized lengths have correct endian format. #10058

Merged
merged 1 commit into from Jun 2, 2017
Merged

HPCC-17721 Ensure serialized lengths have correct endian format. #10058

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 9 additions & 8 deletions system/jlib/jflz.cpp
View file
Open in desktop
Expand Up @@ -703,16 +703,17 @@ class jlib_decl CFastLZExpander : public CFcmpExpander
void fastLZCompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
size32_t outbase = out.length();
size32_t *sz = (size32_t *)out.reserve(len+fastlzSlack(len)+sizeof(size32_t)*2);
*sz = len;
sz++;
*sz = (len>16)?fastlz_compress(src, (int)len, sz+1):16;
if (*sz>=len)
out.append(len);
DelayedMarker<size32_t> cmpSzMarker(out);
void *cmpData = out.reserve(len+fastlzSlack(len));
size32_t sz = (len>16)?fastlz_compress(src, (int)len, cmpData):16;
if (sz>=len)
{
*sz = len;
memcpy(sz+1,src,len);
sz = len;
memcpy(cmpData, src, len);
}
out.setLength(outbase+*sz+sizeof(size32_t)*2);
cmpSzMarker.write(sz);
out.setLength(outbase+sz+sizeof(size32_t)*2);
}

void fastLZDecompressToBuffer(MemoryBuffer & out, const void * src)
Expand Down
25 changes: 11 additions & 14 deletions system/jlib/jlz4.cpp
View file
Open in desktop
Expand Up @@ -147,24 +147,21 @@ class jlib_decl CLZ4Expander : public CFcmpExpander
void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
size32_t outbase = out.length();
size32_t *sz = (size32_t *)out.reserve(LZ4_COMPRESSBOUND(len)+sizeof(size32_t)*2);
*sz = len;
sz++;
out.append(len);
DelayedMarker<size32_t> cmpSzMarker(out);
void *cmpData = out.reserve(LZ4_COMPRESSBOUND(len));
if (len < 64)
{
*sz = len;
memcpy(sz+1,src,len);
}
memcpy(cmpData, src, len);
else
{
*sz = LZ4_compress_default((const char *)src, (char *)(sz+1), len, LZ4_COMPRESSBOUND(len));
if (!*sz)
{
*sz = len;
memcpy(sz+1,src,len);
}
size32_t cmpSz = LZ4_compress_default((const char *)src, (char *)cmpData, len, LZ4_COMPRESSBOUND(len));
if (!cmpSz)
memcpy(cmpData, src, len);
else
len = cmpSz;
}
out.setLength(outbase+*sz+sizeof(size32_t)*2);
cmpSzMarker.write(len);
out.setLength(outbase+len+sizeof(size32_t)*2);
}

void LZ4DecompressToBuffer(MemoryBuffer & out, const void * src)
Expand Down
18 changes: 10 additions & 8 deletions system/jlib/jlzma.cpp
View file
Open in desktop
Expand Up @@ -95,16 +95,18 @@ void LZMACompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
CLZMA lzma;
size32_t outbase = out.length();
size32_t *sz = (size32_t *)out.reserve(len+sizeof(size32_t)*2);
*sz = len;
sz++;
*sz = lzma.compress(src,len,sz+1);
if (*sz>len) {
*sz = len;
memcpy(sz+1,src,len);
out.append(len);
DelayedMarker<size32_t> cmpSzMarker(out);
void *cmpData = out.reserve(len);
size32_t sz = lzma.compress(src, len, cmpData);
Copy link
Contributor

@mckellyln mckellyln Jun 2, 2017 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If lzma compress is ever longer than len then is this a potential buffer overrun ?
Should we reserve len+some extra upper bound as the other compressors do ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mckellyln - Not a new issue, but yes agree think it should

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://track.hpccsystems.com/browse/HPCC-17747 created.

if (sz>len)
{
sz = len;
memcpy(cmpData, src, len);
}
else
out.setLength(outbase+sizeof(size32_t)*2+*sz);
out.setLength(outbase+sizeof(size32_t)*2+sz);
cmpSzMarker.write(sz);
}

void LZMADecompressToBuffer(MemoryBuffer & out, const void * src)
Expand Down