Skip to content

Commit

Permalink
Fixed CVE-2018-8788
Browse files Browse the repository at this point in the history 
Thanks to Eyal Itkin from Check Point Software Technologies.
  • Loading branch information
@akallabeth
akallabeth committed Nov 20, 2018
1 parent 09b9d4f commit d1112c2
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 36 deletions.
4 changes: 2 additions & 2 deletions include/freerdp/codec/nsc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ struct _NSC_CONTEXT
/* color palette allocated by the application */
const BYTE* palette;

void (*decode)(NSC_CONTEXT* context);
void (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData,
BOOL (*decode)(NSC_CONTEXT* context);
BOOL (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData,
UINT32 rowstride);

NSC_CONTEXT_PRIV* priv;
Expand Down
94 changes: 80 additions & 14 deletions libfreerdp/codec/nsc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,24 @@
#define NSC_INIT_SIMD(_nsc_context) do { } while (0)
#endif

static void nsc_decode(NSC_CONTEXT* context)
static BOOL nsc_decode(NSC_CONTEXT* context)
{
UINT16 x;
UINT16 y;
UINT16 rw = ROUND_UP_TO(context->width, 8);
BYTE shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
BYTE* bmpdata = context->BitmapData;
UINT16 rw;
BYTE shift;
BYTE* bmpdata;
size_t pos = 0;

if (!context)
return FALSE;

rw = ROUND_UP_TO(context->width, 8);
shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
bmpdata = context->BitmapData;

if (!bmpdata)
return FALSE;

for (y = 0; y < context->height; y++)
{
Expand Down Expand Up @@ -80,6 +91,11 @@ static void nsc_decode(NSC_CONTEXT* context)
INT16 r_val = y_val + co_val - cg_val;
INT16 g_val = y_val + cg_val;
INT16 b_val = y_val - co_val - cg_val;

if (pos + 4 > context->BitmapDataLength)
return FALSE;

pos += 4;
*bmpdata++ = MINMAX(b_val, 0, 0xFF);
*bmpdata++ = MINMAX(g_val, 0, 0xFF);
*bmpdata++ = MINMAX(r_val, 0, 0xFF);
Expand All @@ -90,9 +106,11 @@ static void nsc_decode(NSC_CONTEXT* context)
aplane++;
}
}

return TRUE;
}

static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 originalSize)
{
UINT32 len;
UINT32 left;
Expand All @@ -105,6 +123,10 @@ static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)

if (left == 5)
{
if (outSize < 1)
return FALSE;

outSize--;
*out++ = value;
left--;
}
Expand All @@ -124,26 +146,42 @@ static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
in += 4;
}

if (outSize < len)
return FALSE;

outSize -= len;
FillMemory(out, len, value);
out += len;
left -= len;
}
else
{
if (outSize < 1)
return FALSE;

outSize--;
*out++ = value;
left--;
}
}

*((UINT32*)out) = *((UINT32*)in);
if ((outSize < 4) || (left < 4))
return FALSE;

memcpy(out, in, 4);
return TRUE;
}

static void nsc_rle_decompress_data(NSC_CONTEXT* context)
static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
{
UINT16 i;
BYTE* rle;
UINT32 planeSize;
UINT32 originalSize;

if (!context)
return FALSE;

rle = context->Planes;

for (i = 0; i < 4; i++)
Expand All @@ -152,14 +190,30 @@ static void nsc_rle_decompress_data(NSC_CONTEXT* context)
planeSize = context->PlaneByteCount[i];

if (planeSize == 0)
{
if (context->priv->PlaneBuffersLength < originalSize)
return FALSE;

FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
}
else if (planeSize < originalSize)
nsc_rle_decode(rle, context->priv->PlaneBuffers[i], originalSize);
{
if (!nsc_rle_decode(rle, context->priv->PlaneBuffers[i], context->priv->PlaneBuffersLength,
originalSize))
return FALSE;
}
else
{
if (context->priv->PlaneBuffersLength < originalSize)
return FALSE;

CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
}

rle += planeSize;
}

return TRUE;
}

static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
Expand Down Expand Up @@ -396,13 +450,25 @@ BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp,
return FALSE;

/* RLE decode */
PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
nsc_rle_decompress_data(context);
PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)
{
BOOL rc;
PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
rc = nsc_rle_decompress_data(context);
PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)

if (!rc)
return FALSE;
}
/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
PROFILER_ENTER(context->priv->prof_nsc_decode)
context->decode(context);
PROFILER_EXIT(context->priv->prof_nsc_decode)
{
BOOL rc;
PROFILER_ENTER(context->priv->prof_nsc_decode)
rc = context->decode(context);
PROFILER_EXIT(context->priv->prof_nsc_decode)

if (!rc)
return FALSE;
}

if (!freerdp_image_copy(pDstData, DstFormat, nDstStride, nXDst, nYDst,
width, height, context->BitmapData,
Expand Down
62 changes: 44 additions & 18 deletions libfreerdp/codec/nsc_encode.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context)
for (i = 0; i < 5; i++)
{
BYTE* tmp = (BYTE*) realloc(context->priv->PlaneBuffers[i], length);

if (!tmp)
goto fail;

Expand Down Expand Up @@ -87,7 +88,7 @@ static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context)
return FALSE;
}

static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
UINT32 scanline)
{
UINT16 x;
Expand All @@ -104,10 +105,20 @@ static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
INT16 b_val;
BYTE a_val;
UINT32 tempWidth;

if (!context || data || (scanline == 0))
return FALSE;

tempWidth = ROUND_UP_TO(context->width, 8);
rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
ccl = context->ColorLossLevel;

if (context->priv->PlaneBuffersLength < rw * scanline)
return FALSE;

if (rw < scanline * 2)
return FALSE;

for (y = 0; y < context->height; y++)
{
src = data + (context->height - 1 - y) * scanline;
Expand Down Expand Up @@ -242,31 +253,37 @@ static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
CopyMemory(coplane, coplane - rw, rw);
CopyMemory(cgplane, cgplane - rw, rw);
}

return TRUE;
}

static void nsc_encode_subsampling(NSC_CONTEXT* context)
static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
{
UINT16 x;
UINT16 y;
BYTE* co_dst;
BYTE* cg_dst;
INT8* co_src0;
INT8* co_src1;
INT8* cg_src0;
INT8* cg_src1;
UINT32 tempWidth;
UINT32 tempHeight;

if (!context)
return FALSE;

tempWidth = ROUND_UP_TO(context->width, 8);
tempHeight = ROUND_UP_TO(context->height, 2);

if (tempHeight == 0)
return FALSE;

if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
return FALSE;

for (y = 0; y < tempHeight >> 1; y++)
{
co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
co_src1 = co_src0 + tempWidth;
cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
cg_src1 = cg_src0 + tempWidth;
BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
const INT8* co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
const INT8* co_src1 = co_src0 + tempWidth;
const INT8* cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
const INT8* cg_src1 = cg_src0 + tempWidth;

for (x = 0; x < tempWidth >> 1; x++)
{
Expand All @@ -280,19 +297,28 @@ static void nsc_encode_subsampling(NSC_CONTEXT* context)
cg_src1 += 2;
}
}

return TRUE;
}

void nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata, UINT32 rowstride)
BOOL nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata, UINT32 rowstride)
{
nsc_encode_argb_to_aycocg(context, bmpdata, rowstride);
if (!context || !bmpdata || (rowstride == 0))
return FALSE;

if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
return FALSE;

if (context->ChromaSubsamplingLevel)
{
nsc_encode_subsampling(context);
if (!nsc_encode_subsampling(context))
return FALSE;
}

return TRUE;
}

static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 originalSize)
static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 originalSize)
{
UINT32 left;
UINT32 runlength = 1;
Expand Down
2 changes: 1 addition & 1 deletion libfreerdp/codec/nsc_encode.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

#include <freerdp/api.h>

FREERDP_LOCAL void nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata,
FREERDP_LOCAL BOOL nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata,
UINT32 rowstride);

#endif /* FREERDP_LIB_CODEC_NSC_ENCODE_H */
4 changes: 3 additions & 1 deletion libfreerdp/codec/nsc_sse2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ static void nsc_encode_subsampling_sse2(NSC_CONTEXT* context)
}
}

static void nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
static BOOL nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
UINT32 scanline)
{
nsc_encode_argb_to_aycocg_sse2(context, data, scanline);
Expand All @@ -394,6 +394,8 @@ static void nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
{
nsc_encode_subsampling_sse2(context);
}

return TRUE;
}

void nsc_init_sse2(NSC_CONTEXT* context)
Expand Down

0 comments on commit d1112c2

Please sign in to comment.