Skip to content

Commit d1112c2

Browse files
committed
Thanks to Eyal Itkin from Check Point Software Technologies.
1 parent 09b9d4f commit d1112c2

File tree

5 files changed

+130
-36
lines changed

5 files changed

+130
-36
lines changed

Diff for: include/freerdp/codec/nsc.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ struct _NSC_CONTEXT
7777
/* color palette allocated by the application */
7878
const BYTE* palette;
7979

80-
void (*decode)(NSC_CONTEXT* context);
81-
void (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData,
80+
BOOL (*decode)(NSC_CONTEXT* context);
81+
BOOL (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData,
8282
UINT32 rowstride);
8383

8484
NSC_CONTEXT_PRIV* priv;

Diff for: libfreerdp/codec/nsc.c

+80-14
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,24 @@
4242
#define NSC_INIT_SIMD(_nsc_context) do { } while (0)
4343
#endif
4444

45-
static void nsc_decode(NSC_CONTEXT* context)
45+
static BOOL nsc_decode(NSC_CONTEXT* context)
4646
{
4747
UINT16 x;
4848
UINT16 y;
49-
UINT16 rw = ROUND_UP_TO(context->width, 8);
50-
BYTE shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
51-
BYTE* bmpdata = context->BitmapData;
49+
UINT16 rw;
50+
BYTE shift;
51+
BYTE* bmpdata;
52+
size_t pos = 0;
53+
54+
if (!context)
55+
return FALSE;
56+
57+
rw = ROUND_UP_TO(context->width, 8);
58+
shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
59+
bmpdata = context->BitmapData;
60+
61+
if (!bmpdata)
62+
return FALSE;
5263

5364
for (y = 0; y < context->height; y++)
5465
{
@@ -80,6 +91,11 @@ static void nsc_decode(NSC_CONTEXT* context)
8091
INT16 r_val = y_val + co_val - cg_val;
8192
INT16 g_val = y_val + cg_val;
8293
INT16 b_val = y_val - co_val - cg_val;
94+
95+
if (pos + 4 > context->BitmapDataLength)
96+
return FALSE;
97+
98+
pos += 4;
8399
*bmpdata++ = MINMAX(b_val, 0, 0xFF);
84100
*bmpdata++ = MINMAX(g_val, 0, 0xFF);
85101
*bmpdata++ = MINMAX(r_val, 0, 0xFF);
@@ -90,9 +106,11 @@ static void nsc_decode(NSC_CONTEXT* context)
90106
aplane++;
91107
}
92108
}
109+
110+
return TRUE;
93111
}
94112

95-
static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
113+
static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 originalSize)
96114
{
97115
UINT32 len;
98116
UINT32 left;
@@ -105,6 +123,10 @@ static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
105123

106124
if (left == 5)
107125
{
126+
if (outSize < 1)
127+
return FALSE;
128+
129+
outSize--;
108130
*out++ = value;
109131
left--;
110132
}
@@ -124,26 +146,42 @@ static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
124146
in += 4;
125147
}
126148

149+
if (outSize < len)
150+
return FALSE;
151+
152+
outSize -= len;
127153
FillMemory(out, len, value);
128154
out += len;
129155
left -= len;
130156
}
131157
else
132158
{
159+
if (outSize < 1)
160+
return FALSE;
161+
162+
outSize--;
133163
*out++ = value;
134164
left--;
135165
}
136166
}
137167

138-
*((UINT32*)out) = *((UINT32*)in);
168+
if ((outSize < 4) || (left < 4))
169+
return FALSE;
170+
171+
memcpy(out, in, 4);
172+
return TRUE;
139173
}
140174

141-
static void nsc_rle_decompress_data(NSC_CONTEXT* context)
175+
static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
142176
{
143177
UINT16 i;
144178
BYTE* rle;
145179
UINT32 planeSize;
146180
UINT32 originalSize;
181+
182+
if (!context)
183+
return FALSE;
184+
147185
rle = context->Planes;
148186

149187
for (i = 0; i < 4; i++)
@@ -152,14 +190,30 @@ static void nsc_rle_decompress_data(NSC_CONTEXT* context)
152190
planeSize = context->PlaneByteCount[i];
153191

154192
if (planeSize == 0)
193+
{
194+
if (context->priv->PlaneBuffersLength < originalSize)
195+
return FALSE;
196+
155197
FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
198+
}
156199
else if (planeSize < originalSize)
157-
nsc_rle_decode(rle, context->priv->PlaneBuffers[i], originalSize);
200+
{
201+
if (!nsc_rle_decode(rle, context->priv->PlaneBuffers[i], context->priv->PlaneBuffersLength,
202+
originalSize))
203+
return FALSE;
204+
}
158205
else
206+
{
207+
if (context->priv->PlaneBuffersLength < originalSize)
208+
return FALSE;
209+
159210
CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
211+
}
160212

161213
rle += planeSize;
162214
}
215+
216+
return TRUE;
163217
}
164218

165219
static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
@@ -396,13 +450,25 @@ BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp,
396450
return FALSE;
397451

398452
/* RLE decode */
399-
PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
400-
nsc_rle_decompress_data(context);
401-
PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)
453+
{
454+
BOOL rc;
455+
PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
456+
rc = nsc_rle_decompress_data(context);
457+
PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)
458+
459+
if (!rc)
460+
return FALSE;
461+
}
402462
/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
403-
PROFILER_ENTER(context->priv->prof_nsc_decode)
404-
context->decode(context);
405-
PROFILER_EXIT(context->priv->prof_nsc_decode)
463+
{
464+
BOOL rc;
465+
PROFILER_ENTER(context->priv->prof_nsc_decode)
466+
rc = context->decode(context);
467+
PROFILER_EXIT(context->priv->prof_nsc_decode)
468+
469+
if (!rc)
470+
return FALSE;
471+
}
406472

407473
if (!freerdp_image_copy(pDstData, DstFormat, nDstStride, nXDst, nYDst,
408474
width, height, context->BitmapData,

Diff for: libfreerdp/codec/nsc_encode.c

+44-18
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context)
5151
for (i = 0; i < 5; i++)
5252
{
5353
BYTE* tmp = (BYTE*) realloc(context->priv->PlaneBuffers[i], length);
54+
5455
if (!tmp)
5556
goto fail;
5657

@@ -87,7 +88,7 @@ static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context)
8788
return FALSE;
8889
}
8990

90-
static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
91+
static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
9192
UINT32 scanline)
9293
{
9394
UINT16 x;
@@ -104,10 +105,20 @@ static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
104105
INT16 b_val;
105106
BYTE a_val;
106107
UINT32 tempWidth;
108+
109+
if (!context || data || (scanline == 0))
110+
return FALSE;
111+
107112
tempWidth = ROUND_UP_TO(context->width, 8);
108113
rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
109114
ccl = context->ColorLossLevel;
110115

116+
if (context->priv->PlaneBuffersLength < rw * scanline)
117+
return FALSE;
118+
119+
if (rw < scanline * 2)
120+
return FALSE;
121+
111122
for (y = 0; y < context->height; y++)
112123
{
113124
src = data + (context->height - 1 - y) * scanline;
@@ -242,31 +253,37 @@ static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
242253
CopyMemory(coplane, coplane - rw, rw);
243254
CopyMemory(cgplane, cgplane - rw, rw);
244255
}
256+
257+
return TRUE;
245258
}
246259

247-
static void nsc_encode_subsampling(NSC_CONTEXT* context)
260+
static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
248261
{
249262
UINT16 x;
250263
UINT16 y;
251-
BYTE* co_dst;
252-
BYTE* cg_dst;
253-
INT8* co_src0;
254-
INT8* co_src1;
255-
INT8* cg_src0;
256-
INT8* cg_src1;
257264
UINT32 tempWidth;
258265
UINT32 tempHeight;
266+
267+
if (!context)
268+
return FALSE;
269+
259270
tempWidth = ROUND_UP_TO(context->width, 8);
260271
tempHeight = ROUND_UP_TO(context->height, 2);
261272

273+
if (tempHeight == 0)
274+
return FALSE;
275+
276+
if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
277+
return FALSE;
278+
262279
for (y = 0; y < tempHeight >> 1; y++)
263280
{
264-
co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
265-
cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
266-
co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
267-
co_src1 = co_src0 + tempWidth;
268-
cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
269-
cg_src1 = cg_src0 + tempWidth;
281+
BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
282+
BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
283+
const INT8* co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
284+
const INT8* co_src1 = co_src0 + tempWidth;
285+
const INT8* cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
286+
const INT8* cg_src1 = cg_src0 + tempWidth;
270287

271288
for (x = 0; x < tempWidth >> 1; x++)
272289
{
@@ -280,19 +297,28 @@ static void nsc_encode_subsampling(NSC_CONTEXT* context)
280297
cg_src1 += 2;
281298
}
282299
}
300+
301+
return TRUE;
283302
}
284303

285-
void nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata, UINT32 rowstride)
304+
BOOL nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata, UINT32 rowstride)
286305
{
287-
nsc_encode_argb_to_aycocg(context, bmpdata, rowstride);
306+
if (!context || !bmpdata || (rowstride == 0))
307+
return FALSE;
308+
309+
if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
310+
return FALSE;
288311

289312
if (context->ChromaSubsamplingLevel)
290313
{
291-
nsc_encode_subsampling(context);
314+
if (!nsc_encode_subsampling(context))
315+
return FALSE;
292316
}
317+
318+
return TRUE;
293319
}
294320

295-
static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 originalSize)
321+
static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 originalSize)
296322
{
297323
UINT32 left;
298324
UINT32 runlength = 1;

Diff for: libfreerdp/codec/nsc_encode.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
#include <freerdp/api.h>
2626

27-
FREERDP_LOCAL void nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata,
27+
FREERDP_LOCAL BOOL nsc_encode(NSC_CONTEXT* context, const BYTE* bmpdata,
2828
UINT32 rowstride);
2929

3030
#endif /* FREERDP_LIB_CODEC_NSC_ENCODE_H */

Diff for: libfreerdp/codec/nsc_sse2.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,7 @@ static void nsc_encode_subsampling_sse2(NSC_CONTEXT* context)
385385
}
386386
}
387387

388-
static void nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
388+
static BOOL nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
389389
UINT32 scanline)
390390
{
391391
nsc_encode_argb_to_aycocg_sse2(context, data, scanline);
@@ -394,6 +394,8 @@ static void nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
394394
{
395395
nsc_encode_subsampling_sse2(context);
396396
}
397+
398+
return TRUE;
397399
}
398400

399401
void nsc_init_sse2(NSC_CONTEXT* context)

0 commit comments

Comments
 (0)