Skip to content
This repository was archived by the owner on Dec 1, 2017. It is now read-only.

Commit 3ca657a

Browse files
author
erouault
committed
* libtiff/tif_predict.h, libtiff/tif_predict.c:
Replace assertions by runtime checks to avoid assertions in debug mode, or buffer overflows in release mode. Can happen when dealing with unusual tile size like YCbCr with subsampling. Reported as MSVR 35105 by Axel Souchet & Vishal Chauhan from the MSRC Vulnerabilities & Mitigations team.
1 parent ed9e07c commit 3ca657a

File tree

3 files changed

+121
-47
lines changed

3 files changed

+121
-47
lines changed

Diff for: ChangeLog

+9
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,12 @@
1+
2016-10-31 Even Rouault <even.rouault at spatialys.com>
2+
3+
* libtiff/tif_predict.h, libtiff/tif_predict.c:
4+
Replace assertions by runtime checks to avoid assertions in debug mode,
5+
or buffer overflows in release mode. Can happen when dealing with
6+
unusual tile size like YCbCr with subsampling. Reported as MSVR 35105
7+
by Axel Souchet & Vishal Chauhan from the MSRC Vulnerabilities & Mitigations
8+
team.
9+
110
2016-10-26 Even Rouault <even.rouault at spatialys.com>
211

312
* tools/fax2tiff.c: fix segfault when specifying -r without

Diff for: libtiff/tif_predict.c

+108-45
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,18 @@
3434

3535
#define PredictorState(tif) ((TIFFPredictorState*) (tif)->tif_data)
3636

37-
static void horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
38-
static void horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
39-
static void horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
40-
static void swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
41-
static void swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
42-
static void horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
43-
static void horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
44-
static void horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
45-
static void swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
46-
static void swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
47-
static void fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
48-
static void fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
37+
static int horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
38+
static int horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
39+
static int horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
40+
static int swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
41+
static int swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
42+
static int horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
43+
static int horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
44+
static int horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
45+
static int swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
46+
static int swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
47+
static int fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
48+
static int fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
4949
static int PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
5050
static int PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
5151
static int PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s);
@@ -273,13 +273,19 @@ PredictorSetupEncode(TIFF* tif)
273273
/* - when storing into the byte stream, we explicitly mask with 0xff so */
274274
/* as to make icc -check=conversions happy (not necessary by the standard) */
275275

276-
static void
276+
static int
277277
horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
278278
{
279279
tmsize_t stride = PredictorState(tif)->stride;
280280

281281
unsigned char* cp = (unsigned char*) cp0;
282-
assert((cc%stride)==0);
282+
if((cc%stride)!=0)
283+
{
284+
TIFFErrorExt(tif->tif_clientdata, "horAcc8",
285+
"%s", "(cc%stride)!=0");
286+
return 0;
287+
}
288+
283289
if (cc > stride) {
284290
/*
285291
* Pipeline the most common cases.
@@ -321,26 +327,32 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
321327
} while (cc>0);
322328
}
323329
}
330+
return 1;
324331
}
325332

326-
static void
333+
static int
327334
swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
328335
{
329336
uint16* wp = (uint16*) cp0;
330337
tmsize_t wc = cc / 2;
331338

332339
TIFFSwabArrayOfShort(wp, wc);
333-
horAcc16(tif, cp0, cc);
340+
return horAcc16(tif, cp0, cc);
334341
}
335342

336-
static void
343+
static int
337344
horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
338345
{
339346
tmsize_t stride = PredictorState(tif)->stride;
340347
uint16* wp = (uint16*) cp0;
341348
tmsize_t wc = cc / 2;
342349

343-
assert((cc%(2*stride))==0);
350+
if((cc%(2*stride))!=0)
351+
{
352+
TIFFErrorExt(tif->tif_clientdata, "horAcc16",
353+
"%s", "cc%(2*stride))!=0");
354+
return 0;
355+
}
344356

345357
if (wc > stride) {
346358
wc -= stride;
@@ -349,26 +361,32 @@ horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
349361
wc -= stride;
350362
} while (wc > 0);
351363
}
364+
return 1;
352365
}
353366

354-
static void
367+
static int
355368
swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
356369
{
357370
uint32* wp = (uint32*) cp0;
358371
tmsize_t wc = cc / 4;
359372

360373
TIFFSwabArrayOfLong(wp, wc);
361-
horAcc32(tif, cp0, cc);
374+
return horAcc32(tif, cp0, cc);
362375
}
363376

364-
static void
377+
static int
365378
horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
366379
{
367380
tmsize_t stride = PredictorState(tif)->stride;
368381
uint32* wp = (uint32*) cp0;
369382
tmsize_t wc = cc / 4;
370383

371-
assert((cc%(4*stride))==0);
384+
if((cc%(4*stride))!=0)
385+
{
386+
TIFFErrorExt(tif->tif_clientdata, "horAcc32",
387+
"%s", "cc%(4*stride))!=0");
388+
return 0;
389+
}
372390

373391
if (wc > stride) {
374392
wc -= stride;
@@ -377,12 +395,13 @@ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
377395
wc -= stride;
378396
} while (wc > 0);
379397
}
398+
return 1;
380399
}
381400

382401
/*
383402
* Floating point predictor accumulation routine.
384403
*/
385-
static void
404+
static int
386405
fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
387406
{
388407
tmsize_t stride = PredictorState(tif)->stride;
@@ -392,10 +411,15 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
392411
uint8 *cp = (uint8 *) cp0;
393412
uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
394413

395-
assert((cc%(bps*stride))==0);
414+
if(cc%(bps*stride)!=0)
415+
{
416+
TIFFErrorExt(tif->tif_clientdata, "fpAcc",
417+
"%s", "cc%(bps*stride))!=0");
418+
return 0;
419+
}
396420

397421
if (!tmp)
398-
return;
422+
return 0;
399423

400424
while (count > stride) {
401425
REPEAT4(stride, cp[stride] =
@@ -417,6 +441,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
417441
}
418442
}
419443
_TIFFfree(tmp);
444+
return 1;
420445
}
421446

422447
/*
@@ -432,8 +457,7 @@ PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
432457
assert(sp->decodepfunc != NULL);
433458

434459
if ((*sp->decoderow)(tif, op0, occ0, s)) {
435-
(*sp->decodepfunc)(tif, op0, occ0);
436-
return 1;
460+
return (*sp->decodepfunc)(tif, op0, occ0);
437461
} else
438462
return 0;
439463
}
@@ -456,10 +480,16 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
456480
if ((*sp->decodetile)(tif, op0, occ0, s)) {
457481
tmsize_t rowsize = sp->rowsize;
458482
assert(rowsize > 0);
459-
assert((occ0%rowsize)==0);
483+
if((occ0%rowsize) !=0)
484+
{
485+
TIFFErrorExt(tif->tif_clientdata, "PredictorDecodeTile",
486+
"%s", "occ0%rowsize != 0");
487+
return 0;
488+
}
460489
assert(sp->decodepfunc != NULL);
461490
while (occ0 > 0) {
462-
(*sp->decodepfunc)(tif, op0, rowsize);
491+
if( !(*sp->decodepfunc)(tif, op0, rowsize) )
492+
return 0;
463493
occ0 -= rowsize;
464494
op0 += rowsize;
465495
}
@@ -468,14 +498,19 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
468498
return 0;
469499
}
470500

471-
static void
501+
static int
472502
horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
473503
{
474504
TIFFPredictorState* sp = PredictorState(tif);
475505
tmsize_t stride = sp->stride;
476506
unsigned char* cp = (unsigned char*) cp0;
477507

478-
assert((cc%stride)==0);
508+
if((cc%stride)!=0)
509+
{
510+
TIFFErrorExt(tif->tif_clientdata, "horDiff8",
511+
"%s", "(cc%stride)!=0");
512+
return 0;
513+
}
479514

480515
if (cc > stride) {
481516
cc -= stride;
@@ -513,17 +548,23 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
513548
} while ((cc -= stride) > 0);
514549
}
515550
}
551+
return 1;
516552
}
517553

518-
static void
554+
static int
519555
horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
520556
{
521557
TIFFPredictorState* sp = PredictorState(tif);
522558
tmsize_t stride = sp->stride;
523559
uint16 *wp = (uint16*) cp0;
524560
tmsize_t wc = cc/2;
525561

526-
assert((cc%(2*stride))==0);
562+
if((cc%(2*stride))!=0)
563+
{
564+
TIFFErrorExt(tif->tif_clientdata, "horDiff8",
565+
"%s", "(cc%(2*stride))!=0");
566+
return 0;
567+
}
527568

528569
if (wc > stride) {
529570
wc -= stride;
@@ -533,28 +574,36 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
533574
wc -= stride;
534575
} while (wc > 0);
535576
}
577+
return 1;
536578
}
537579

538-
static void
580+
static int
539581
swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
540582
{
541583
uint16* wp = (uint16*) cp0;
542584
tmsize_t wc = cc / 2;
543585

544-
horDiff16(tif, cp0, cc);
586+
if( !horDiff16(tif, cp0, cc) )
587+
return 0;
545588

546589
TIFFSwabArrayOfShort(wp, wc);
590+
return 1;
547591
}
548592

549-
static void
593+
static int
550594
horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
551595
{
552596
TIFFPredictorState* sp = PredictorState(tif);
553597
tmsize_t stride = sp->stride;
554598
uint32 *wp = (uint32*) cp0;
555599
tmsize_t wc = cc/4;
556600

557-
assert((cc%(4*stride))==0);
601+
if((cc%(4*stride))!=0)
602+
{
603+
TIFFErrorExt(tif->tif_clientdata, "horDiff32",
604+
"%s", "(cc%(4*stride))!=0");
605+
return 0;
606+
}
558607

559608
if (wc > stride) {
560609
wc -= stride;
@@ -564,23 +613,26 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
564613
wc -= stride;
565614
} while (wc > 0);
566615
}
616+
return 1;
567617
}
568618

569-
static void
619+
static int
570620
swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
571621
{
572622
uint32* wp = (uint32*) cp0;
573623
tmsize_t wc = cc / 4;
574624

575-
horDiff32(tif, cp0, cc);
625+
if( !horDiff32(tif, cp0, cc) )
626+
return 0;
576627

577628
TIFFSwabArrayOfLong(wp, wc);
629+
return 1;
578630
}
579631

580632
/*
581633
* Floating point predictor differencing routine.
582634
*/
583-
static void
635+
static int
584636
fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
585637
{
586638
tmsize_t stride = PredictorState(tif)->stride;
@@ -590,10 +642,14 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
590642
uint8 *cp = (uint8 *) cp0;
591643
uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
592644

593-
assert((cc%(bps*stride))==0);
594-
645+
if((cc%(bps*stride))!=0)
646+
{
647+
TIFFErrorExt(tif->tif_clientdata, "fpDiff",
648+
"%s", "(cc%(bps*stride))!=0");
649+
return 0;
650+
}
595651
if (!tmp)
596-
return;
652+
return 0;
597653

598654
_TIFFmemcpy(tmp, cp0, cc);
599655
for (count = 0; count < wc; count++) {
@@ -613,6 +669,7 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
613669
cp += cc - stride - 1;
614670
for (count = cc; count > stride; count -= stride)
615671
REPEAT4(stride, cp[stride] = (unsigned char)((cp[stride] - cp[0])&0xff); cp--)
672+
return 1;
616673
}
617674

618675
static int
@@ -625,7 +682,8 @@ PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s)
625682
assert(sp->encoderow != NULL);
626683

627684
/* XXX horizontal differencing alters user's data XXX */
628-
(*sp->encodepfunc)(tif, bp, cc);
685+
if( !(*sp->encodepfunc)(tif, bp, cc) )
686+
return 0;
629687
return (*sp->encoderow)(tif, bp, cc, s);
630688
}
631689

@@ -660,7 +718,12 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)
660718

661719
rowsize = sp->rowsize;
662720
assert(rowsize > 0);
663-
assert((cc0%rowsize)==0);
721+
if((cc0%rowsize)!=0)
722+
{
723+
TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",
724+
"%s", "(cc0%rowsize)!=0");
725+
return 0;
726+
}
664727
while (cc > 0) {
665728
(*sp->encodepfunc)(tif, bp, rowsize);
666729
cc -= rowsize;

0 commit comments

Comments
 (0)