Skip to content
Permalink
Browse files

Fixed numerous integer overflow problems in the code for packet itera…

…tors

in the JPC decoder.
  • Loading branch information...
mdadams committed Nov 27, 2016
1 parent f4a6b95 commit aa0b0f79ade5eef8b0e7a214c03f5af54b36ba7d
Showing with 102 additions and 74 deletions.
  1. +4 −0 src/libjasper/include/jasper/jas_types.h
  2. +77 −53 src/libjasper/jpc/jpc_t2cod.c
  3. +21 −21 src/libjasper/jpc/jpc_t2dec.c
@@ -116,6 +116,10 @@
#define JAS_CAST(t, e) \
((t) (e))

/* The number of bits in the integeral type uint_fast32_t. */
/* NOTE: This could underestimate the size on some exotic architectures. */
#define JAS_UINTFAST32_NUMBITS (8 * sizeof(uint_fast32_t))

#ifdef __cplusplus
extern "C" {
#endif
@@ -198,7 +198,8 @@ static int jpc_pi_nextrlcp(register jpc_pi_t *pi)
JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
for (pi->compno = pchg->compnostart, pi->picomp =
&pi->picomps[pi->compno]; pi->compno < pi->numcomps &&
pi->compno < JAS_CAST(int, pchg->compnoend); ++pi->compno, ++pi->picomp) {
pi->compno < JAS_CAST(int, pchg->compnoend); ++pi->compno,
++pi->picomp) {
if (pi->rlvlno >= pi->picomp->numrlvls) {
continue;
}
@@ -247,10 +248,17 @@ static int jpc_pi_nextrpcl(register jpc_pi_t *pi)
++compno, ++picomp) {
for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
picomp->numrlvls; ++rlvlno, ++pirlvl) {
xstep = picomp->hsamp * (1 << (pirlvl->prcwidthexpn +
picomp->numrlvls - rlvlno - 1));
ystep = picomp->vsamp * (1 << (pirlvl->prcheightexpn +
picomp->numrlvls - rlvlno - 1));
// Check for the potential for overflow problems.
if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2 ||
pirlvl->prcheightexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2) {
return -1;
}
xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
(pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
(pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
}
@@ -280,21 +288,24 @@ static int jpc_pi_nextrpcl(register jpc_pi_t *pi)
rpy = r + pi->pirlvl->prcheightexpn;
trx0 = JPC_CEILDIV(pi->xstart, pi->picomp->hsamp << r);
try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx)))
|| !(pi->x % (1 << rpx))) &&
((pi->y == pi->ystart && ((try0 << r) % (1 << rpy)))
|| !(pi->y % (1 << rpy)))) {
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
<< r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
<< r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
pi->pirlvl->prcheightexpn);
if (((pi->x == pi->xstart &&
((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx)))
|| !(pi->x % (JAS_CAST(uint_fast32_t, 1) << rpx))) &&
((pi->y == pi->ystart &&
((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy)))
|| !(pi->y % (JAS_CAST(uint_fast32_t, 1) << rpy)))) {
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;

assert(pi->prcno < pi->pirlvl->numprcs);
for (pi->lyrno = 0; pi->lyrno <
pi->numlyrs && pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
pi->numlyrs && pi->lyrno < JAS_CAST(int,
pchg->lyrnoend); ++pi->lyrno) {
prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
if (pi->lyrno >= *prclyrno) {
++(*prclyrno);
@@ -339,16 +350,19 @@ static int jpc_pi_nextpcrl(register jpc_pi_t *pi)
++compno, ++picomp) {
for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
picomp->numrlvls; ++rlvlno, ++pirlvl) {
xstep = picomp->hsamp * (1 <<
(pirlvl->prcwidthexpn + picomp->numrlvls -
rlvlno - 1));
ystep = picomp->vsamp * (1 <<
(pirlvl->prcheightexpn + picomp->numrlvls -
rlvlno - 1));
pi->xstep = (!pi->xstep) ? xstep :
JAS_MIN(pi->xstep, xstep);
pi->ystep = (!pi->ystep) ? ystep :
JAS_MIN(pi->ystep, ystep);
// Check for the potential for overflow problems.
if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2 ||
pirlvl->prcheightexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2) {
return -1;
}
xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
(pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
(pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
}
}
pi->prgvolfirst = 0;
@@ -375,20 +389,23 @@ static int jpc_pi_nextpcrl(register jpc_pi_t *pi)
try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
rpx = r + pi->pirlvl->prcwidthexpn;
rpy = r + pi->pirlvl->prcheightexpn;
if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx))) ||
if (((pi->x == pi->xstart &&
((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
!(pi->x % (pi->picomp->hsamp << rpx))) &&
((pi->y == pi->ystart && ((try0 << r) % (1 << rpy))) ||
((pi->y == pi->ystart &&
((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
!(pi->y % (pi->picomp->vsamp << rpy)))) {
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
<< r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
<< r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
pi->pirlvl->prcheightexpn);
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
assert(pi->prcno < pi->pirlvl->numprcs);
for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
++pi->lyrno) {
prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
if (pi->lyrno >= *prclyrno) {
++(*prclyrno);
@@ -426,10 +443,17 @@ static int jpc_pi_nextcprl(register jpc_pi_t *pi)
pi->prgvolfirst = 0;
}

for (pi->compno = pchg->compnostart, pi->picomp =
&pi->picomps[pi->compno]; pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno < pi->numcomps; ++pi->compno,
++pi->picomp) {
for (pi->compno = pchg->compnostart, pi->picomp = &pi->picomps[pi->compno];
pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno < pi->numcomps;
++pi->compno, ++pi->picomp) {
pirlvl = pi->picomp->pirlvls;
// Check for the potential for overflow problems.
if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2 ||
pirlvl->prcheightexpn + pi->picomp->numrlvls >
JAS_UINTFAST32_NUMBITS - 2) {
return -1;
}
pi->xstep = pi->picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
(pirlvl->prcwidthexpn + pi->picomp->numrlvls - 1));
pi->ystep = pi->picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
@@ -459,23 +483,23 @@ static int jpc_pi_nextcprl(register jpc_pi_t *pi)
try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
rpx = r + pi->pirlvl->prcwidthexpn;
rpy = r + pi->pirlvl->prcheightexpn;
if (((pi->x == pi->xstart && ((trx0 << r) % (1 << rpx))) ||
if (((pi->x == pi->xstart &&
((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
!(pi->x % (pi->picomp->hsamp << rpx))) &&
((pi->y == pi->ystart && ((try0 << r) % (1 << rpy))) ||
((pi->y == pi->ystart &&
((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
!(pi->y % (pi->picomp->vsamp << rpy)))) {
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x, pi->picomp->hsamp
<< r), pi->pirlvl->prcwidthexpn) - JPC_FLOORDIVPOW2(trx0,
pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y, pi->picomp->vsamp
<< r), pi->pirlvl->prcheightexpn) - JPC_FLOORDIVPOW2(try0,
pi->pirlvl->prcheightexpn);
pi->prcno = prcvind *
pi->pirlvl->numhprcs +
prchind;
assert(pi->prcno <
pi->pirlvl->numprcs);
for (pi->lyrno = 0; pi->lyrno <
pi->numlyrs && pi->lyrno < JAS_CAST(int, pchg->lyrnoend); ++pi->lyrno) {
prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
assert(pi->prcno < pi->pirlvl->numprcs);
for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
++pi->lyrno) {
prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
if (pi->lyrno >= *prclyrno) {
++(*prclyrno);
@@ -423,39 +423,39 @@ int jpc_dec_decodepkts(jpc_dec_t *dec, jas_stream_t *pkthdrstream, jas_stream_t
tile = dec->curtile;
pi = tile->pi;
for (;;) {
if (!tile->pkthdrstream || jas_stream_peekc(tile->pkthdrstream) == EOF) {
switch (jpc_dec_lookahead(in)) {
case JPC_MS_EOC:
case JPC_MS_SOT:
return 0;
break;
case JPC_MS_SOP:
case JPC_MS_EPH:
case 0:
break;
default:
return -1;
break;
if (!tile->pkthdrstream || jas_stream_peekc(tile->pkthdrstream) == EOF) {
switch (jpc_dec_lookahead(in)) {
case JPC_MS_EOC:
case JPC_MS_SOT:
return 0;
break;
case JPC_MS_SOP:
case JPC_MS_EPH:
case 0:
break;
default:
return -1;
break;
}
}
}
if ((ret = jpc_pi_next(pi))) {
return ret;
}
if (dec->maxpkts >= 0 && dec->numpkts >= dec->maxpkts) {
jas_eprintf("warning: stopping decode prematurely as requested\n");
return 0;
}
if (dec->maxpkts >= 0 && dec->numpkts >= dec->maxpkts) {
jas_eprintf("warning: stopping decode prematurely as requested\n");
return 0;
}
if (jas_getdbglevel() >= 1) {
jas_eprintf("packet offset=%08ld prg=%d cmptno=%02d "
"rlvlno=%02d prcno=%03d lyrno=%02d\n", (long)
jas_stream_getrwcount(in), jpc_pi_prg(pi), jpc_pi_cmptno(pi),
jpc_pi_rlvlno(pi), jpc_pi_prcno(pi), jpc_pi_lyrno(pi));
}
if (jpc_dec_decodepkt(dec, pkthdrstream, in, jpc_pi_cmptno(pi), jpc_pi_rlvlno(pi),
jpc_pi_prcno(pi), jpc_pi_lyrno(pi))) {
if (jpc_dec_decodepkt(dec, pkthdrstream, in, jpc_pi_cmptno(pi),
jpc_pi_rlvlno(pi), jpc_pi_prcno(pi), jpc_pi_lyrno(pi))) {
return -1;
}
++dec->numpkts;
++dec->numpkts;
}

return 0;

0 comments on commit aa0b0f7

Please sign in to comment.
You can’t perform that action at this time.