Fixed a few bugs in the RAS encoder and decoder where errors were tested

with assertions instead of being gracefully handled.

src/libjasper/ras/ras_dec.c

@@ -260,9 +260,16 @@ static int ras_getdatastd(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap,
 	/* Avoid compiler warnings about unused parameters. */
 	cmap = 0;
 
+	assert(jas_image_numcmpts(image) <= 3);
+
+	for (i = 0; i < 3; ++i) {
+		data[i] = 0;
+	}
+
 	for (i = 0; i < jas_image_numcmpts(image); ++i) {
-		data[i] = jas_matrix_create(1, jas_image_width(image));
-		assert(data[i]);
+		if (!(data[i] = jas_matrix_create(1, jas_image_width(image)))) {
+			goto error;
+		}
 	}
 
 	pad = RAS_ROWSIZE(hdr) - (hdr->width * hdr->depth + 7) / 8;
@@ -273,7 +280,7 @@ static int ras_getdatastd(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap,
 		for (x = 0; x < hdr->width; x++) {
 			while (nz < hdr->depth) {
 				if ((c = jas_stream_getc(in)) == EOF) {
-					return -1;
+					goto error;
 				}
 				z = (z << 8) | c;
 				nz += 8;
@@ -293,22 +300,31 @@ static int ras_getdatastd(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap,
 		}
 		if (pad) {
 			if ((c = jas_stream_getc(in)) == EOF) {
-				return -1;
+				goto error;
 			}
 		}
 		for (i = 0; i < jas_image_numcmpts(image); ++i) {
 			if (jas_image_writecmpt(image, i, 0, y, hdr->width, 1,
 			  data[i])) {
-				return -1;
+				goto error;
 			}
 		}
 	}
 
 	for (i = 0; i < jas_image_numcmpts(image); ++i) {
 		jas_matrix_destroy(data[i]);
+		data[i] = 0;
 	}
 
 	return 0;
+
+error:
+	for (i = 0; i < 3; ++i) {
+		if (data[i]) {
+			jas_matrix_destroy(data[i]);
+		}
+	}
+	return -1;
 }
 
 static int ras_getcmap(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap)
@@ -327,7 +343,9 @@ static int ras_getcmap(jas_stream_t *in, ras_hdr_t *hdr, ras_cmap_t *cmap)
 		{
 		jas_eprintf("warning: palettized images not fully supported\n");
 		numcolors = 1 << hdr->depth;
-		assert(numcolors <= RAS_CMAP_MAXSIZ);
+		if (numcolors > RAS_CMAP_MAXSIZ) {
+			return -1;
+		}
 		actualnumcolors = hdr->maplength / 3;
 		for (i = 0; i < numcolors; i++) {
 			cmap->data[i] = 0;

src/libjasper/ras/ras_enc.c

@@ -230,9 +230,17 @@ static int ras_putdatastd(jas_stream_t *out, ras_hdr_t *hdr, jas_image_t *image,
 	jas_matrix_t *data[3];
 	int i;
 
+	assert(numcmpts <= 3);
+
+	for (i = 0; i < 3; ++i) {
+		data[i] = 0;
+	}
+
 	for (i = 0; i < numcmpts; ++i) {
-		data[i] = jas_matrix_create(jas_image_height(image), jas_image_width(image));
-		assert(data[i]);
+		if (!(data[i] = jas_matrix_create(jas_image_height(image),
+		  jas_image_width(image)))) {
+			goto error;
+		}
 	}
 
 	rowsize = RAS_ROWSIZE(hdr);
@@ -244,7 +252,7 @@ static int ras_putdatastd(jas_stream_t *out, ras_hdr_t *hdr, jas_image_t *image,
 		for (i = 0; i < numcmpts; ++i) {
 			if (jas_image_readcmpt(image, cmpts[i], 0, y,
 					jas_image_width(image), 1, data[i])) {
-				return -1;
+				goto error;
 			}
 		}
 		z = 0;
@@ -263,7 +271,7 @@ static int ras_putdatastd(jas_stream_t *out, ras_hdr_t *hdr, jas_image_t *image,
 			while (nz >= 8) {
 				c = (z >> (nz - 8)) & 0xff;
 				if (jas_stream_putc(out, c) == EOF) {
-					return -1;
+					goto error;
 				}
 				nz -= 8;
 				z &= RAS_ONES(nz);
@@ -272,21 +280,30 @@ static int ras_putdatastd(jas_stream_t *out, ras_hdr_t *hdr, jas_image_t *image,
 		if (nz > 0) {
 			c = (z >> (8 - nz)) & RAS_ONES(nz);
 			if (jas_stream_putc(out, c) == EOF) {
-				return -1;
+				goto error;
 			}
 		}
 		if (pad % 2) {
 			if (jas_stream_putc(out, 0) == EOF) {
-				return -1;
+				goto error;
 			}
 		}
 	}
 
 	for (i = 0; i < numcmpts; ++i) {
 		jas_matrix_destroy(data[i]);
+		data[i] = 0;
 	}
 
 	return 0;
+
+error:
+	for (i = 0; i < numcmpts; ++i) {
+		if (data[i]) {
+			jas_matrix_destroy(data[i]);
+		}
+	}
+	return -1;
 }
 
 static int ras_puthdr(jas_stream_t *out, ras_hdr_t *hdr)