openjpeg: Prevent an integer overflow in opj_jp2_apply_pclr.

This patch also prevent a null pointer access problem.

BUG=chromium:638829
R=ochang@chromium.org

Review-Url: https://codereview.chromium.org/2270343002

3 files changed, 73 insertions, 5 deletions

diff --git a/third_party/libopenjpeg20/0022-jp2_apply_pclr_overflow.patch b/third_party/libopenjpeg20/0022-jp2_apply_pclr_overflow.patch
new file mode 100644
index 0000000000..72105fec4f
--- /dev/null
+++ b/third_party/libopenjpeg20/0022-jp2_apply_pclr_overflow.patch
@@ -0,0 +1,53 @@
+diff --git a/third_party/libopenjpeg20/jp2.c b/third_party/libopenjpeg20/jp2.c
+index a6648f6..8128d98 100644
+--- a/third_party/libopenjpeg20/jp2.c
++++ b/third_party/libopenjpeg20/jp2.c
+@@ -972,6 +972,14 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
+ 	nr_channels = color->jp2_pclr->nr_channels;
+ 
+ 	old_comps = image->comps;
++	/* Overflow check: prevent integer overflow */
++	for (i = 0; i < nr_channels; ++i) {
++		cmp = cmap[i].cmp;
++		if (old_comps[cmp].h == 0 || old_comps[cmp].w > ((OPJ_UINT32)-1) / sizeof(OPJ_INT32) / old_comps[cmp].h) {
++			return;
++		}
++	}
++
+ 	new_comps = (opj_image_comp_t*)
+ 			opj_malloc(nr_channels * sizeof(opj_image_comp_t));
+ 	if (!new_comps) {
+@@ -1011,22 +1019,28 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
+ 		/* Palette mapping: */
+ 		cmp = cmap[i].cmp; pcol = cmap[i].pcol;
+ 		src = old_comps[cmp].data;
+-    assert( src );
++		dst = new_comps[i].data;
+ 		max = new_comps[i].w * new_comps[i].h;
+ 
++		/* Prevent null pointer access */
++		if (!src || !dst) {
++			for (j = 0; j < nr_channels; ++j) {
++				opj_free(new_comps[j].data);
++			}
++			opj_free(new_comps);
++			new_comps = NULL;
++			return;
++		}
++
+ 		/* Direct use: */
+     if(cmap[i].mtyp == 0) {
+       assert( cmp == 0 ); // probably wrong.
+-      dst = new_comps[i].data;
+-      assert( dst );
+       for(j = 0; j < max; ++j) {
+         dst[j] = src[j];
+       }
+     }
+     else {
+       assert( i == pcol ); // probably wrong?
+-      dst = new_comps[i].data;
+-      assert( dst );
+       for(j = 0; j < max; ++j) {
+         /* The index */
+         if((k = src[j]) < 0) k = 0; else if(k > top_k) k = top_k;
diff --git a/third_party/libopenjpeg20/README.pdfium b/third_party/libopenjpeg20/README.pdfium
index 7779044799..2c8d93c1d0 100644
--- a/third_party/libopenjpeg20/README.pdfium
+++ b/third_party/libopenjpeg20/README.pdfium
@@ -31,4 +31,5 @@ Local Modifications:
 0019-tcd_init_tile.patch: Prevent integer overflows during calculation of |l_nb_code_blocks_size|.
 0020-opj_aligned_malloc.patch: Prevent overflows when using opj_aligned_malloc().
 0021-tcd_init_tile_negative.patch: Prevent negative x, y values in opj_tcd_init_tile.
+0022-jp2_apply_pclr_overflow.patch: Prevent integer overflow in opj_jp2_apply_pclr.
 TODO(thestig): List all the other patches.
diff --git a/third_party/libopenjpeg20/jp2.c b/third_party/libopenjpeg20/jp2.c
index a6648f637d..8128d98e8f 100644
--- a/third_party/libopenjpeg20/jp2.c
+++ b/third_party/libopenjpeg20/jp2.c
@@ -972,6 +972,14 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
 	nr_channels = color->jp2_pclr->nr_channels;
 
 	old_comps = image->comps;
+	/* Overflow check: prevent integer overflow */
+	for (i = 0; i < nr_channels; ++i) {
+		cmp = cmap[i].cmp;
+		if (old_comps[cmp].h == 0 || old_comps[cmp].w > ((OPJ_UINT32)-1) / sizeof(OPJ_INT32) / old_comps[cmp].h) {
+			return;
+		}
+	}
+
 	new_comps = (opj_image_comp_t*)
 			opj_malloc(nr_channels * sizeof(opj_image_comp_t));
 	if (!new_comps) {
@@ -1011,22 +1019,28 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
 		/* Palette mapping: */
 		cmp = cmap[i].cmp; pcol = cmap[i].pcol;
 		src = old_comps[cmp].data;
-    assert( src );
+		dst = new_comps[i].data;
 		max = new_comps[i].w * new_comps[i].h;
 
+		/* Prevent null pointer access */
+		if (!src || !dst) {
+			for (j = 0; j < nr_channels; ++j) {
+				opj_free(new_comps[j].data);
+			}
+			opj_free(new_comps);
+			new_comps = NULL;
+			return;
+		}
+
 		/* Direct use: */
     if(cmap[i].mtyp == 0) {
       assert( cmp == 0 ); // probably wrong.
-      dst = new_comps[i].data;
-      assert( dst );
       for(j = 0; j < max; ++j) {
         dst[j] = src[j];
       }
     }
     else {
       assert( i == pcol ); // probably wrong?
-      dst = new_comps[i].data;
-      assert( dst );
       for(j = 0; j < max; ++j) {
         /* The index */
         if((k = src[j]) < 0) k = 0; else if(k > top_k) k = top_k;