openjpeg: Fix potential bad precno value in opj_pi_next* functions.

R=thestig@chromium.org
BUG=571479

Review URL: https://codereview.chromium.org/1585243003 .

3 files changed, 44 insertions, 0 deletions

diff --git a/third_party/libopenjpeg20/0009-opj_pi_next.patch b/third_party/libopenjpeg20/0009-opj_pi_next.patch
new file mode 100644
index 0000000000..a7701f0d05
--- /dev/null
+++ b/third_party/libopenjpeg20/0009-opj_pi_next.patch
@@ -0,0 +1,34 @@
+diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
+index 06f1e41..462e07c 100644
+--- a/third_party/libopenjpeg20/pi.c
++++ b/third_party/libopenjpeg20/pi.c
+@@ -377,6 +377,9 @@ if (!pi->tp_on){
+ 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+ 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+ 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
++					if (pi->precno >= res->pw * res->ph) {
++						return OPJ_FALSE;
++					}
+ 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
+ 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
+ 						if (!pi->include[index]) {
+@@ -458,6 +461,9 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
+ 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+ 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+ 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
++					if (pi->precno >= res->pw * res->ph) {
++						return OPJ_FALSE;
++					}
+ 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
+ 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
+ 						if (!pi->include[index]) {
+@@ -537,6 +543,9 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi) {
+ 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+ 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+ 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
++					if (pi->precno >= res->pw * res->ph) {
++						return OPJ_FALSE;
++					}
+ 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
+ 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
+ 						if (!pi->include[index]) {
diff --git a/third_party/libopenjpeg20/README.pdfium b/third_party/libopenjpeg20/README.pdfium
index cbe052aaac..b5c93f8fe3 100644
--- a/third_party/libopenjpeg20/README.pdfium
+++ b/third_party/libopenjpeg20/README.pdfium
@@ -18,4 +18,5 @@ Local Modifications:
 0006-tcd_init_tile.patch: Fix a dividing zero bug in opj_tcd_init_tile().
 0007-jp2_read_cmap.patch: Fix wrong rendering on greyscale images with index colorspace.
 0008-jp2_check_color.patch: Replace an assertion with returning false.
+0009-opj_pi_next.patch: Fix potential bad precno value in opj_pi_next* functions.
 TODO(thestig): List all the other patches.
diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
index 06f1e41157..462e07c836 100644
--- a/third_party/libopenjpeg20/pi.c
+++ b/third_party/libopenjpeg20/pi.c
@@ -377,6 +377,9 @@ if (!pi->tp_on){
 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+					if (pi->precno >= res->pw * res->ph) {
+						return OPJ_FALSE;
+					}
 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
 						if (!pi->include[index]) {
@@ -458,6 +461,9 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+					if (pi->precno >= res->pw * res->ph) {
+						return OPJ_FALSE;
+					}
 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
 						if (!pi->include[index]) {
@@ -537,6 +543,9 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi) {
 					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+					if (pi->precno >= res->pw * res->ph) {
+						return OPJ_FALSE;
+					}
 					for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
 						index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno * pi->step_c + pi->precno * pi->step_p;
 						if (!pi->include[index]) {