LibopenJPEG: Fix some divisions by 0 in pi.c

The undefined shifts in libopenjpeg are sometimes used as divisors. This CL
checks that we are not trying to divide by 0 or mod by 0 in some places in
pi.c.

BUG=chromium:699491

Change-Id: Iaf629112437068d6479dbbb52b339bec6edefed0
Reviewed-on: https://pdfium-review.googlesource.com/2962
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Nicolás Peña <npm@chromium.org>

3 files changed, 162 insertions, 19 deletions

diff --git a/third_party/libopenjpeg20/0029-avoid-division-by-0.patch b/third_party/libopenjpeg20/0029-avoid-division-by-0.patch
new file mode 100644
index 0000000000..4ea2b7ce3c
--- /dev/null
+++ b/third_party/libopenjpeg20/0029-avoid-division-by-0.patch
@@ -0,0 +1,119 @@
+diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
+index 9097e31a0..083674222 100644
+--- a/third_party/libopenjpeg20/pi.c
++++ b/third_party/libopenjpeg20/pi.c
+@@ -355,12 +355,20 @@ if (!pi->tp_on){
+                                        }
+                                        res = &comp->resolutions[pi->resno];
+                                        levelno = comp->numresolutions - 1 - pi->resno;
+-                                       trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
+-                                       try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
+-                                       trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
+-                                       try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
++                                       OPJ_INT32 x_divisor = comp->dx << levelno;
++                                       OPJ_INT32 y_divisor = comp->dy << levelno;
++                                       if (x_divisor == 0 || y_divisor == 0) {
++                                               continue;
++                                       }
++                                       trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
++                                       try0 = opj_int_ceildiv(pi->ty0, y_divisor);
++                                       trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
++                                       try1 = opj_int_ceildiv(pi->ty1, y_divisor);
+                                        rpx = res->pdx + levelno;
+                                        rpy = res->pdy + levelno;
++                                       if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
++                                               continue;
++                                       }
+                                        if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
+                                                continue;       
+                                        }
+@@ -372,9 +380,9 @@ if (!pi->tp_on){
+                                        
+                                        if ((trx0==trx1)||(try0==try1)) continue;
+                                        
+-                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
++                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
+                                                 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
+-                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
++                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
+                                                 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+                                        pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+                                        if (pi->precno >= res->pw * res->ph) {
+@@ -439,12 +447,20 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
+                                        OPJ_INT32 prci, prcj;
+                                        res = &comp->resolutions[pi->resno];
+                                        levelno = comp->numresolutions - 1 - pi->resno;
+-                                       trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
+-                                       try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
+-                                       trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
+-                                       try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
++                                       OPJ_INT32 x_divisor = comp->dx << levelno;
++                                       OPJ_INT32 y_divisor = comp->dy << levelno;
++                                       if (x_divisor == 0 || y_divisor == 0) {
++                                               continue;
++                                       }
++                                       trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
++                                       try0 = opj_int_ceildiv(pi->ty0, y_divisor);
++                                       trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
++                                       try1 = opj_int_ceildiv(pi->ty1, y_divisor);
+                                        rpx = res->pdx + levelno;
+                                        rpy = res->pdy + levelno;
++                                       if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
++                                               continue;
++                                       }
+                                        if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
+                                                continue;       
+                                        }
+@@ -456,9 +472,9 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
+                                        
+                                        if ((trx0==trx1)||(try0==try1)) continue;
+                                        
+-                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
++                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
+                                                 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
+-                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
++                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
+                                                 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+                                        pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+                                        if (pi->precno >= res->pw * res->ph) {
+@@ -521,26 +537,33 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi) {
+                                        OPJ_INT32 prci, prcj;
+                                        res = &comp->resolutions[pi->resno];
+                                        levelno = comp->numresolutions - 1 - pi->resno;
+-                                       trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
+-                                       try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
+-                                       trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
+-                                       try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
++                                       OPJ_INT32 x_divisor = comp->dx << levelno;
++                                       OPJ_INT32 y_divisor = comp->dy << levelno;
++                                       if (x_divisor == 0 || y_divisor == 0) {
++                                               continue;
++                                       }
++                                       trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
++                                       try0 = opj_int_ceildiv(pi->ty0, y_divisor);
++                                       trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
++                                       try1 = opj_int_ceildiv(pi->ty1, y_divisor);
+                                        rpx = res->pdx + levelno;
+                                        rpy = res->pdy + levelno;
++                                       if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
++                                               continue;
++                                       }
+                                        if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
+                                                continue;       
+                                        }
+                                        if (!((pi->x % (OPJ_INT32)(comp->dx << rpx) == 0) || ((pi->x == pi->tx0) && ((trx0 << levelno) % (1 << rpx))))){
+                                                continue;
+                                        }
+-                                       
+                                        if ((res->pw==0)||(res->ph==0)) continue;
+                                        
+                                        if ((trx0==trx1)||(try0==try1)) continue;
+                                        
+-                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
++                                       prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
+                                                 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
+-                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
++                                       prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
+                                                 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
+                                        pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
+                                        if (pi->precno >= res->pw * res->ph) {
diff --git a/third_party/libopenjpeg20/README.pdfium b/third_party/libopenjpeg20/README.pdfium
index 6c2a3c74ba..9ce0c05706 100644
--- a/third_party/libopenjpeg20/README.pdfium
+++ b/third_party/libopenjpeg20/README.pdfium
@@ -38,4 +38,5 @@ Local Modifications:
 0026-use_opj_uint_ceildiv.patch: Remove (OPJ_UINT32)opj_int_ceildiv((OPJ_INT32)a, (OPJ_INT32) b).
 0027-undefined-shift-opj_t1_decode_cblk.patch: upstream fix for a ubsan bug.
 0028-upstream-check-size-in-opj_j2k_read_siz.patch: upstream patch in j2k.c.
+0029-avoid-division-by-0: fix some /0 and %0 in pi.c (caused by bad shifts).
 TODO(thestig): List all the other patches.
diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
index 9097e31a0e..0836742222 100644
--- a/third_party/libopenjpeg20/pi.c
+++ b/third_party/libopenjpeg20/pi.c
@@ -355,12 +355,20 @@ if (!pi->tp_on){
 					}
 					res = &comp->resolutions[pi->resno];
 					levelno = comp->numresolutions - 1 - pi->resno;
-					trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
-					try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
-					trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
-					try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
+					OPJ_INT32 x_divisor = comp->dx << levelno;
+					OPJ_INT32 y_divisor = comp->dy << levelno;
+					if (x_divisor == 0 || y_divisor == 0) {
+						continue;
+					}
+					trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
+					try0 = opj_int_ceildiv(pi->ty0, y_divisor);
+					trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
+					try1 = opj_int_ceildiv(pi->ty1, y_divisor);
 					rpx = res->pdx + levelno;
 					rpy = res->pdy + levelno;
+					if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
+						continue;
+					}
 					if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
 						continue;	
 					}
@@ -372,9 +380,9 @@ if (!pi->tp_on){
 					
 					if ((trx0==trx1)||(try0==try1)) continue;
 					
-					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
+					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
 						 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
-					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
 					if (pi->precno >= res->pw * res->ph) {
@@ -439,12 +447,20 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
 					OPJ_INT32 prci, prcj;
 					res = &comp->resolutions[pi->resno];
 					levelno = comp->numresolutions - 1 - pi->resno;
-					trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
-					try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
-					trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
-					try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
+					OPJ_INT32 x_divisor = comp->dx << levelno;
+					OPJ_INT32 y_divisor = comp->dy << levelno;
+					if (x_divisor == 0 || y_divisor == 0) {
+						continue;
+					}
+					trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
+					try0 = opj_int_ceildiv(pi->ty0, y_divisor);
+					trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
+					try1 = opj_int_ceildiv(pi->ty1, y_divisor);
 					rpx = res->pdx + levelno;
 					rpy = res->pdy + levelno;
+					if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
+						continue;
+					}
 					if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
 						continue;	
 					}
@@ -456,9 +472,9 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi) {
 					
 					if ((trx0==trx1)||(try0==try1)) continue;
 					
-					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
+					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
 						 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
-					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
 					if (pi->precno >= res->pw * res->ph) {
@@ -521,26 +537,33 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi) {
 					OPJ_INT32 prci, prcj;
 					res = &comp->resolutions[pi->resno];
 					levelno = comp->numresolutions - 1 - pi->resno;
-					trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
-					try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
-					trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
-					try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
+					OPJ_INT32 x_divisor = comp->dx << levelno;
+					OPJ_INT32 y_divisor = comp->dy << levelno;
+					if (x_divisor == 0 || y_divisor == 0) {
+						continue;
+					}
+					trx0 = opj_int_ceildiv(pi->tx0, x_divisor);
+					try0 = opj_int_ceildiv(pi->ty0, y_divisor);
+					trx1 = opj_int_ceildiv(pi->tx1, x_divisor);
+					try1 = opj_int_ceildiv(pi->ty1, y_divisor);
 					rpx = res->pdx + levelno;
 					rpy = res->pdy + levelno;
+					if (comp->dy << rpy == 0 || 1 << rpy == 0 || comp->dx << rpx == 0 || 1 << rpx == 0) {
+						continue;
+					}
 					if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) && ((try0 << levelno) % (1 << rpy))))){
 						continue;	
 					}
 					if (!((pi->x % (OPJ_INT32)(comp->dx << rpx) == 0) || ((pi->x == pi->tx0) && ((trx0 << levelno) % (1 << rpx))))){
 						continue;
 					}
-					
 					if ((res->pw==0)||(res->ph==0)) continue;
 					
 					if ((trx0==trx1)||(try0==try1)) continue;
 					
-					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
+					prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x, x_divisor), (OPJ_INT32)res->pdx)
 						 - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
-					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
+					prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y, y_divisor), (OPJ_INT32)res->pdy)
 						 - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
 					pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
 					if (pi->precno >= res->pw * res->ph) {