Prevent skew overflows in gtTileContig

Using int64 to check whether uint32 operations have overflowed.

BUG=681300

Change-Id: I4470d34f2e5e61c0bf96f1c8587cdb7805afe87b
Reviewed-on: https://pdfium-review.googlesource.com/2355
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Nicolás Peña <npm@chromium.org>

3 files changed, 129 insertions, 5 deletions

diff --git a/third_party/libtiff/0017-safe_skews_in_gtTileContig.patch b/third_party/libtiff/0017-safe_skews_in_gtTileContig.patch
new file mode 100644
index 0000000000..10c5077392
--- /dev/null
+++ b/third_party/libtiff/0017-safe_skews_in_gtTileContig.patch
@@ -0,0 +1,88 @@
+diff --git a/third_party/libtiff/tif_getimage.c b/third_party/libtiff/tif_getimage.c
+index 2861cdd1e..5ed1b7a37 100644
+--- a/third_party/libtiff/tif_getimage.c
++++ b/third_party/libtiff/tif_getimage.c
+@@ -31,6 +31,7 @@
+  */
+ #include "tiffiop.h"
+ #include <stdio.h>
++#include <limits.h>
+ 
+ static int gtTileContig(TIFFRGBAImage*, uint32*, uint32, uint32);
+ static int gtTileSeparate(TIFFRGBAImage*, uint32*, uint32, uint32);
+@@ -612,6 +613,7 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+     uint32 tw, th;
+     unsigned char* buf;
+     int32 fromskew, toskew;
++    int64 safeskew;
+     uint32 nrow;
+     int ret = 1, flip;
+     uint32 this_tw, tocol;
+@@ -631,19 +633,37 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+     flip = setorientation(img);
+     if (flip & FLIP_VERTICALLY) {
+            y = h - 1;
+-           toskew = -(int32)(tw + w);
++           safeskew = 0;
++           safeskew -= tw;
++           safeskew -= w;
+     }
+     else {
+            y = 0;
+-           toskew = -(int32)(tw - w);
++           safeskew = 0;
++           safeskew -= tw;
++           safeskew +=w;
+     }
++    if(safeskew > INT_MAX || safeskew < INT_MIN){
++       _TIFFfree(buf);
++       TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
++       return (0);
++    }
++    toskew = safeskew;
++
+      
+     /*
+      * Leftmost tile is clipped on left side if col_offset > 0.
+      */
+     leftmost_fromskew = img->col_offset % tw;
+     leftmost_tw = tw - leftmost_fromskew;
+-    leftmost_toskew = toskew + leftmost_fromskew;
++    safeskew = toskew;
++    safeskew += leftmost_fromskew;
++    if(safeskew > INT_MAX || safeskew < INT_MIN){
++       _TIFFfree(buf);
++       TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
++       return (0);
++    }
++    leftmost_toskew = safeskew;
+     for (row = 0; row < h; row += nrow)
+     {
+         rowstoread = th - (row + img->row_offset) % th;
+@@ -668,9 +688,24 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                /*
+                 * Rightmost tile is clipped on right side.
+                 */
+-               fromskew = tw - (w - tocol);
++               safeskew = tw;
++               safeskew -= w;
++               safeskew += tocol;
++               if(safeskew > INT_MAX || safeskew < INT_MIN){
++                       _TIFFfree(buf);
++                       TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
++                       return (0);
++               }
++               fromskew = safeskew;
+                this_tw = tw - fromskew;
+-               this_toskew = toskew + fromskew;
++               safeskew = toskew;
++               safeskew += fromskew;
++               if(safeskew > INT_MAX || safeskew < INT_MIN){
++                       _TIFFfree(buf);
++                       TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
++                       return (0);
++               }
++               this_toskew = safeskew;
+            }
+            (*put)(img, raster+y*w+tocol, tocol, y, this_tw, nrow, fromskew, this_toskew, buf + pos);
+            tocol += this_tw;
diff --git a/third_party/libtiff/README.pdfium b/third_party/libtiff/README.pdfium
index 23c8450eff..04f728e3f7 100644
--- a/third_party/libtiff/README.pdfium
+++ b/third_party/libtiff/README.pdfium
@@ -26,3 +26,4 @@ Local Modifications:
 0014-cast-to-unsigned-in-putagreytile.patch: casting to avoid undefined shifts.
 0015-fix-leaks-in-tif_ojpeg.patch: fix direct leaks in tif_ojpeg.c methods
 0016-fix-leak-in-pixarlogsetupdecode.patch: Free sp->tbuf if setup fails
+0017-safe_skews_in_gtTileContig.patch: return error if to/from skews overflow from int32.
diff --git a/third_party/libtiff/tif_getimage.c b/third_party/libtiff/tif_getimage.c
index 2861cdd1e2..5ed1b7a370 100644
--- a/third_party/libtiff/tif_getimage.c
+++ b/third_party/libtiff/tif_getimage.c
@@ -31,6 +31,7 @@
  */
 #include "tiffiop.h"
 #include <stdio.h>
+#include <limits.h>
 
 static int gtTileContig(TIFFRGBAImage*, uint32*, uint32, uint32);
 static int gtTileSeparate(TIFFRGBAImage*, uint32*, uint32, uint32);
@@ -612,6 +613,7 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     uint32 tw, th;
     unsigned char* buf;
     int32 fromskew, toskew;
+    int64 safeskew;
     uint32 nrow;
     int ret = 1, flip;
     uint32 this_tw, tocol;
@@ -631,19 +633,37 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     flip = setorientation(img);
     if (flip & FLIP_VERTICALLY) {
 	    y = h - 1;
-	    toskew = -(int32)(tw + w);
+	    safeskew = 0;
+	    safeskew -= tw;
+	    safeskew -= w;
     }
     else {
 	    y = 0;
-	    toskew = -(int32)(tw - w);
+	    safeskew = 0;
+	    safeskew -= tw;
+	    safeskew +=w;
     }
+    if(safeskew > INT_MAX || safeskew < INT_MIN){
+    	_TIFFfree(buf);
+    	TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
+    	return (0);
+    }
+    toskew = safeskew;
+
      
     /*
      *	Leftmost tile is clipped on left side if col_offset > 0.
      */
     leftmost_fromskew = img->col_offset % tw;
     leftmost_tw = tw - leftmost_fromskew;
-    leftmost_toskew = toskew + leftmost_fromskew;
+    safeskew = toskew;
+    safeskew += leftmost_fromskew;
+    if(safeskew > INT_MAX || safeskew < INT_MIN){
+    	_TIFFfree(buf);
+    	TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
+    	return (0);
+    }
+    leftmost_toskew = safeskew;
     for (row = 0; row < h; row += nrow)
     {
         rowstoread = th - (row + img->row_offset) % th;
@@ -668,9 +688,24 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		/*
 		 * Rightmost tile is clipped on right side.
 		 */
-		fromskew = tw - (w - tocol);
+		safeskew = tw;
+		safeskew -= w;
+		safeskew += tocol;
+		if(safeskew > INT_MAX || safeskew < INT_MIN){
+			_TIFFfree(buf);
+			TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
+			return (0);
+		}
+		fromskew = safeskew;
 		this_tw = tw - fromskew;
-		this_toskew = toskew + fromskew;
+		safeskew = toskew;
+		safeskew += fromskew;
+		if(safeskew > INT_MAX || safeskew < INT_MIN){
+			_TIFFfree(buf);
+			TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "Invalid skew");
+			return (0);
+		}
+		this_toskew = safeskew;
 	    }
 	    (*put)(img, raster+y*w+tocol, tocol, y, this_tw, nrow, fromskew, this_toskew, buf + pos);
 	    tocol += this_tw;