Libtiff OOM upstream patch

This CL applies two upstream patches that help with OOM problems:
https://github.com/vadz/libtiff/commit/1077fad562e03d1cad591dd10163dd80ad63ab0e
https://github.com/vadz/libtiff/commit/0a619f1e553e46df8022b889ff44f8a1faa1e48d

These do not yet fix the bug below.

Bug: chromium:718494
Change-Id: If68c20f504b27c07dba2765f8e5ef708c1a54d7e
Reviewed-on: https://pdfium-review.googlesource.com/7731
Reviewed-by: dsinclair <dsinclair@chromium.org>
Commit-Queue: Nicolás Peña <npm@chromium.org>

1 files changed, 74 insertions, 48 deletions

diff --git a/third_party/libtiff/tif_getimage.c b/third_party/libtiff/tif_getimage.c
index 84cc1d1a76..d1f1f45ac6 100644
--- a/third_party/libtiff/tif_getimage.c
+++ b/third_party/libtiff/tif_getimage.c
@@ -627,7 +627,7 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     uint32 col, row, y, rowstoread;
     tmsize_t pos;
     uint32 tw, th;
-    unsigned char* buf;
+    unsigned char* buf = NULL;
     int32 fromskew, toskew;
     int64 safeskew;
     uint32 nrow;
@@ -636,13 +636,14 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     int32 this_toskew, leftmost_toskew;
     int32 leftmost_fromskew;
     uint32 leftmost_tw;
+    tmsize_t bufsize;
 
-    buf = (unsigned char*) _TIFFmalloc(TIFFTileSize(tif));
-    if (buf == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
-		return (0);
+    bufsize = TIFFTileSize(tif);
+    if (bufsize == 0) {
+        TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
+        return (0);
     }
-    _TIFFmemset(buf, 0, TIFFTileSize(tif));
+
     TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
     TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
 
@@ -691,8 +692,9 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	col = img->col_offset;
 	while (tocol < w)
         {
-	    if (TIFFReadTile(tif, buf, col,  
-			     row+img->row_offset, 0, 0)==(tmsize_t)(-1) && img->stoponerr)
+	    if (_TIFFReadTileAndAllocBuffer(tif, (void**) &buf, bufsize, col,
+			     row+img->row_offset, 0, 0)==(tmsize_t)(-1) &&
+                (buf == NULL || img->stoponerr))
             {
                 ret = 0;
                 break;
@@ -772,11 +774,11 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	uint32 col, row, y, rowstoread;
 	tmsize_t pos;
 	uint32 tw, th;
-	unsigned char* buf;
-	unsigned char* p0;
-	unsigned char* p1;
-	unsigned char* p2;
-	unsigned char* pa;
+	unsigned char* buf = NULL;
+	unsigned char* p0 = NULL;
+	unsigned char* p1 = NULL;
+	unsigned char* p2 = NULL;
+	unsigned char* pa = NULL;
 	tmsize_t tilesize;
 	tmsize_t bufsize;
 	int32 fromskew, toskew;
@@ -795,16 +797,7 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
 		return (0);
 	}
-	buf = (unsigned char*) _TIFFmalloc(bufsize);
-	if (buf == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
-		return (0);
-	}
-	_TIFFmemset(buf, 0, bufsize);
-	p0 = buf;
-	p1 = p0 + tilesize;
-	p2 = p1 + tilesize;
-	pa = (alpha?(p2+tilesize):NULL);
+
 	TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
 	TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
 
@@ -824,7 +817,6 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
           case PHOTOMETRIC_MINISBLACK:
           case PHOTOMETRIC_PALETTE:
             colorchannels = 1;
-            p2 = p1 = p0;
             break;
 
           default:
@@ -849,7 +841,30 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		col = img->col_offset;
 		while (tocol < w)
 		{
-			if (TIFFReadTile(tif, p0, col,  
+                        if( buf == NULL )
+                        {
+                            if (_TIFFReadTileAndAllocBuffer(
+                                    tif, (void**) &buf, bufsize, col,
+                                    row+img->row_offset,0,0)==(tmsize_t)(-1)
+                                && (buf == NULL || img->stoponerr))
+                            {
+                                    ret = 0;
+                                    break;
+                            }
+                            p0 = buf;
+                            if( colorchannels == 1 )
+                            {
+                                p2 = p1 = p0;
+                                pa = (alpha?(p0+3*tilesize):NULL);
+                            }
+                            else
+                            {
+                                p1 = p0 + tilesize;
+                                p2 = p1 + tilesize;
+                                pa = (alpha?(p2+tilesize):NULL);
+                            }
+                        }
+			else if (TIFFReadTile(tif, p0, col,
 			    row+img->row_offset,0,0)==(tmsize_t)(-1) && img->stoponerr)
 			{
 				ret = 0;
@@ -940,13 +955,14 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	tileContigRoutine put = img->put.contig;
 	uint32 row, y, nrow, nrowsub, rowstoread;
 	tmsize_t pos;
-	unsigned char* buf;
+	unsigned char* buf = NULL;
 	uint32 rowsperstrip;
 	uint16 subsamplinghor,subsamplingver;
 	uint32 imagewidth = img->width;
 	tmsize_t scanline;
 	int32 fromskew, toskew;
 	int ret = 1, flip;
+	tmsize_t maxstripsize;
 
 	TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
 	if( subsamplingver == 0 ) {
@@ -954,12 +970,7 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		return (0);
 	}
 
-	buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
-	if (buf == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
-		return (0);
-	}
-	_TIFFmemset(buf, 0, TIFFStripSize(tif));
+	maxstripsize = TIFFStripSize(tif);
 
 	flip = setorientation(img);
 	if (flip & FLIP_VERTICALLY) {
@@ -981,11 +992,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		nrowsub = nrow;
 		if ((nrowsub%subsamplingver)!=0)
 			nrowsub+=subsamplingver-nrowsub%subsamplingver;
-		if (TIFFReadEncodedStrip(tif,
+		if (_TIFFReadEncodedStripAndAllocBuffer(tif,
 		    TIFFComputeStrip(tif,row+img->row_offset, 0),
-		    buf,
+		    (void**)(&buf),
+		    maxstripsize,
 		    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
-		    && img->stoponerr)
+		    && (buf == NULL || img->stoponerr))
 		{
 			ret = 0;
 			break;
@@ -1029,8 +1041,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 {
 	TIFF* tif = img->tif;
 	tileSeparateRoutine put = img->put.separate;
-	unsigned char *buf;
-	unsigned char *p0, *p1, *p2, *pa;
+	unsigned char *buf = NULL;
+	unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
 	uint32 row, y, nrow, rowstoread;
 	tmsize_t pos;
 	tmsize_t scanline;
@@ -1049,15 +1061,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
 		return (0);
 	}
-	p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
-	if (buf == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
-		return (0);
-	}
-	_TIFFmemset(buf, 0, bufsize);
-	p1 = p0 + stripsize;
-	p2 = p1 + stripsize;
-	pa = (alpha?(p2+stripsize):NULL);
 
 	flip = setorientation(img);
 	if (flip & FLIP_VERTICALLY) {
@@ -1075,7 +1078,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
           case PHOTOMETRIC_MINISBLACK:
           case PHOTOMETRIC_PALETTE:
             colorchannels = 1;
-            p2 = p1 = p0;
             break;
 
           default:
@@ -1091,7 +1093,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
 		nrow = (row + rowstoread > h ? h - row : rowstoread);
 		offset_row = row + img->row_offset;
-		if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+    if( buf == NULL )
+	  {
+	      if (_TIFFReadEncodedStripAndAllocBuffer(
+	              tif, TIFFComputeStrip(tif, offset_row, 0),
+	              (void**) &buf, bufsize,
+	              ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+	          && (buf == NULL || img->stoponerr))
+	      {
+	              ret = 0;
+	              break;
+	      }
+	      p0 = buf;
+	      if( colorchannels == 1 )
+	      {
+	          p2 = p1 = p0;
+	          pa = (alpha?(p0+3*stripsize):NULL);
+	      }
+	      else
+	      {
+	          p1 = p0 + stripsize;
+	          p2 = p1 + stripsize;
+	          pa = (alpha?(p2+stripsize):NULL);
+	      }
+	  }
+ 		else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
 		    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
 		    && img->stoponerr)
 		{