Libtiff OOM upstream patch

This CL applies two upstream patches that help with OOM problems:
https://github.com/vadz/libtiff/commit/1077fad562e03d1cad591dd10163dd80ad63ab0e
https://github.com/vadz/libtiff/commit/0a619f1e553e46df8022b889ff44f8a1faa1e48d

These do not yet fix the bug below.

Bug: chromium:718494
Change-Id: If68c20f504b27c07dba2765f8e5ef708c1a54d7e
Reviewed-on: https://pdfium-review.googlesource.com/7731
Reviewed-by: dsinclair <dsinclair@chromium.org>
Commit-Queue: Nicolás Peña <npm@chromium.org>

1 files changed, 459 insertions, 0 deletions

diff --git a/third_party/libtiff/0025-upstream-OOM-gtTileContig.patch b/third_party/libtiff/0025-upstream-OOM-gtTileContig.patch
new file mode 100644
index 0000000000..81492303d8
--- /dev/null
+++ b/third_party/libtiff/0025-upstream-OOM-gtTileContig.patch
@@ -0,0 +1,459 @@
+diff --git a/third_party/libtiff/tif_getimage.c b/third_party/libtiff/tif_getimage.c
+index 84cc1d1a7..d1f1f45ac 100644
+--- a/third_party/libtiff/tif_getimage.c
++++ b/third_party/libtiff/tif_getimage.c
+@@ -627,7 +627,7 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+     uint32 col, row, y, rowstoread;
+     tmsize_t pos;
+     uint32 tw, th;
+-    unsigned char* buf;
++    unsigned char* buf = NULL;
+     int32 fromskew, toskew;
+     int64 safeskew;
+     uint32 nrow;
+@@ -636,13 +636,14 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+     int32 this_toskew, leftmost_toskew;
+     int32 leftmost_fromskew;
+     uint32 leftmost_tw;
++    tmsize_t bufsize;
+ 
+-    buf = (unsigned char*) _TIFFmalloc(TIFFTileSize(tif));
+-    if (buf == 0) {
+-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
+-               return (0);
++    bufsize = TIFFTileSize(tif);
++    if (bufsize == 0) {
++        TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
++        return (0);
+     }
+-    _TIFFmemset(buf, 0, TIFFTileSize(tif));
++
+     TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
+     TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
+ 
+@@ -691,8 +692,9 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+        col = img->col_offset;
+        while (tocol < w)
+         {
+-           if (TIFFReadTile(tif, buf, col,  
+-                            row+img->row_offset, 0, 0)==(tmsize_t)(-1) && img->stoponerr)
++           if (_TIFFReadTileAndAllocBuffer(tif, (void**) &buf, bufsize, col,
++                            row+img->row_offset, 0, 0)==(tmsize_t)(-1) &&
++                (buf == NULL || img->stoponerr))
+             {
+                 ret = 0;
+                 break;
+@@ -772,11 +774,11 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+        uint32 col, row, y, rowstoread;
+        tmsize_t pos;
+        uint32 tw, th;
+-       unsigned char* buf;
+-       unsigned char* p0;
+-       unsigned char* p1;
+-       unsigned char* p2;
+-       unsigned char* pa;
++       unsigned char* buf = NULL;
++       unsigned char* p0 = NULL;
++       unsigned char* p1 = NULL;
++       unsigned char* p2 = NULL;
++       unsigned char* pa = NULL;
+        tmsize_t tilesize;
+        tmsize_t bufsize;
+        int32 fromskew, toskew;
+@@ -795,16 +797,7 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
+                return (0);
+        }
+-       buf = (unsigned char*) _TIFFmalloc(bufsize);
+-       if (buf == 0) {
+-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
+-               return (0);
+-       }
+-       _TIFFmemset(buf, 0, bufsize);
+-       p0 = buf;
+-       p1 = p0 + tilesize;
+-       p2 = p1 + tilesize;
+-       pa = (alpha?(p2+tilesize):NULL);
++
+        TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
+        TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
+ 
+@@ -824,7 +817,6 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+           case PHOTOMETRIC_MINISBLACK:
+           case PHOTOMETRIC_PALETTE:
+             colorchannels = 1;
+-            p2 = p1 = p0;
+             break;
+ 
+           default:
+@@ -849,7 +841,30 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                col = img->col_offset;
+                while (tocol < w)
+                {
+-                       if (TIFFReadTile(tif, p0, col,  
++                        if( buf == NULL )
++                        {
++                            if (_TIFFReadTileAndAllocBuffer(
++                                    tif, (void**) &buf, bufsize, col,
++                                    row+img->row_offset,0,0)==(tmsize_t)(-1)
++                                && (buf == NULL || img->stoponerr))
++                            {
++                                    ret = 0;
++                                    break;
++                            }
++                            p0 = buf;
++                            if( colorchannels == 1 )
++                            {
++                                p2 = p1 = p0;
++                                pa = (alpha?(p0+3*tilesize):NULL);
++                            }
++                            else
++                            {
++                                p1 = p0 + tilesize;
++                                p2 = p1 + tilesize;
++                                pa = (alpha?(p2+tilesize):NULL);
++                            }
++                        }
++                       else if (TIFFReadTile(tif, p0, col,
+                            row+img->row_offset,0,0)==(tmsize_t)(-1) && img->stoponerr)
+                        {
+                                ret = 0;
+@@ -940,13 +955,14 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+        tileContigRoutine put = img->put.contig;
+        uint32 row, y, nrow, nrowsub, rowstoread;
+        tmsize_t pos;
+-       unsigned char* buf;
++       unsigned char* buf = NULL;
+        uint32 rowsperstrip;
+        uint16 subsamplinghor,subsamplingver;
+        uint32 imagewidth = img->width;
+        tmsize_t scanline;
+        int32 fromskew, toskew;
+        int ret = 1, flip;
++       tmsize_t maxstripsize;
+ 
+        TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
+        if( subsamplingver == 0 ) {
+@@ -954,12 +970,7 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                return (0);
+        }
+ 
+-       buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
+-       if (buf == 0) {
+-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+-               return (0);
+-       }
+-       _TIFFmemset(buf, 0, TIFFStripSize(tif));
++       maxstripsize = TIFFStripSize(tif);
+ 
+        flip = setorientation(img);
+        if (flip & FLIP_VERTICALLY) {
+@@ -981,11 +992,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                nrowsub = nrow;
+                if ((nrowsub%subsamplingver)!=0)
+                        nrowsub+=subsamplingver-nrowsub%subsamplingver;
+-               if (TIFFReadEncodedStrip(tif,
++               if (_TIFFReadEncodedStripAndAllocBuffer(tif,
+                    TIFFComputeStrip(tif,row+img->row_offset, 0),
+-                   buf,
++                   (void**)(&buf),
++                   maxstripsize,
+                    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
+-                   && img->stoponerr)
++                   && (buf == NULL || img->stoponerr))
+                {
+                        ret = 0;
+                        break;
+@@ -1029,8 +1041,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ {
+        TIFF* tif = img->tif;
+        tileSeparateRoutine put = img->put.separate;
+-       unsigned char *buf;
+-       unsigned char *p0, *p1, *p2, *pa;
++       unsigned char *buf = NULL;
++       unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
+        uint32 row, y, nrow, rowstoread;
+        tmsize_t pos;
+        tmsize_t scanline;
+@@ -1049,15 +1061,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
+                return (0);
+        }
+-       p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
+-       if (buf == 0) {
+-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
+-               return (0);
+-       }
+-       _TIFFmemset(buf, 0, bufsize);
+-       p1 = p0 + stripsize;
+-       p2 = p1 + stripsize;
+-       pa = (alpha?(p2+stripsize):NULL);
+ 
+        flip = setorientation(img);
+        if (flip & FLIP_VERTICALLY) {
+@@ -1075,7 +1078,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+           case PHOTOMETRIC_MINISBLACK:
+           case PHOTOMETRIC_PALETTE:
+             colorchannels = 1;
+-            p2 = p1 = p0;
+             break;
+ 
+           default:
+@@ -1091,7 +1093,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
+                nrow = (row + rowstoread > h ? h - row : rowstoread);
+                offset_row = row + img->row_offset;
+-               if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
++    if( buf == NULL )
++         {
++             if (_TIFFReadEncodedStripAndAllocBuffer(
++                     tif, TIFFComputeStrip(tif, offset_row, 0),
++                     (void**) &buf, bufsize,
++                     ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
++                 && (buf == NULL || img->stoponerr))
++             {
++                     ret = 0;
++                     break;
++             }
++             p0 = buf;
++             if( colorchannels == 1 )
++             {
++                 p2 = p1 = p0;
++                 pa = (alpha?(p0+3*stripsize):NULL);
++             }
++             else
++             {
++                 p1 = p0 + stripsize;
++                 p2 = p1 + stripsize;
++                 pa = (alpha?(p2+stripsize):NULL);
++             }
++         }
++               else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+                    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+                    && img->stoponerr)
+                {
+diff --git a/third_party/libtiff/tif_read.c b/third_party/libtiff/tif_read.c
+index c916ac8ac..12c331b12 100644
+--- a/third_party/libtiff/tif_read.c
++++ b/third_party/libtiff/tif_read.c
+@@ -315,18 +315,16 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
+ }
+ 
+ /*
+- * Read a strip of data and decompress the specified
+- * amount into the user-supplied buffer.
+- */
+-tmsize_t
+-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
++ * Calculate the strip size according to the number of
++ * rows in the strip (check for truncated last strip on any
++ * of the separations). */
++static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
+ {
+        static const char module[] = "TIFFReadEncodedStrip";
+        TIFFDirectory *td = &tif->tif_dir;
+        uint32 rowsperstrip;
+        uint32 stripsperplane;
+        uint32 stripinplane;
+-       uint16 plane;
+        uint32 rows;
+        tmsize_t stripsize;
+        if (!TIFFCheckRead(tif,0))
+@@ -338,23 +336,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+                    (unsigned long)td->td_nstrips);
+                return((tmsize_t)(-1));
+        }
+-       /*
+-        * Calculate the strip size according to the number of
+-        * rows in the strip (check for truncated last strip on any
+-        * of the separations).
+-        */
++
+        rowsperstrip=td->td_rowsperstrip;
+        if (rowsperstrip>td->td_imagelength)
+                rowsperstrip=td->td_imagelength;
+        stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
+        stripinplane=(strip%stripsperplane);
+-       plane=(uint16)(strip/stripsperplane);
++       if( pplane ) *pplane=(uint16)(strip/stripsperplane);
+        rows=td->td_imagelength-stripinplane*rowsperstrip;
+        if (rows>rowsperstrip)
+                rows=rowsperstrip;
+        stripsize=TIFFVStripSize(tif,rows);
+        if (stripsize==0)
+                return((tmsize_t)(-1));
++  return stripsize;
++ }
++
++/*
++ * Read a strip of data and decompress the specified
++ * amount into the user-supplied buffer.
++ */
++tmsize_t
++TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
++{
++  static const char module[] = "TIFFReadEncodedStrip";
++  TIFFDirectory *td = &tif->tif_dir;
++  tmsize_t stripsize;
++  uint16 plane;
++
++  stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
++  if (stripsize==((tmsize_t)(-1)))
++    return((tmsize_t)(-1));
+ 
+     /* shortcut to avoid an extra memcpy() */
+     if( td->td_compression == COMPRESSION_NONE &&
+@@ -383,6 +395,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+        return(stripsize);
+ }
+ 
++/* Variant of TIFFReadEncodedStrip() that does 
++ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
++ *   suceeded. This avoid excessive memory allocation in case of truncated
++ *   file.
++ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
++ */
++tmsize_t
++_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read)
++{
++    tmsize_t this_stripsize;
++    uint16 plane;
++
++    if( *buf != NULL )
++    {
++        return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
++    }
++
++    this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
++    if (this_stripsize==((tmsize_t)(-1)))
++            return((tmsize_t)(-1));
++
++    if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
++            this_stripsize=size_to_read;
++    if (!TIFFFillStrip(tif,strip))
++            return((tmsize_t)(-1));
++
++    *buf = _TIFFmalloc(bufsizetoalloc);
++    if (*buf == NULL) {
++            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
++            return((tmsize_t)(-1));
++    }
++    _TIFFmemset(*buf, 0, bufsizetoalloc);
++
++    if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
++            return((tmsize_t)(-1));
++    (*tif->tif_postdecode)(tif,*buf,this_stripsize);
++    return(this_stripsize);
++
++
++}
++
+ static tmsize_t
+ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
+     const char* module)
+@@ -730,6 +785,78 @@ TIFFReadEncodedTile(TIFF* tif, uint32 tile, void* buf, tmsize_t size)
+                return ((tmsize_t)(-1));
+ }
+ 
++
++/* Variant of TIFFReadTile() that does 
++ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillTile() has
++ *   suceeded. This avoid excessive memory allocation in case of truncated
++ *   file.
++ * * calls regular TIFFReadEncodedTile() if *buf != NULL
++ */
++tmsize_t
++_TIFFReadTileAndAllocBuffer(TIFF* tif,
++                            void **buf, tmsize_t bufsizetoalloc,
++                            uint32 x, uint32 y, uint32 z, uint16 s)
++{
++    if (!TIFFCheckRead(tif, 1) || !TIFFCheckTile(tif, x, y, z, s))
++            return ((tmsize_t)(-1));
++    return (_TIFFReadEncodedTileAndAllocBuffer(tif,
++                                               TIFFComputeTile(tif, x, y, z, s),
++                                               buf, bufsizetoalloc,
++                                               (tmsize_t)(-1)));
++}
++
++/* Variant of TIFFReadEncodedTile() that does 
++ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillTile() has
++ *   suceeded. This avoid excessive memory allocation in case of truncated
++ *   file.
++ * * calls regular TIFFReadEncodedTile() if *buf != NULL
++ */
++tmsize_t
++_TIFFReadEncodedTileAndAllocBuffer(TIFF* tif, uint32 tile,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read)
++{
++    static const char module[] = "_TIFFReadEncodedTileAndAllocBuffer";
++    TIFFDirectory *td = &tif->tif_dir;
++    tmsize_t tilesize = tif->tif_tilesize;
++
++    if( *buf != NULL )
++    {
++        return TIFFReadEncodedTile(tif, tile, *buf, size_to_read);
++    }
++
++    if (!TIFFCheckRead(tif, 1))
++            return ((tmsize_t)(-1));
++    if (tile >= td->td_nstrips) {
++            TIFFErrorExt(tif->tif_clientdata, module,
++                "%lu: Tile out of range, max %lu",
++                (unsigned long) tile, (unsigned long) td->td_nstrips);
++            return ((tmsize_t)(-1));
++    }
++
++    if (!TIFFFillTile(tif,tile))
++            return((tmsize_t)(-1));
++
++    *buf = _TIFFmalloc(bufsizetoalloc);
++    if (*buf == NULL) {
++            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif),
++                         "No space for tile buffer");
++            return((tmsize_t)(-1));
++    }
++    _TIFFmemset(*buf, 0, bufsizetoalloc);
++
++    if (size_to_read == (tmsize_t)(-1))
++        size_to_read = tilesize;
++    else if (size_to_read > tilesize)
++        size_to_read = tilesize;
++    if( (*tif->tif_decodetile)(tif,
++        (uint8*) *buf, size_to_read, (uint16)(tile/td->td_stripsperimage))) {
++        (*tif->tif_postdecode)(tif, (uint8*) *buf, size_to_read);
++        return (size_to_read);
++    } else
++        return ((tmsize_t)(-1));
++}
++
+ static tmsize_t
+ TIFFReadRawTile1(TIFF* tif, uint32 tile, void* buf, tmsize_t size, const char* module)
+ {
+diff --git a/third_party/libtiff/tiffiop.h b/third_party/libtiff/tiffiop.h
+index 1925a6b5e..c42ebef43 100644
+--- a/third_party/libtiff/tiffiop.h
++++ b/third_party/libtiff/tiffiop.h
+@@ -364,6 +364,20 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
+ extern double _TIFFUInt64ToDouble(uint64);
+ extern float _TIFFUInt64ToFloat(uint64);
+ 
++extern tmsize_t
++_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read);
++
++extern tmsize_t
++_TIFFReadEncodedTileAndAllocBuffer(TIFF* tif, uint32 tile,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read);
++extern tmsize_t
++_TIFFReadTileAndAllocBuffer(TIFF* tif,
++                            void **buf, tmsize_t bufsizetoalloc,
++                            uint32 x, uint32 y, uint32 z, uint16 s);
++
+ extern int TIFFInitDumpMode(TIFF*, int);
+ #ifdef PACKBITS_SUPPORT
+ extern int TIFFInitPackBits(TIFF*, int);