path: root/third_party/libtiff/0025-upstream-OOM-gtTileContig.patch

diff --git a/third_party/libtiff/tif_getimage.c b/third_party/libtiff/tif_getimage.c
index 53c938a89..03c9a81fb 100644
--- a/third_party/libtiff/tif_getimage.c
+++ b/third_party/libtiff/tif_getimage.c
@@ -627,7 +627,7 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     uint32 col, row, y, rowstoread;
     tmsize_t pos;
     uint32 tw, th;
-    unsigned char* buf;
+    unsigned char* buf = NULL;
     int32 fromskew, toskew;
     int64 safeskew;
     uint32 nrow;
@@ -636,13 +636,14 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
     int32 this_toskew, leftmost_toskew;
     int32 leftmost_fromskew;
     uint32 leftmost_tw;
+    tmsize_t bufsize;
 
-    buf = (unsigned char*) _TIFFmalloc(TIFFTileSize(tif));
-    if (buf == 0) {
-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
-               return (0);
+    bufsize = TIFFTileSize(tif);
+    if (bufsize == 0) {
+        TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
+        return (0);
     }
-    _TIFFmemset(buf, 0, TIFFTileSize(tif));
+
     TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
     TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
 
@@ -691,8 +692,9 @@ gtTileContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
        col = img->col_offset;
        while (tocol < w)
         {
-           if (TIFFReadTile(tif, buf, col,  
-                            row+img->row_offset, 0, 0)==(tmsize_t)(-1) && img->stoponerr)
+           if (_TIFFReadTileAndAllocBuffer(tif, (void**) &buf, bufsize, col,
+                            row+img->row_offset, 0, 0)==(tmsize_t)(-1) &&
+                (buf == NULL || img->stoponerr))
             {
                 ret = 0;
                 break;
@@ -772,11 +774,11 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
        uint32 col, row, y, rowstoread;
        tmsize_t pos;
        uint32 tw, th;
-       unsigned char* buf;
-       unsigned char* p0;
-       unsigned char* p1;
-       unsigned char* p2;
-       unsigned char* pa;
+       unsigned char* buf = NULL;
+       unsigned char* p0 = NULL;
+       unsigned char* p1 = NULL;
+       unsigned char* p2 = NULL;
+       unsigned char* pa = NULL;
        tmsize_t tilesize;
        tmsize_t bufsize;
        int32 fromskew, toskew;
@@ -795,16 +797,7 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
                return (0);
        }
-       buf = (unsigned char*) _TIFFmalloc(bufsize);
-       if (buf == 0) {
-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "%s", "No space for tile buffer");
-               return (0);
-       }
-       _TIFFmemset(buf, 0, bufsize);
-       p0 = buf;
-       p1 = p0 + tilesize;
-       p2 = p1 + tilesize;
-       pa = (alpha?(p2+tilesize):NULL);
+
        TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tw);
        TIFFGetField(tif, TIFFTAG_TILELENGTH, &th);
 
@@ -824,7 +817,6 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
           case PHOTOMETRIC_MINISBLACK:
           case PHOTOMETRIC_PALETTE:
             colorchannels = 1;
-            p2 = p1 = p0;
             break;
 
           default:
@@ -849,7 +841,30 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                col = img->col_offset;
                while (tocol < w)
                {
-                       if (TIFFReadTile(tif, p0, col,  
+                        if( buf == NULL )
+                        {
+                            if (_TIFFReadTileAndAllocBuffer(
+                                    tif, (void**) &buf, bufsize, col,
+                                    row+img->row_offset,0,0)==(tmsize_t)(-1)
+                                && (buf == NULL || img->stoponerr))
+                            {
+                                    ret = 0;
+                                    break;
+                            }
+                            p0 = buf;
+                            if( colorchannels == 1 )
+                            {
+                                p2 = p1 = p0;
+                                pa = (alpha?(p0+3*tilesize):NULL);
+                            }
+                            else
+                            {
+                                p1 = p0 + tilesize;
+                                p2 = p1 + tilesize;
+                                pa = (alpha?(p2+tilesize):NULL);
+                            }
+                        }
+                       else if (TIFFReadTile(tif, p0, col,  
                            row+img->row_offset,0,0)==(tmsize_t)(-1) && img->stoponerr)
                        {
                                ret = 0;
@@ -940,13 +955,14 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
        tileContigRoutine put = img->put.contig;
        uint32 row, y, nrow, nrowsub, rowstoread;
        tmsize_t pos;
-       unsigned char* buf;
+       unsigned char* buf = NULL;
        uint32 rowsperstrip;
        uint16 subsamplinghor,subsamplingver;
        uint32 imagewidth = img->width;
        tmsize_t scanline;
        int32 fromskew, toskew;
        int ret = 1, flip;
+       tmsize_t maxstripsize;
 
        TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
        if( subsamplingver == 0 ) {
@@ -954,12 +970,7 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                return (0);
        }
 
-       buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
-       if (buf == 0) {
-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
-               return (0);
-       }
-       _TIFFmemset(buf, 0, TIFFStripSize(tif));
+       maxstripsize = TIFFStripSize(tif);
 
        flip = setorientation(img);
        if (flip & FLIP_VERTICALLY) {
@@ -981,11 +992,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                nrowsub = nrow;
                if ((nrowsub%subsamplingver)!=0)
                        nrowsub+=subsamplingver-nrowsub%subsamplingver;
-               if (TIFFReadEncodedStrip(tif,
+               if (_TIFFReadEncodedStripAndAllocBuffer(tif,
                    TIFFComputeStrip(tif,row+img->row_offset, 0),
-                   buf,
+                   (void**)(&buf),
+                   maxstripsize,
                    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
-                   && img->stoponerr)
+                   && (buf == NULL || img->stoponerr))
                {
                        ret = 0;
                        break;
@@ -1029,8 +1041,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 {
        TIFF* tif = img->tif;
        tileSeparateRoutine put = img->put.separate;
-       unsigned char *buf;
-       unsigned char *p0, *p1, *p2, *pa;
+       unsigned char *buf = NULL;
+       unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
        uint32 row, y, nrow, rowstoread;
        tmsize_t pos;
        tmsize_t scanline;
@@ -1049,15 +1061,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
                return (0);
        }
-       p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
-       if (buf == 0) {
-               TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
-               return (0);
-       }
-       _TIFFmemset(buf, 0, bufsize);
-       p1 = p0 + stripsize;
-       p2 = p1 + stripsize;
-       pa = (alpha?(p2+stripsize):NULL);
 
        flip = setorientation(img);
        if (flip & FLIP_VERTICALLY) {
@@ -1075,7 +1078,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
           case PHOTOMETRIC_MINISBLACK:
           case PHOTOMETRIC_PALETTE:
             colorchannels = 1;
-            p2 = p1 = p0;
             break;
 
           default:
@@ -1091,7 +1093,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
                nrow = (row + rowstoread > h ? h - row : rowstoread);
                offset_row = row + img->row_offset;
-               if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+                if( buf == NULL )
+                {
+                    if (_TIFFReadEncodedStripAndAllocBuffer(
+                            tif, TIFFComputeStrip(tif, offset_row, 0),
+                            (void**) &buf, bufsize,
+                            ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+                        && (buf == NULL || img->stoponerr))
+                    {
+                            ret = 0;
+                            break;
+                    }
+                    p0 = buf;
+                    if( colorchannels == 1 )
+                    {
+                        p2 = p1 = p0;
+                        pa = (alpha?(p0+3*stripsize):NULL);
+                    }
+                    else
+                    {
+                        p1 = p0 + stripsize;
+                        p2 = p1 + stripsize;
+                        pa = (alpha?(p2+stripsize):NULL);
+                    }
+                }
+               else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
                    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
                    && img->stoponerr)
                {
diff --git a/third_party/libtiff/tif_read.c b/third_party/libtiff/tif_read.c
index cc4f5d2f6..ad0a778c0 100644
--- a/third_party/libtiff/tif_read.c
+++ b/third_party/libtiff/tif_read.c
@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
 }
 
 /*
- * Read a strip of data and decompress the specified
- * amount into the user-supplied buffer.
+ * Calculate the strip size according to the number of
+ * rows in the strip (check for truncated last strip on any
+ * of the separations).
  */
-tmsize_t
-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
 {
        static const char module[] = "TIFFReadEncodedStrip";
        TIFFDirectory *td = &tif->tif_dir;
        uint32 rowsperstrip;
        uint32 stripsperplane;
        uint32 stripinplane;
-       uint16 plane;
        uint32 rows;
        tmsize_t stripsize;
        if (!TIFFCheckRead(tif,0))
@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
                    (unsigned long)td->td_nstrips);
                return((tmsize_t)(-1));
        }
-       /*
-        * Calculate the strip size according to the number of
-        * rows in the strip (check for truncated last strip on any
-        * of the separations).
-        */
+
        rowsperstrip=td->td_rowsperstrip;
        if (rowsperstrip>td->td_imagelength)
                rowsperstrip=td->td_imagelength;
        stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
        stripinplane=(strip%stripsperplane);
-       plane=(uint16)(strip/stripsperplane);
+       if( pplane ) *pplane=(uint16)(strip/stripsperplane);
        rows=td->td_imagelength-stripinplane*rowsperstrip;
        if (rows>rowsperstrip)
                rows=rowsperstrip;
        stripsize=TIFFVStripSize(tif,rows);
        if (stripsize==0)
                return((tmsize_t)(-1));
+    return stripsize;
+}
+
+/*
+ * Read a strip of data and decompress the specified
+ * amount into the user-supplied buffer.
+ */
+tmsize_t
+TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+{
+  static const char module[] = "TIFFReadEncodedStrip";
+  TIFFDirectory *td = &tif->tif_dir;
+  tmsize_t stripsize;
+  uint16 plane;
+
+  stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+  if (stripsize==((tmsize_t)(-1)))
+      return((tmsize_t)(-1));
 
     /* shortcut to avoid an extra memcpy() */
     if( td->td_compression == COMPRESSION_NONE &&
@@ -510,6 +523,50 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
        return(stripsize);
 }
 
+/* Variant of TIFFReadEncodedStrip() that does 
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
+ *   suceeded. This avoid excessive memory allocation in case of truncated
+ *   file.
+ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+                                    void **buf, tmsize_t bufsizetoalloc,
+                                    tmsize_t size_to_read)
+{
+    tmsize_t this_stripsize;
+    uint16 plane;
+
+    if( *buf != NULL )
+    {
+        return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
+    }
+
+    this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+    if (this_stripsize==((tmsize_t)(-1)))
+            return((tmsize_t)(-1));
+
+    if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
+            this_stripsize=size_to_read;
+    if (!TIFFFillStrip(tif,strip))
+            return((tmsize_t)(-1));
+
+    *buf = _TIFFmalloc(bufsizetoalloc);
+    if (*buf == NULL) {
+            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+            return((tmsize_t)(-1));
+    }
+    _TIFFmemset(*buf, 0, bufsizetoalloc);
+
+    if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
+            return((tmsize_t)(-1));
+    (*tif->tif_postdecode)(tif,*buf,this_stripsize);
+    return(this_stripsize);
+
+
+}
+
+
 static tmsize_t
 TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
     const char* module)
@@ -939,6 +996,78 @@ TIFFReadEncodedTile(TIFF* tif, uint32 tile, void* buf, tmsize_t size)
                return ((tmsize_t)(-1));
 }
 
+/* Variant of TIFFReadTile() that does 
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillTile() has
+ *   suceeded. This avoid excessive memory allocation in case of truncated
+ *   file.
+ * * calls regular TIFFReadEncodedTile() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadTileAndAllocBuffer(TIFF* tif,
+                            void **buf, tmsize_t bufsizetoalloc,
+                            uint32 x, uint32 y, uint32 z, uint16 s)
+{
+    if (!TIFFCheckRead(tif, 1) || !TIFFCheckTile(tif, x, y, z, s))
+            return ((tmsize_t)(-1));
+    return (_TIFFReadEncodedTileAndAllocBuffer(tif,
+                                               TIFFComputeTile(tif, x, y, z, s),
+                                               buf, bufsizetoalloc,
+                                               (tmsize_t)(-1)));
+}
+
+/* Variant of TIFFReadEncodedTile() that does 
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillTile() has
+ *   suceeded. This avoid excessive memory allocation in case of truncated
+ *   file.
+ * * calls regular TIFFReadEncodedTile() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadEncodedTileAndAllocBuffer(TIFF* tif, uint32 tile,
+                                    void **buf, tmsize_t bufsizetoalloc,
+                                    tmsize_t size_to_read)
+{
+    static const char module[] = "_TIFFReadEncodedTileAndAllocBuffer";
+    TIFFDirectory *td = &tif->tif_dir;
+    tmsize_t tilesize = tif->tif_tilesize;
+
+    if( *buf != NULL )
+    {
+        return TIFFReadEncodedTile(tif, tile, *buf, size_to_read);
+    }
+
+    if (!TIFFCheckRead(tif, 1))
+            return ((tmsize_t)(-1));
+    if (tile >= td->td_nstrips) {
+            TIFFErrorExt(tif->tif_clientdata, module,
+                "%lu: Tile out of range, max %lu",
+                (unsigned long) tile, (unsigned long) td->td_nstrips);
+            return ((tmsize_t)(-1));
+    }
+
+    if (!TIFFFillTile(tif,tile))
+            return((tmsize_t)(-1));
+
+    *buf = _TIFFmalloc(bufsizetoalloc);
+    if (*buf == NULL) {
+            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif),
+                         "No space for tile buffer");
+            return((tmsize_t)(-1));
+    }
+    _TIFFmemset(*buf, 0, bufsizetoalloc);
+
+    if (size_to_read == (tmsize_t)(-1))
+        size_to_read = tilesize;
+    else if (size_to_read > tilesize)
+        size_to_read = tilesize;
+    if( (*tif->tif_decodetile)(tif,
+        (uint8*) *buf, size_to_read, (uint16)(tile/td->td_stripsperimage))) {
+        (*tif->tif_postdecode)(tif, (uint8*) *buf, size_to_read);
+        return (size_to_read);
+    } else
+        return ((tmsize_t)(-1));
+}
+
+
 static tmsize_t
 TIFFReadRawTile1(TIFF* tif, uint32 tile, void* buf, tmsize_t size, const char* module)
 {
diff --git a/third_party/libtiff/tiffiop.h b/third_party/libtiff/tiffiop.h
index 7e415c750..6fb47de5b 100644
--- a/third_party/libtiff/tiffiop.h
+++ b/third_party/libtiff/tiffiop.h
@@ -364,6 +364,20 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
 extern double _TIFFUInt64ToDouble(uint64);
 extern float _TIFFUInt64ToFloat(uint64);
 
+extern tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+                                    void **buf, tmsize_t bufsizetoalloc,
+                                    tmsize_t size_to_read);
+extern tmsize_t
+_TIFFReadEncodedTileAndAllocBuffer(TIFF* tif, uint32 tile,
+                                    void **buf, tmsize_t bufsizetoalloc,
+                                    tmsize_t size_to_read);
+extern tmsize_t
+_TIFFReadTileAndAllocBuffer(TIFF* tif,
+                            void **buf, tmsize_t bufsizetoalloc,
+                            uint32 x, uint32 y, uint32 z, uint16 s);
+
+
 extern int TIFFInitDumpMode(TIFF*, int);
 #ifdef PACKBITS_SUPPORT
 extern int TIFFInitPackBits(TIFF*, int);