Do more checks before big allocs in TIFFReadDirEntryArray

This CL fixes the only caller to TIFFReadDirEntryData with potentially large
size so that we avoid big mallocs when we know we will fail. It does this as
follows:
- Avoid the unnecessary computations if datasize is very small. We don't want
to be slower in this case.
- If !isMapped(tif), we will Seek and Read. Check that ending position is
reachable. In the other case, do a simple check for out of bounds.

Bug: chromium:681311
Change-Id: Ia172d8b4d401753b7c8d5455dc1ada5335f6fa6b
Reviewed-on: https://pdfium-review.googlesource.com/3253
Commit-Queue: Nicolás Peña <npm@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>

1 files changed, 81 insertions, 0 deletions

diff --git a/third_party/libtiff/0019-oom-TIFFReadDirEntryArray.patch b/third_party/libtiff/0019-oom-TIFFReadDirEntryArray.patch
new file mode 100644
index 0000000000..1144e06ef4
--- /dev/null
+++ b/third_party/libtiff/0019-oom-TIFFReadDirEntryArray.patch
@@ -0,0 +1,81 @@
+diff --git a/third_party/libtiff/tif_dirread.c b/third_party/libtiff/tif_dirread.c
+index 7dd944483..d50b39a80 100644
+--- a/third_party/libtiff/tif_dirread.c
++++ b/third_party/libtiff/tif_dirread.c
+@@ -790,44 +790,43 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArray(TIFF* tif, TIFFDirEntry* d
+        *count=(uint32)direntry->tdir_count;
+        datasize=(*count)*typesize;
+        assert((tmsize_t)datasize>0);
+-       data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
+-       if (data==0)
+-               return(TIFFReadDirEntryErrAlloc);
++       const uint32 small_alloc_threshold=(tif->tif_flags&TIFF_BIGTIFF)? 8 : 4;
++       if (datasize <= small_alloc_threshold)
++       {
++               data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
++               if (data==0)
++                       return(TIFFReadDirEntryErrAlloc);
++               _TIFFmemcpy(data,&direntry->tdir_offset,datasize);
++               *value=data;
++               return(TIFFReadDirEntryErrOk);
++       }
++       uint64 offset;
+        if (!(tif->tif_flags&TIFF_BIGTIFF))
+        {
+-               if (datasize<=4)
+-                       _TIFFmemcpy(data,&direntry->tdir_offset,datasize);
+-               else
+-               {
+-                       enum TIFFReadDirEntryErr err;
+-                       uint32 offset = direntry->tdir_offset.toff_long;
+-                       if (tif->tif_flags&TIFF_SWAB)
+-                               TIFFSwabLong(&offset);
+-                       err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
+-                       if (err!=TIFFReadDirEntryErrOk)
+-                       {
+-                               _TIFFfree(data);
+-                               return(err);
+-                       }
+-               }
++               uint32 small_offset=direntry->tdir_offset.toff_long;
++               if (tif->tif_flags&TIFF_SWAB)
++                       TIFFSwabLong(&small_offset);
++               offset=(uint64)small_offset;
+        }
+        else
+        {
+-               if (datasize<=8)
+-                       _TIFFmemcpy(data,&direntry->tdir_offset,datasize);
+-               else
+-               {
+-                       enum TIFFReadDirEntryErr err;
+-                       uint64 offset = direntry->tdir_offset.toff_long8;
+-                       if (tif->tif_flags&TIFF_SWAB)
+-                               TIFFSwabLong8(&offset);
+-                       err=TIFFReadDirEntryData(tif,offset,(tmsize_t)datasize,data);
+-                       if (err!=TIFFReadDirEntryErrOk)
+-                       {
+-                               _TIFFfree(data);
+-                               return(err);
+-                       }
+-               }
++               offset = direntry->tdir_offset.toff_long8;
++               if (tif->tif_flags&TIFF_SWAB)
++                       TIFFSwabLong8(&offset);
++       }
++       if ((uint64)(-1) - offset < datasize)
++               return(TIFFReadDirEntryErrIo);
++       const uint64 size=isMapped(tif)? (uint64)tif->tif_size : TIFFGetFileSize(tif);
++       if (offset + datasize > size)
++               return(TIFFReadDirEntryErrIo);
++       data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
++       if (data==0)
++               return(TIFFReadDirEntryErrAlloc);
++       enum TIFFReadDirEntryErr err=TIFFReadDirEntryData(tif,offset,(tmsize_t)datasize,data);
++       if (err!=TIFFReadDirEntryErrOk)
++       {
++               _TIFFfree(data);
++               return(err);
+        }
+        *value=data;
+        return(TIFFReadDirEntryErrOk);