Add safe FX_Alloc2D() macro

This avoids unchecked multiplications when computing a size argument
to malloc(). Such an overflow is very scary, and can result in
exploitable bugs.

Along the way, kill off some return checks, since we know this can't
return NULL.

R=thestig@chromium.org

Review URL: https://codereview.chromium.org/1143663004

3 files changed, 5 insertions, 15 deletions

diff --git a/core/src/fxcodec/codec/fx_codec_fax.cpp b/core/src/fxcodec/codec/fx_codec_fax.cpp
index 667b713df8..33e89e4f92 100644
--- a/core/src/fxcodec/codec/fx_codec_fax.cpp
+++ b/core/src/fxcodec/codec/fx_codec_fax.cpp
@@ -949,10 +949,7 @@ CCodec_FaxEncoder::CCodec_FaxEncoder(FX_LPCBYTE src_buf, int width, int height,
         return;
     }
     FXSYS_memset8(m_pRefLine, 0xff, m_Pitch);
-    m_pLineBuf = FX_Alloc(FX_BYTE, m_Pitch * 8);
-    if (m_pLineBuf == NULL) {
-        return;
-    }
+    m_pLineBuf = FX_Alloc2D(FX_BYTE, m_Pitch, 8);
     m_DestBuf.EstimateSize(0, 10240);
 }
 CCodec_FaxEncoder::~CCodec_FaxEncoder()
diff --git a/core/src/fxcodec/codec/fx_codec_flate.cpp b/core/src/fxcodec/codec/fx_codec_flate.cpp
index bbee167f3d..4d43cc554b 100644
--- a/core/src/fxcodec/codec/fx_codec_flate.cpp
+++ b/core/src/fxcodec/codec/fx_codec_flate.cpp
@@ -13,7 +13,7 @@ extern "C"
 {
     static void* my_alloc_func (void* opaque, unsigned int items, unsigned int size)
     {
-        return FX_Alloc(FX_BYTE, items * size);
+        return FX_Alloc2D(FX_BYTE, items, size);
     }
     static void   my_free_func  (void* opaque, void* address)
     {
@@ -241,9 +241,7 @@ static FX_BOOL PNG_PredictorEncode(FX_LPBYTE& data_buf, FX_DWORD& data_size,
         return FALSE;
     const int row_count = (data_size + row_size - 1) / row_size;
     const int last_row_size = data_size % row_size;
-    FX_LPBYTE dest_buf = FX_Alloc( FX_BYTE, (row_size + 1) * row_count);
-    if (dest_buf == NULL)
-        return FALSE;
+    FX_LPBYTE dest_buf = FX_Alloc2D(FX_BYTE, row_size + 1, row_count);
     int byte_cnt = 0;
     FX_LPBYTE pSrcData = data_buf;
     FX_LPBYTE pDestData = dest_buf;
@@ -397,9 +395,7 @@ static FX_BOOL PNG_Predictor(FX_LPBYTE& data_buf, FX_DWORD& data_size,
         return FALSE;
     const int row_count = (data_size + row_size) / (row_size + 1);
     const int last_row_size = data_size % (row_size + 1);
-    FX_LPBYTE dest_buf = FX_Alloc( FX_BYTE, row_size * row_count);
-    if (dest_buf == NULL)
-        return FALSE;
+    FX_LPBYTE dest_buf = FX_Alloc2D(FX_BYTE, row_size, row_count);
     int byte_cnt = 0;
     FX_LPBYTE pSrcData = data_buf;
     FX_LPBYTE pDestData = dest_buf;
diff --git a/core/src/fxcodec/codec/fx_codec_jpeg.cpp b/core/src/fxcodec/codec/fx_codec_jpeg.cpp
index d4c9926254..60575d4e34 100644
--- a/core/src/fxcodec/codec/fx_codec_jpeg.cpp
+++ b/core/src/fxcodec/codec/fx_codec_jpeg.cpp
@@ -153,10 +153,7 @@ static void _JpegEncode(const CFX_DIBSource* pSource, FX_LPBYTE& dest_buf, FX_ST
     }
     FX_LPBYTE line_buf = NULL;
     if (nComponents > 1) {
-        line_buf = FX_Alloc(FX_BYTE, width * nComponents);
-        if (line_buf == NULL) {
-            return;
-        }
+        line_buf = FX_Alloc2D(FX_BYTE, width, nComponents);
     }
     jpeg_set_defaults(&cinfo);
     if(quality != 75) {