Add safe FX_Alloc2D() macro

This avoids unchecked multiplications when computing a size argument
to malloc(). Such an overflow is very scary, and can result in
exploitable bugs.

Along the way, kill off some return checks, since we know this can't
return NULL.

R=thestig@chromium.org

Review URL: https://codereview.chromium.org/1143663004

2 files changed, 10 insertions, 4 deletions

diff --git a/core/include/fxcrt/fx_basic.h b/core/include/fxcrt/fx_basic.h
index bd53d9e7a4..f258029a11 100644
--- a/core/include/fxcrt/fx_basic.h
+++ b/core/include/fxcrt/fx_basic.h
@@ -1454,10 +1454,7 @@ public:
         while (nCount > 0) {
             FX_INT32 temp_count = FX_MIN(nCount, FX_DATALIST_LENGTH);
             DataList list;
-            list.data = FX_Alloc(FX_BYTE, temp_count * unit);
-            if (!list.data) {
-                break;
-            }
+            list.data = FX_Alloc2D(FX_BYTE, temp_count, unit);
             list.start = nStart;
             list.count = temp_count;
             Append(list);
diff --git a/core/include/fxcrt/fx_memory.h b/core/include/fxcrt/fx_memory.h
index afce91169a..a75ff25818 100644
--- a/core/include/fxcrt/fx_memory.h
+++ b/core/include/fxcrt/fx_memory.h
@@ -41,6 +41,14 @@ inline void* FX_AllocOrDie(size_t num_members, size_t member_size) {
     return nullptr;  // Suppress compiler warning.
 }
 
+inline void* FX_AllocOrDie2D(size_t w, size_t h, size_t member_size) {
+    if (w < std::numeric_limits<size_t>::max() / h) {
+        return FX_AllocOrDie(w * h, member_size);
+    }
+    FX_OutOfMemoryTerminate();  // Never returns.
+    return nullptr;  // Suppress compiler warning.
+}
+
 inline void* FX_ReallocOrDie(void* ptr, size_t num_members, size_t member_size) {
     if (void* result = FX_SafeRealloc(ptr, num_members, member_size)) {
         return result;
@@ -51,6 +59,7 @@ inline void* FX_ReallocOrDie(void* ptr, size_t num_members, size_t member_size)
 
 // Never returns NULL.
 #define FX_Alloc(type, size) (type*)FX_AllocOrDie(size, sizeof(type))
+#define FX_Alloc2D(type, w, h) (type*)FX_AllocOrDie2D(w, h, sizeof(type))
 #define FX_Realloc(type, ptr, size) \
     (type*)FX_ReallocOrDie(ptr, size, sizeof(type))