Abort on OOM by default in FX_Alloc().

Add a FX_TryAlloc() for those few cases where we might need to continue
in face of OOM.

Remove FX_AllocNL() (the context of its use would suggest that NL
means "No Limit"). This is used for some big allocations, so replace
it with TryAlloc().  Large allocations may be worth trying to continue
from, since there are few and they have a large chance of failing.

R=thestig@chromium.org

Review URL: https://codereview.chromium.org/1128043009

2 files changed, 56 insertions, 6 deletions

diff --git a/core/src/fxcrt/fx_basic_memmgr.cpp b/core/src/fxcrt/fx_basic_memmgr.cpp
index 3b3211c20f..63c609daec 100644
--- a/core/src/fxcrt/fx_basic_memmgr.cpp
+++ b/core/src/fxcrt/fx_basic_memmgr.cpp
@@ -4,10 +4,9 @@
 
 // Original code copyright 2014 Foxit Software Inc. http://www.foxitsoftware.com
 
-#include "../../include/fxcrt/fx_basic.h"
-#ifdef __cplusplus
-extern "C" {
-#endif
+#include <stdlib.h>  // For abort().
+#include "../../include/fxcrt/fx_memory.h"
+
 void*	FXMEM_DefaultAlloc(size_t byte_size, int flags)
 {
     return (void*)malloc(byte_size);
@@ -20,9 +19,13 @@ void	FXMEM_DefaultFree(void* pointer, int flags)
 {
     free(pointer);
 }
-#ifdef __cplusplus
+
+NEVER_INLINE void FX_OutOfMemoryTerminate() {
+    // Termimate cleanly if we can, else crash at a specific address (0xbd).
+    abort();
+    reinterpret_cast<void(*)()>(0xbd)();
 }
-#endif
+
 CFX_GrowOnlyPool::CFX_GrowOnlyPool(size_t trunk_size)
 {
     m_TrunkSize = trunk_size;
diff --git a/core/src/fxcrt/fx_basic_memmgr_unittest.cpp b/core/src/fxcrt/fx_basic_memmgr_unittest.cpp
new file mode 100644
index 0000000000..20fd93d3f8
--- /dev/null
+++ b/core/src/fxcrt/fx_basic_memmgr_unittest.cpp
@@ -0,0 +1,47 @@
+// Copyright 2015 PDFium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <limits>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "../../include/fxcrt/fx_memory.h"
+
+namespace {
+
+const size_t kMaxByteAlloc = std::numeric_limits<size_t>::max();
+const size_t kMaxIntAlloc = kMaxByteAlloc / sizeof(int);
+const size_t kOverflowIntAlloc = kMaxIntAlloc + 100;
+
+}  // namespace
+
+TEST(fxcrt, FX_AllocOOM) {
+    EXPECT_DEATH_IF_SUPPORTED(FX_Alloc(int, kMaxIntAlloc), "");
+    EXPECT_DEATH_IF_SUPPORTED(FX_Alloc(int, kOverflowIntAlloc), "");
+
+    int* ptr = FX_Alloc(int, 1);
+    EXPECT_TRUE(ptr);
+    EXPECT_DEATH_IF_SUPPORTED(FX_Realloc(int, ptr, kMaxIntAlloc), "");
+    EXPECT_DEATH_IF_SUPPORTED(FX_Realloc(int, ptr, kOverflowIntAlloc), "");
+    FX_Free(ptr);
+}
+
+TEST(fxcrt, FX_TryAllocOOM) {
+    EXPECT_FALSE(FX_TryAlloc(int, kMaxIntAlloc));
+    EXPECT_FALSE(FX_TryAlloc(int, kOverflowIntAlloc));
+
+    int* ptr = FX_Alloc(int, 1);
+    EXPECT_TRUE(ptr);
+    EXPECT_FALSE(FX_TryRealloc(int, ptr, kMaxIntAlloc));
+    EXPECT_FALSE(FX_TryRealloc(int, ptr, kOverflowIntAlloc));
+    FX_Free(ptr);
+}
+
+TEST(fxcrt, FXMEM_DefaultOOM) {
+    EXPECT_FALSE(FXMEM_DefaultAlloc(kMaxByteAlloc, 0));
+
+    void* ptr = FXMEM_DefaultAlloc(1, 0);
+    EXPECT_TRUE(ptr);
+    EXPECT_FALSE(FXMEM_DefaultRealloc(ptr, kMaxByteAlloc, 0));
+    FXMEM_DefaultFree(ptr, 0);
+}