Abort on OOM by default in FX_Alloc().

Add a FX_TryAlloc() for those few cases where we might need to continue
in face of OOM.

Remove FX_AllocNL() (the context of its use would suggest that NL
means "No Limit"). This is used for some big allocations, so replace
it with TryAlloc().  Large allocations may be worth trying to continue
from, since there are few and they have a large chance of failing.

R=thestig@chromium.org

Review URL: https://codereview.chromium.org/1128043009

2 files changed, 58 insertions, 11 deletions

diff --git a/core/include/fxcrt/fx_memory.h b/core/include/fxcrt/fx_memory.h
index 25d82f69b7..afce91169a 100644
--- a/core/include/fxcrt/fx_memory.h
+++ b/core/include/fxcrt/fx_memory.h
@@ -10,20 +10,57 @@
 #include "fx_system.h"
 
 #ifdef __cplusplus
-#include <new>
 extern "C" {
 #endif
-#define FX_Alloc(type, size)						(type*)calloc(size, sizeof(type))
-#define FX_Realloc(type, ptr, size)					(type*)realloc(ptr, sizeof(type) * (size))
-#define FX_AllocNL(type, size)						FX_Alloc(type, size)
-#define FX_ReallocNL(type, ptr, size)				FX_Realloc(type, ptr, size)
-#define FX_Free(ptr)								free(ptr)
+// For external C libraries to malloc through PDFium. These may return NULL.
 void*	FXMEM_DefaultAlloc(size_t byte_size, int flags);
 void*	FXMEM_DefaultRealloc(void* pointer, size_t new_size, int flags);
 void	FXMEM_DefaultFree(void* pointer, int flags);
 #ifdef __cplusplus
+}  // extern "C"
+
+#include <stdlib.h>
+#include <limits>
+#include <new>
+
+NEVER_INLINE void FX_OutOfMemoryTerminate();
+
+inline void* FX_SafeRealloc(void* ptr, size_t num_members, size_t member_size) {
+    if (num_members < std::numeric_limits<size_t>::max() / member_size) {
+        return realloc(ptr, num_members * member_size);
+    }
+    return nullptr;
+}
+
+inline void* FX_AllocOrDie(size_t num_members, size_t member_size) {
+    // TODO(tsepez): See if we can avoid the implicit memset(0).
+    if (void* result = calloc(num_members, member_size)) {
+        return result;
+    }
+    FX_OutOfMemoryTerminate();  // Never returns.
+    return nullptr;  // Suppress compiler warning.
 }
 
+inline void* FX_ReallocOrDie(void* ptr, size_t num_members, size_t member_size) {
+    if (void* result = FX_SafeRealloc(ptr, num_members, member_size)) {
+        return result;
+    }
+    FX_OutOfMemoryTerminate();  // Never returns.
+    return nullptr;  // Suppress compiler warning.
+}
+
+// Never returns NULL.
+#define FX_Alloc(type, size) (type*)FX_AllocOrDie(size, sizeof(type))
+#define FX_Realloc(type, ptr, size) \
+    (type*)FX_ReallocOrDie(ptr, size, sizeof(type))
+
+// May return NULL.
+#define FX_TryAlloc(type, size) (type*)calloc(size, sizeof(type))
+#define FX_TryRealloc(type, ptr, size) \
+    (type*)FX_SafeRealloc(ptr, size, sizeof(type))
+
+#define FX_Free(ptr) free(ptr)
+
 class CFX_DestructObject 
 {
 public:
@@ -69,5 +106,5 @@ private:
 
     void*	m_pFirstTrunk;
 };
-#endif
-#endif
+#endif  // __cplusplust
+#endif  // _FX_MEMORY_H_
diff --git a/core/include/fxcrt/fx_system.h b/core/include/fxcrt/fx_system.h
index 9cc165f7a5..96030ca551 100644
--- a/core/include/fxcrt/fx_system.h
+++ b/core/include/fxcrt/fx_system.h
@@ -6,10 +6,12 @@
 
 #ifndef _FX_SYSTEM_H_
 #define _FX_SYSTEM_H_
+
 #define _FX_WIN32_DESKTOP_		1
 #define _FX_LINUX_DESKTOP_		4
 #define _FX_MACOSX_				7
 #define _FX_ANDROID_			12
+
 #define _FXM_PLATFORM_WINDOWS_		1
 #define _FXM_PLATFORM_LINUX_		2
 #define _FXM_PLATFORM_APPLE_		3
@@ -341,12 +343,20 @@ int			FXSYS_round(FX_FLOAT f);
 #define PRIuS "zu"
 #endif
 
-#else  // _FXM_PLATFORM_ == _FXM_PLATFORM_WINDOWS_
+#else  // _FXM_PLATFORM_ != _FXM_PLATFORM_WINDOWS_
 
 #if !defined(PRIuS)
 #define PRIuS "Iu"
 #endif
 
-#endif
+#endif  // _FXM_PLATFORM_ != _FXM_PLATFORM_WINDOWS_
 
-#endif
+// Prevent a function from ever being inlined, typically because we'd
+// like it to appear in stack traces.
+#if  _FXM_PLATFORM_ == _FXM_PLATFORM_WINDOWS_
+#define NEVER_INLINE __declspec(noinline)
+#else  // _FXM_PLATFORM_ == _FXM_PLATFORM_WINDOWS_
+#define NEVER_INLINE __attribute__((__noinline__))
+#endif  // _FXM_PLATFORM_ == _FXM_PLATFORM_WINDOWS_
+
+#endif  // _FX_SYSTEM_H_