Avoid stack overflow when loading CPDF_Function.chromium/3395

CPDF_StitchFuncs that reference each other create a Load() loop.
Maintaining a set of the visited CPDF_Objects during a Load()
call tree prevents that.

Bug: chromium:830221
Change-Id: I6f494da16c6d79f05870ff85cff38ff8fe69ecfe
Reviewed-on: https://pdfium-review.googlesource.com/30050
Commit-Queue: dsinclair <dsinclair@chromium.org>
Reviewed-by: dsinclair <dsinclair@chromium.org>

10 files changed, 38 insertions, 14 deletions

diff --git a/core/fpdfapi/page/cpdf_expintfunc.cpp b/core/fpdfapi/page/cpdf_expintfunc.cpp
index 5d1213131b..f26df1b01e 100644
--- a/core/fpdfapi/page/cpdf_expintfunc.cpp
+++ b/core/fpdfapi/page/cpdf_expintfunc.cpp
@@ -20,7 +20,8 @@ CPDF_ExpIntFunc::~CPDF_ExpIntFunc() {
   FX_Free(m_pEndValues);
 }
 
-bool CPDF_ExpIntFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_ExpIntFunc::v_Init(CPDF_Object* pObj,
+                             std::set<CPDF_Object*>* pVisited) {
   CPDF_Dictionary* pDict = pObj->GetDict();
   if (!pDict)
     return false;
diff --git a/core/fpdfapi/page/cpdf_expintfunc.h b/core/fpdfapi/page/cpdf_expintfunc.h
index 867d2fa265..f46d6c2f0a 100644
--- a/core/fpdfapi/page/cpdf_expintfunc.h
+++ b/core/fpdfapi/page/cpdf_expintfunc.h
@@ -7,6 +7,8 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_EXPINTFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_EXPINTFUNC_H_
 
+#include <set>
+
 #include "core/fpdfapi/page/cpdf_function.h"
 
 class CPDF_ExpIntFunc : public CPDF_Function {
@@ -15,7 +17,7 @@ class CPDF_ExpIntFunc : public CPDF_Function {
   ~CPDF_ExpIntFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   uint32_t m_nOrigOutputs;
diff --git a/core/fpdfapi/page/cpdf_function.cpp b/core/fpdfapi/page/cpdf_function.cpp
index 635c53a95f..3e99eb8176 100644
--- a/core/fpdfapi/page/cpdf_function.cpp
+++ b/core/fpdfapi/page/cpdf_function.cpp
@@ -17,10 +17,22 @@
 
 // static
 std::unique_ptr<CPDF_Function> CPDF_Function::Load(CPDF_Object* pFuncObj) {
+  std::set<CPDF_Object*> visited;
+  return Load(pFuncObj, &visited);
+}
+
+// static
+std::unique_ptr<CPDF_Function> CPDF_Function::Load(
+    CPDF_Object* pFuncObj,
+    std::set<CPDF_Object*>* pVisited) {
   std::unique_ptr<CPDF_Function> pFunc;
   if (!pFuncObj)
     return pFunc;
 
+  if (pdfium::ContainsKey(*pVisited, pFuncObj))
+    return nullptr;
+  pdfium::ScopedSetInsertion<CPDF_Object*> insertion(pVisited, pFuncObj);
+
   int iType = -1;
   if (CPDF_Stream* pStream = pFuncObj->AsStream())
     iType = pStream->GetDict()->GetIntegerFor("FunctionType");
@@ -37,7 +49,7 @@ std::unique_ptr<CPDF_Function> CPDF_Function::Load(CPDF_Object* pFuncObj) {
   else if (type == Type::kType4PostScript)
     pFunc = pdfium::MakeUnique<CPDF_PSFunc>();
 
-  if (!pFunc || !pFunc->Init(pFuncObj))
+  if (!pFunc || !pFunc->Init(pFuncObj, pVisited))
     return nullptr;
 
   return pFunc;
@@ -64,7 +76,7 @@ CPDF_Function::~CPDF_Function() {
   FX_Free(m_pRanges);
 }
 
-bool CPDF_Function::Init(CPDF_Object* pObj) {
+bool CPDF_Function::Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) {
   CPDF_Stream* pStream = pObj->AsStream();
   CPDF_Dictionary* pDict = pStream ? pStream->GetDict() : pObj->AsDictionary();
 
@@ -89,7 +101,7 @@ bool CPDF_Function::Init(CPDF_Object* pObj) {
       m_pRanges[i] = pRanges->GetFloatAt(i);
   }
   uint32_t old_outputs = m_nOutputs;
-  if (!v_Init(pObj))
+  if (!v_Init(pObj, pVisited))
     return false;
   if (m_pRanges && m_nOutputs > old_outputs) {
     m_pRanges = FX_Realloc(float, m_pRanges, m_nOutputs * 2);
diff --git a/core/fpdfapi/page/cpdf_function.h b/core/fpdfapi/page/cpdf_function.h
index ff7cf847d2..7966194ade 100644
--- a/core/fpdfapi/page/cpdf_function.h
+++ b/core/fpdfapi/page/cpdf_function.h
@@ -8,6 +8,7 @@
 #define CORE_FPDFAPI_PAGE_CPDF_FUNCTION_H_
 
 #include <memory>
+#include <set>
 
 class CPDF_ExpIntFunc;
 class CPDF_Object;
@@ -50,8 +51,10 @@ class CPDF_Function {
  protected:
   explicit CPDF_Function(Type type);
 
-  bool Init(CPDF_Object* pObj);
-  virtual bool v_Init(CPDF_Object* pObj) = 0;
+  static std::unique_ptr<CPDF_Function> Load(CPDF_Object* pFuncObj,
+                                             std::set<CPDF_Object*>* pVisited);
+  bool Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited);
+  virtual bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) = 0;
   virtual bool v_Call(float* inputs, float* results) const = 0;
 
   uint32_t m_nInputs;
diff --git a/core/fpdfapi/page/cpdf_psfunc.cpp b/core/fpdfapi/page/cpdf_psfunc.cpp
index 5499a47f99..6df7dff20c 100644
--- a/core/fpdfapi/page/cpdf_psfunc.cpp
+++ b/core/fpdfapi/page/cpdf_psfunc.cpp
@@ -13,7 +13,7 @@ CPDF_PSFunc::CPDF_PSFunc() : CPDF_Function(Type::kType4PostScript) {}
 
 CPDF_PSFunc::~CPDF_PSFunc() {}
 
-bool CPDF_PSFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_PSFunc::v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) {
   auto pAcc = pdfium::MakeRetain<CPDF_StreamAcc>(pObj->AsStream());
   pAcc->LoadAllDataFiltered();
   return m_PS.Parse(pAcc->GetSpan());
diff --git a/core/fpdfapi/page/cpdf_psfunc.h b/core/fpdfapi/page/cpdf_psfunc.h
index b8c18c9472..7212fea392 100644
--- a/core/fpdfapi/page/cpdf_psfunc.h
+++ b/core/fpdfapi/page/cpdf_psfunc.h
@@ -7,6 +7,8 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_PSFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_PSFUNC_H_
 
+#include <set>
+
 #include "core/fpdfapi/page/cpdf_function.h"
 #include "core/fpdfapi/page/cpdf_psengine.h"
 
@@ -18,7 +20,7 @@ class CPDF_PSFunc : public CPDF_Function {
   ~CPDF_PSFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
  private:
diff --git a/core/fpdfapi/page/cpdf_sampledfunc.cpp b/core/fpdfapi/page/cpdf_sampledfunc.cpp
index 98e1fd03e0..78c4f2c755 100644
--- a/core/fpdfapi/page/cpdf_sampledfunc.cpp
+++ b/core/fpdfapi/page/cpdf_sampledfunc.cpp
@@ -36,7 +36,8 @@ CPDF_SampledFunc::CPDF_SampledFunc() : CPDF_Function(Type::kType0Sampled) {}
 
 CPDF_SampledFunc::~CPDF_SampledFunc() {}
 
-bool CPDF_SampledFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_SampledFunc::v_Init(CPDF_Object* pObj,
+                              std::set<CPDF_Object*>* pVisited) {
   CPDF_Stream* pStream = pObj->AsStream();
   if (!pStream)
     return false;
diff --git a/core/fpdfapi/page/cpdf_sampledfunc.h b/core/fpdfapi/page/cpdf_sampledfunc.h
index d6dfed575f..b06b14bac1 100644
--- a/core/fpdfapi/page/cpdf_sampledfunc.h
+++ b/core/fpdfapi/page/cpdf_sampledfunc.h
@@ -7,6 +7,7 @@
 #ifndef CORE_FPDFAPI_PAGE_CPDF_SAMPLEDFUNC_H_
 #define CORE_FPDFAPI_PAGE_CPDF_SAMPLEDFUNC_H_
 
+#include <set>
 #include <vector>
 
 #include "core/fpdfapi/page/cpdf_function.h"
@@ -30,7 +31,7 @@ class CPDF_SampledFunc : public CPDF_Function {
   ~CPDF_SampledFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   const std::vector<SampleEncodeInfo>& GetEncodeInfo() const {
diff --git a/core/fpdfapi/page/cpdf_stitchfunc.cpp b/core/fpdfapi/page/cpdf_stitchfunc.cpp
index 336c74bf5c..fd83fd651a 100644
--- a/core/fpdfapi/page/cpdf_stitchfunc.cpp
+++ b/core/fpdfapi/page/cpdf_stitchfunc.cpp
@@ -22,7 +22,8 @@ CPDF_StitchFunc::~CPDF_StitchFunc() {
   FX_Free(m_pEncode);
 }
 
-bool CPDF_StitchFunc::v_Init(CPDF_Object* pObj) {
+bool CPDF_StitchFunc::v_Init(CPDF_Object* pObj,
+                             std::set<CPDF_Object*>* pVisited) {
   CPDF_Dictionary* pDict = pObj->GetDict();
   if (!pDict) {
     return false;
@@ -42,7 +43,7 @@ bool CPDF_StitchFunc::v_Init(CPDF_Object* pObj) {
     CPDF_Object* pSub = pArray->GetDirectObjectAt(i);
     if (pSub == pObj)
       return false;
-    std::unique_ptr<CPDF_Function> pFunc(CPDF_Function::Load(pSub));
+    std::unique_ptr<CPDF_Function> pFunc(CPDF_Function::Load(pSub, pVisited));
     if (!pFunc)
       return false;
     // Check that the input dimensionality is 1, and that all output
diff --git a/core/fpdfapi/page/cpdf_stitchfunc.h b/core/fpdfapi/page/cpdf_stitchfunc.h
index 69e5e41041..11396f12d7 100644
--- a/core/fpdfapi/page/cpdf_stitchfunc.h
+++ b/core/fpdfapi/page/cpdf_stitchfunc.h
@@ -8,6 +8,7 @@
 #define CORE_FPDFAPI_PAGE_CPDF_STITCHFUNC_H_
 
 #include <memory>
+#include <set>
 #include <vector>
 
 #include "core/fpdfapi/page/cpdf_function.h"
@@ -18,7 +19,7 @@ class CPDF_StitchFunc : public CPDF_Function {
   ~CPDF_StitchFunc() override;
 
   // CPDF_Function
-  bool v_Init(CPDF_Object* pObj) override;
+  bool v_Init(CPDF_Object* pObj, std::set<CPDF_Object*>* pVisited) override;
   bool v_Call(float* inputs, float* results) const override;
 
   const std::vector<std::unique_ptr<CPDF_Function>>& GetSubFunctions() const {