Revert "Revert "Redo range check in CPDF_SampledFunc::v_Call().""

This reverts commit 327080bacabb3d90094144e66ae57037d4a8d8e6.
This re-lands the patch, but removes the tests which would require
512MB buffers to be reliably allocatable on small bots. It
also adds
small fixes accumulated while diagnosing it.

R=sanqingsong@gmail.com

Review URL: https://codereview.chromium.org/1214403002.

2 files changed, 36 insertions, 29 deletions

diff --git a/core/src/fpdfapi/fpdf_page/fpdf_page_colors.cpp b/core/src/fpdfapi/fpdf_page/fpdf_page_colors.cpp
index 0ee7a0da7d..9e6ca67041 100644
--- a/core/src/fpdfapi/fpdf_page/fpdf_page_colors.cpp
+++ b/core/src/fpdfapi/fpdf_page/fpdf_page_colors.cpp
@@ -950,11 +950,14 @@ FX_BOOL CPDF_SeparationCS::v_Load(CPDF_Document* pDoc, CPDF_Array* pArray)
             return FALSE;
         }
         m_pAltCS = Load(pDoc, pAltCS);
+        if (!m_pAltCS) {
+            return FALSE;
+        }
         CPDF_Object* pFuncObj = pArray->GetElementValue(3);
         if (pFuncObj && pFuncObj->GetType() != PDFOBJ_NAME) {
             m_pFunc = CPDF_Function::Load(pFuncObj);
         }
-        if (m_pFunc && m_pAltCS && m_pFunc->CountOutputs() < m_pAltCS->CountComponents()) {
+        if (m_pFunc && m_pFunc->CountOutputs() < m_pAltCS->CountComponents()) {
             delete m_pFunc;
             m_pFunc = NULL;
         }
diff --git a/core/src/fpdfapi/fpdf_page/fpdf_page_func.cpp b/core/src/fpdfapi/fpdf_page/fpdf_page_func.cpp
index f115b6770d..c961f8b20f 100644
--- a/core/src/fpdfapi/fpdf_page/fpdf_page_func.cpp
+++ b/core/src/fpdfapi/fpdf_page/fpdf_page_func.cpp
@@ -449,7 +449,8 @@ public:
     virtual FX_BOOL		v_Call(FX_FLOAT* inputs, FX_FLOAT* results) const;
     SampleEncodeInfo*	m_pEncodeInfo;
     SampleDecodeInfo*	m_pDecodeInfo;
-    FX_DWORD	m_nBitsPerSample, m_SampleMax;
+    FX_DWORD	m_nBitsPerSample;
+    FX_DWORD	m_SampleMax;
     CPDF_StreamAcc*	m_pSampleStream;
 };
 CPDF_SampledFunc::CPDF_SampledFunc()
@@ -479,21 +480,20 @@ FX_BOOL CPDF_SampledFunc::v_Init(CPDF_Object* pObj)
     CPDF_Array* pEncode = pDict->GetArray(FX_BSTRC("Encode"));
     CPDF_Array* pDecode = pDict->GetArray(FX_BSTRC("Decode"));
     m_nBitsPerSample = pDict->GetInteger(FX_BSTRC("BitsPerSample"));
+    if (m_nBitsPerSample > 32) {
+        return FALSE;
+    }
     m_SampleMax = 0xffffffff >> (32 - m_nBitsPerSample);
     m_pSampleStream = new CPDF_StreamAcc;
     m_pSampleStream->LoadAllData(pStream, FALSE);
     m_pEncodeInfo = FX_Alloc(SampleEncodeInfo, m_nInputs);
-    int i;
-    FX_DWORD nTotalSamples = 1;
-    for (i = 0; i < m_nInputs; i ++) {
+    FX_SAFE_DWORD nTotalSampleBits = 1;
+    for (int i = 0; i < m_nInputs; i ++) {
         m_pEncodeInfo[i].sizes = pSize ? pSize->GetInteger(i) : 0;
         if (!pSize && i == 0) {
             m_pEncodeInfo[i].sizes = pDict->GetInteger(FX_BSTRC("Size"));
         }
-        if (nTotalSamples > 0 && (uint32_t)(m_pEncodeInfo[i].sizes) > UINT_MAX / nTotalSamples) {
-            return FALSE;
-        }
-        nTotalSamples *= m_pEncodeInfo[i].sizes;
+        nTotalSampleBits *= m_pEncodeInfo[i].sizes;
         if (pEncode) {
             m_pEncodeInfo[i].encode_min = pEncode->GetFloat(i * 2);
             m_pEncodeInfo[i].encode_max = pEncode->GetFloat(i * 2 + 1);
@@ -506,19 +506,18 @@ FX_BOOL CPDF_SampledFunc::v_Init(CPDF_Object* pObj)
             }
         }
     }
-    if (nTotalSamples > 0 && m_nBitsPerSample > UINT_MAX / nTotalSamples) {
-        return FALSE;
-    }
-    nTotalSamples *= m_nBitsPerSample;
-    if (nTotalSamples > 0 && ((uint32_t)m_nOutputs) > UINT_MAX / nTotalSamples) {
-        return FALSE;
-    }
-    nTotalSamples *= m_nOutputs;
-    if (nTotalSamples == 0 || m_pSampleStream->GetSize() * 8 < nTotalSamples) {
+    nTotalSampleBits *= m_nBitsPerSample;
+    nTotalSampleBits *= m_nOutputs;
+    FX_SAFE_DWORD nTotalSampleBytes = nTotalSampleBits;
+    nTotalSampleBytes += 7;
+    nTotalSampleBytes /= 8;
+    if (!nTotalSampleBytes.IsValid() ||
+        nTotalSampleBytes.ValueOrDie() == 0 ||
+        nTotalSampleBytes.ValueOrDie() > m_pSampleStream->GetSize()) {
         return FALSE;
     }
     m_pDecodeInfo = FX_Alloc(SampleDecodeInfo, m_nOutputs);
-    for (i = 0; i < m_nOutputs; i ++) {
+    for (int i = 0; i < m_nOutputs; i ++) {
         if (pDecode) {
             m_pDecodeInfo[i].decode_min = pDecode->GetFloat(2 * i);
             m_pDecodeInfo[i].decode_max = pDecode->GetFloat(2 * i + 1);
@@ -553,20 +552,23 @@ FX_BOOL CPDF_SampledFunc::v_Call(FX_FLOAT* inputs, FX_FLOAT* results) const
         }
         pos += index[i] * blocksize[i];
     }
+    FX_SAFE_INT32 bits_to_output = m_nOutputs;
+    bits_to_output *= m_nBitsPerSample;
+    if (!bits_to_output.IsValid()) {
+        return FALSE;
+    }
     FX_SAFE_INT32 bitpos = pos;
-    bitpos *= m_nBitsPerSample;
-    bitpos *= m_nOutputs;
+    bitpos *= bits_to_output.ValueOrDie();
     if (!bitpos.IsValid()) {
         return FALSE;
     }
-    const uint8_t* pSampleData = m_pSampleStream->GetData();
-    if (pSampleData == NULL) {
+    FX_SAFE_INT32 range_check = bitpos;
+    range_check += bits_to_output.ValueOrDie();
+    if (!range_check.IsValid()) {
         return FALSE;
     }
-    FX_SAFE_INT32 bitpos1 = m_nOutputs - 1 > 0 ? m_nOutputs - 1 : 0;
-    bitpos1 *= m_nBitsPerSample;
-    bitpos1 += bitpos.ValueOrDie();
-    if (!bitpos1.IsValid()) {
+    const uint8_t* pSampleData = m_pSampleStream->GetData();
+    if (!pSampleData) {
         return FALSE;
     }
     for (int j = 0; j < m_nOutputs; j ++) {
@@ -865,14 +867,16 @@ FX_BOOL CPDF_Function::Init(CPDF_Object* pObj)
         }
     }
     FX_DWORD old_outputs = m_nOutputs;
-    FX_BOOL ret = v_Init(pObj);
+    if (!v_Init(pObj)) {
+        return FALSE;
+    }
     if (m_pRanges && m_nOutputs > (int)old_outputs) {
         m_pRanges = FX_Realloc(FX_FLOAT, m_pRanges, m_nOutputs * 2);
         if (m_pRanges) {
             FXSYS_memset(m_pRanges + (old_outputs * 2), 0, sizeof(FX_FLOAT) * (m_nOutputs - old_outputs) * 2);
         }
     }
-    return ret;
+    return TRUE;
 }
 FX_BOOL CPDF_Function::Call(FX_FLOAT* inputs, int ninputs, FX_FLOAT* results, int& nresults) const
 {