Cleanup: Remove unused CFX_PSRenderer and various encoders it used.

- Simplify remaining encoders.
- Hook up more of the encoding paths so failures get returned.
- Clean up code to not pass by non-const ref, use bools, etc.

R=ochang@chromium.org

Review URL: https://codereview.chromium.org/1545563003 .

2 files changed, 60 insertions, 42 deletions

diff --git a/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp b/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
index 49ec0bae92..1151c6390e 100644
--- a/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
+++ b/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
@@ -307,11 +307,11 @@ class CPDF_FlateEncoder {
  public:
   CPDF_FlateEncoder();
   ~CPDF_FlateEncoder();
-  FX_BOOL Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode);
-  FX_BOOL Initialize(const uint8_t* pBuffer,
-                     FX_DWORD size,
-                     FX_BOOL bFlateEncode,
-                     FX_BOOL bXRefStream = FALSE);
+  bool Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode);
+  bool Initialize(const uint8_t* pBuffer,
+                  FX_DWORD size,
+                  bool bFlateEncode,
+                  bool bXRefStream);
   void CloneDict();
   uint8_t* m_pData;
   FX_DWORD m_dwSize;
@@ -334,8 +334,7 @@ void CPDF_FlateEncoder::CloneDict() {
     m_bCloned = TRUE;
   }
 }
-FX_BOOL CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream,
-                                      FX_BOOL bFlateEncode) {
+bool CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode) {
   m_Acc.LoadAllData(pStream, TRUE);
   if ((pStream && pStream->GetDict() &&
        pStream->GetDict()->KeyExist("Filter")) ||
@@ -354,36 +353,41 @@ FX_BOOL CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream,
       m_dwSize = m_Acc.GetSize();
       m_pDict = pStream->GetDict();
     }
-    return TRUE;
+    return true;
   }
-  m_pData = NULL;
+  m_pData = nullptr;
   m_dwSize = 0;
   m_bNewData = TRUE;
   m_bCloned = TRUE;
-  ::FlateEncode(m_Acc.GetData(), m_Acc.GetSize(), m_pData, m_dwSize);
+  if (!::FlateEncode(m_Acc.GetData(), m_Acc.GetSize(), &m_pData, &m_dwSize))
+    return false;
+
   m_pDict = ToDictionary(pStream->GetDict()->Clone());
   m_pDict->SetAtInteger("Length", m_dwSize);
   m_pDict->SetAtName("Filter", "FlateDecode");
   m_pDict->RemoveAt("DecodeParms");
-  return TRUE;
+  return true;
 }
-FX_BOOL CPDF_FlateEncoder::Initialize(const uint8_t* pBuffer,
-                                      FX_DWORD size,
-                                      FX_BOOL bFlateEncode,
-                                      FX_BOOL bXRefStream) {
+
+bool CPDF_FlateEncoder::Initialize(const uint8_t* pBuffer,
+                                   FX_DWORD size,
+                                   bool bFlateEncode,
+                                   bool bXRefStream) {
   if (!bFlateEncode) {
     m_pData = (uint8_t*)pBuffer;
     m_dwSize = size;
-    return TRUE;
+    return true;
   }
   m_bNewData = TRUE;
+  bool ret;
   if (bXRefStream) {
-    ::FlateEncode(pBuffer, size, 12, 1, 8, 7, m_pData, m_dwSize);
+    ret = ::PngEncode(pBuffer, size, &m_pData, &m_dwSize);
   } else {
-    ::FlateEncode(pBuffer, size, m_pData, m_dwSize);
+    ret = ::FlateEncode(pBuffer, size, &m_pData, &m_dwSize);
   }
-  return TRUE;
+  return ret;
 }
+
 CPDF_FlateEncoder::~CPDF_FlateEncoder() {
   if (m_bCloned && m_pDict) {
     m_pDict->Release();
@@ -518,11 +522,15 @@ FX_FILESIZE CPDF_ObjectStream::End(CPDF_Creator* pCreator) {
   } else {
     tempBuffer << m_Buffer;
     CPDF_FlateEncoder encoder;
-    encoder.Initialize(tempBuffer.GetBuffer(), tempBuffer.GetLength(),
-                       pCreator->m_bCompress);
+    if (!encoder.Initialize(tempBuffer.GetBuffer(), tempBuffer.GetLength(),
+                            pCreator->m_bCompress, false)) {
+      return -1;
+    }
     CPDF_Encryptor encryptor;
-    encryptor.Initialize(pHandler, m_dwObjNum, encoder.m_pData,
-                         encoder.m_dwSize);
+    if (!encryptor.Initialize(pHandler, m_dwObjNum, encoder.m_pData,
+                              encoder.m_dwSize)) {
+      return -1;
+    }
     if ((len = pFile->AppendDWord(encryptor.m_dwSize)) < 0) {
       return -1;
     }
@@ -768,22 +776,21 @@ FX_BOOL CPDF_XRefStream::GenerateXRefStream(CPDF_Creator* pCreator,
     }
     offset += len + 6;
   }
-  FX_BOOL bPredictor = TRUE;
   CPDF_FlateEncoder encoder;
-  encoder.Initialize(m_Buffer.GetBuffer(), m_Buffer.GetLength(),
-                     pCreator->m_bCompress, bPredictor);
+  if (!encoder.Initialize(m_Buffer.GetBuffer(), m_Buffer.GetLength(),
+                          pCreator->m_bCompress, true)) {
+    return FALSE;
+  }
   if (pCreator->m_bCompress) {
     if (pFile->AppendString("/Filter /FlateDecode") < 0) {
       return FALSE;
     }
     offset += 20;
-    if (bPredictor) {
-      if ((len = pFile->AppendString(
-               "/DecodeParms<</Columns 7/Predictor 12>>")) < 0) {
-        return FALSE;
-      }
-      offset += len;
+    if ((len = pFile->AppendString("/DecodeParms<</Columns 7/Predictor 12>>")) <
+        0) {
+      return FALSE;
     }
+    offset += len;
   }
   if (pFile->AppendString("/Length ") < 0) {
     return FALSE;
@@ -1002,8 +1009,10 @@ int32_t CPDF_Creator::WriteStream(const CPDF_Object* pStream,
                                   FX_DWORD objnum,
                                   CPDF_CryptoHandler* pCrypto) {
   CPDF_FlateEncoder encoder;
-  encoder.Initialize(const_cast<CPDF_Stream*>(pStream->AsStream()),
-                     pStream == m_pMetadata ? FALSE : m_bCompress);
+  if (!encoder.Initialize(const_cast<CPDF_Stream*>(pStream->AsStream()),
+                          pStream == m_pMetadata ? FALSE : m_bCompress)) {
+    return -1;
+  }
   CPDF_Encryptor encryptor;
   if (!encryptor.Initialize(pCrypto, objnum, encoder.m_pData,
                             encoder.m_dwSize)) {
@@ -1107,8 +1116,10 @@ int32_t CPDF_Creator::WriteDirectObj(FX_DWORD objnum,
         break;
       }
       CPDF_Encryptor encryptor;
-      encryptor.Initialize(m_pCryptoHandler, objnum, (uint8_t*)str.c_str(),
-                           str.GetLength());
+      if (!encryptor.Initialize(m_pCryptoHandler, objnum, (uint8_t*)str.c_str(),
+                                str.GetLength())) {
+        return -1;
+      }
       CFX_ByteString content = PDF_EncodeString(
           CFX_ByteString((const FX_CHAR*)encryptor.m_pData, encryptor.m_dwSize),
           bHex);
@@ -1120,11 +1131,17 @@ int32_t CPDF_Creator::WriteDirectObj(FX_DWORD objnum,
     }
     case PDFOBJ_STREAM: {
       CPDF_FlateEncoder encoder;
-      encoder.Initialize(const_cast<CPDF_Stream*>(pObj->AsStream()),
-                         m_bCompress);
+      if (!encoder.Initialize(const_cast<CPDF_Stream*>(pObj->AsStream()),
+                              m_bCompress)) {
+        return -1;
+      }
+
       CPDF_Encryptor encryptor;
       CPDF_CryptoHandler* pHandler = m_pCryptoHandler;
-      encryptor.Initialize(pHandler, objnum, encoder.m_pData, encoder.m_dwSize);
+      if (!encryptor.Initialize(pHandler, objnum, encoder.m_pData,
+                                encoder.m_dwSize)) {
+        return -1;
+      }
       if ((FX_DWORD)encoder.m_pDict->GetInteger("Length") !=
           encryptor.m_dwSize) {
         encoder.CloneDict();
diff --git a/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp b/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
index 2b68a0403f..386c49d005 100644
--- a/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
+++ b/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
@@ -16,11 +16,12 @@ CPDF_Dictionary* CPDF_Image::InitJPEG(uint8_t* pData, FX_DWORD size) {
   int32_t height;
   int32_t num_comps;
   int32_t bits;
-  FX_BOOL color_trans;
+  bool color_trans;
   if (!CPDF_ModuleMgr::Get()->GetJpegModule()->LoadInfo(
-          pData, size, width, height, num_comps, bits, color_trans)) {
-    return NULL;
+          pData, size, &width, &height, &num_comps, &bits, &color_trans)) {
+    return nullptr;
   }
+
   CPDF_Dictionary* pDict = new CPDF_Dictionary;
   pDict->SetAtName("Type", "XObject");
   pDict->SetAtName("Subtype", "Image");