Cleanup: Remove unused CFX_PSRenderer and various encoders it used.

- Simplify remaining encoders.
- Hook up more of the encoding paths so failures get returned.
- Clean up code to not pass by non-const ref, use bools, etc.

R=ochang@chromium.org

Review URL: https://codereview.chromium.org/1545563003 .

5 files changed, 85 insertions, 70 deletions

diff --git a/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp b/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
index 49ec0bae92..1151c6390e 100644
--- a/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
+++ b/core/src/fpdfapi/fpdf_edit/fpdf_edit_create.cpp
@@ -307,11 +307,11 @@ class CPDF_FlateEncoder {
  public:
   CPDF_FlateEncoder();
   ~CPDF_FlateEncoder();
-  FX_BOOL Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode);
-  FX_BOOL Initialize(const uint8_t* pBuffer,
-                     FX_DWORD size,
-                     FX_BOOL bFlateEncode,
-                     FX_BOOL bXRefStream = FALSE);
+  bool Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode);
+  bool Initialize(const uint8_t* pBuffer,
+                  FX_DWORD size,
+                  bool bFlateEncode,
+                  bool bXRefStream);
   void CloneDict();
   uint8_t* m_pData;
   FX_DWORD m_dwSize;
@@ -334,8 +334,7 @@ void CPDF_FlateEncoder::CloneDict() {
     m_bCloned = TRUE;
   }
 }
-FX_BOOL CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream,
-                                      FX_BOOL bFlateEncode) {
+bool CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream, FX_BOOL bFlateEncode) {
   m_Acc.LoadAllData(pStream, TRUE);
   if ((pStream && pStream->GetDict() &&
        pStream->GetDict()->KeyExist("Filter")) ||
@@ -354,36 +353,41 @@ FX_BOOL CPDF_FlateEncoder::Initialize(CPDF_Stream* pStream,
       m_dwSize = m_Acc.GetSize();
       m_pDict = pStream->GetDict();
     }
-    return TRUE;
+    return true;
   }
-  m_pData = NULL;
+  m_pData = nullptr;
   m_dwSize = 0;
   m_bNewData = TRUE;
   m_bCloned = TRUE;
-  ::FlateEncode(m_Acc.GetData(), m_Acc.GetSize(), m_pData, m_dwSize);
+  if (!::FlateEncode(m_Acc.GetData(), m_Acc.GetSize(), &m_pData, &m_dwSize))
+    return false;
+
   m_pDict = ToDictionary(pStream->GetDict()->Clone());
   m_pDict->SetAtInteger("Length", m_dwSize);
   m_pDict->SetAtName("Filter", "FlateDecode");
   m_pDict->RemoveAt("DecodeParms");
-  return TRUE;
+  return true;
 }
-FX_BOOL CPDF_FlateEncoder::Initialize(const uint8_t* pBuffer,
-                                      FX_DWORD size,
-                                      FX_BOOL bFlateEncode,
-                                      FX_BOOL bXRefStream) {
+
+bool CPDF_FlateEncoder::Initialize(const uint8_t* pBuffer,
+                                   FX_DWORD size,
+                                   bool bFlateEncode,
+                                   bool bXRefStream) {
   if (!bFlateEncode) {
     m_pData = (uint8_t*)pBuffer;
     m_dwSize = size;
-    return TRUE;
+    return true;
   }
   m_bNewData = TRUE;
+  bool ret;
   if (bXRefStream) {
-    ::FlateEncode(pBuffer, size, 12, 1, 8, 7, m_pData, m_dwSize);
+    ret = ::PngEncode(pBuffer, size, &m_pData, &m_dwSize);
   } else {
-    ::FlateEncode(pBuffer, size, m_pData, m_dwSize);
+    ret = ::FlateEncode(pBuffer, size, &m_pData, &m_dwSize);
   }
-  return TRUE;
+  return ret;
 }
+
 CPDF_FlateEncoder::~CPDF_FlateEncoder() {
   if (m_bCloned && m_pDict) {
     m_pDict->Release();
@@ -518,11 +522,15 @@ FX_FILESIZE CPDF_ObjectStream::End(CPDF_Creator* pCreator) {
   } else {
     tempBuffer << m_Buffer;
     CPDF_FlateEncoder encoder;
-    encoder.Initialize(tempBuffer.GetBuffer(), tempBuffer.GetLength(),
-                       pCreator->m_bCompress);
+    if (!encoder.Initialize(tempBuffer.GetBuffer(), tempBuffer.GetLength(),
+                            pCreator->m_bCompress, false)) {
+      return -1;
+    }
     CPDF_Encryptor encryptor;
-    encryptor.Initialize(pHandler, m_dwObjNum, encoder.m_pData,
-                         encoder.m_dwSize);
+    if (!encryptor.Initialize(pHandler, m_dwObjNum, encoder.m_pData,
+                              encoder.m_dwSize)) {
+      return -1;
+    }
     if ((len = pFile->AppendDWord(encryptor.m_dwSize)) < 0) {
       return -1;
     }
@@ -768,22 +776,21 @@ FX_BOOL CPDF_XRefStream::GenerateXRefStream(CPDF_Creator* pCreator,
     }
     offset += len + 6;
   }
-  FX_BOOL bPredictor = TRUE;
   CPDF_FlateEncoder encoder;
-  encoder.Initialize(m_Buffer.GetBuffer(), m_Buffer.GetLength(),
-                     pCreator->m_bCompress, bPredictor);
+  if (!encoder.Initialize(m_Buffer.GetBuffer(), m_Buffer.GetLength(),
+                          pCreator->m_bCompress, true)) {
+    return FALSE;
+  }
   if (pCreator->m_bCompress) {
     if (pFile->AppendString("/Filter /FlateDecode") < 0) {
       return FALSE;
     }
     offset += 20;
-    if (bPredictor) {
-      if ((len = pFile->AppendString(
-               "/DecodeParms<</Columns 7/Predictor 12>>")) < 0) {
-        return FALSE;
-      }
-      offset += len;
+    if ((len = pFile->AppendString("/DecodeParms<</Columns 7/Predictor 12>>")) <
+        0) {
+      return FALSE;
     }
+    offset += len;
   }
   if (pFile->AppendString("/Length ") < 0) {
     return FALSE;
@@ -1002,8 +1009,10 @@ int32_t CPDF_Creator::WriteStream(const CPDF_Object* pStream,
                                   FX_DWORD objnum,
                                   CPDF_CryptoHandler* pCrypto) {
   CPDF_FlateEncoder encoder;
-  encoder.Initialize(const_cast<CPDF_Stream*>(pStream->AsStream()),
-                     pStream == m_pMetadata ? FALSE : m_bCompress);
+  if (!encoder.Initialize(const_cast<CPDF_Stream*>(pStream->AsStream()),
+                          pStream == m_pMetadata ? FALSE : m_bCompress)) {
+    return -1;
+  }
   CPDF_Encryptor encryptor;
   if (!encryptor.Initialize(pCrypto, objnum, encoder.m_pData,
                             encoder.m_dwSize)) {
@@ -1107,8 +1116,10 @@ int32_t CPDF_Creator::WriteDirectObj(FX_DWORD objnum,
         break;
       }
       CPDF_Encryptor encryptor;
-      encryptor.Initialize(m_pCryptoHandler, objnum, (uint8_t*)str.c_str(),
-                           str.GetLength());
+      if (!encryptor.Initialize(m_pCryptoHandler, objnum, (uint8_t*)str.c_str(),
+                                str.GetLength())) {
+        return -1;
+      }
       CFX_ByteString content = PDF_EncodeString(
           CFX_ByteString((const FX_CHAR*)encryptor.m_pData, encryptor.m_dwSize),
           bHex);
@@ -1120,11 +1131,17 @@ int32_t CPDF_Creator::WriteDirectObj(FX_DWORD objnum,
     }
     case PDFOBJ_STREAM: {
       CPDF_FlateEncoder encoder;
-      encoder.Initialize(const_cast<CPDF_Stream*>(pObj->AsStream()),
-                         m_bCompress);
+      if (!encoder.Initialize(const_cast<CPDF_Stream*>(pObj->AsStream()),
+                              m_bCompress)) {
+        return -1;
+      }
+
       CPDF_Encryptor encryptor;
       CPDF_CryptoHandler* pHandler = m_pCryptoHandler;
-      encryptor.Initialize(pHandler, objnum, encoder.m_pData, encoder.m_dwSize);
+      if (!encryptor.Initialize(pHandler, objnum, encoder.m_pData,
+                                encoder.m_dwSize)) {
+        return -1;
+      }
       if ((FX_DWORD)encoder.m_pDict->GetInteger("Length") !=
           encryptor.m_dwSize) {
         encoder.CloneDict();
diff --git a/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp b/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
index 2b68a0403f..386c49d005 100644
--- a/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
+++ b/core/src/fpdfapi/fpdf_edit/fpdf_edit_image.cpp
@@ -16,11 +16,12 @@ CPDF_Dictionary* CPDF_Image::InitJPEG(uint8_t* pData, FX_DWORD size) {
   int32_t height;
   int32_t num_comps;
   int32_t bits;
-  FX_BOOL color_trans;
+  bool color_trans;
   if (!CPDF_ModuleMgr::Get()->GetJpegModule()->LoadInfo(
-          pData, size, width, height, num_comps, bits, color_trans)) {
-    return NULL;
+          pData, size, &width, &height, &num_comps, &bits, &color_trans)) {
+    return nullptr;
   }
+
   CPDF_Dictionary* pDict = new CPDF_Dictionary;
   pDict->SetAtName("Type", "XObject");
   pDict->SetAtName("Subtype", "Image");
diff --git a/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode.cpp b/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode.cpp
index c855a7c5c2..8832940f80 100644
--- a/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode.cpp
+++ b/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode.cpp
@@ -540,30 +540,27 @@ CFX_ByteString PDF_EncodeString(const CFX_ByteString& src, FX_BOOL bHex) {
   result.AppendChar(')');
   return result.GetByteString();
 }
-void FlateEncode(const uint8_t* src_buf,
+
+bool FlateEncode(const uint8_t* src_buf,
                  FX_DWORD src_size,
-                 uint8_t*& dest_buf,
-                 FX_DWORD& dest_size) {
+                 uint8_t** dest_buf,
+                 FX_DWORD* dest_size) {
   CCodec_ModuleMgr* pEncoders = CPDF_ModuleMgr::Get()->GetCodecModule();
-  if (pEncoders) {
-    pEncoders->GetFlateModule()->Encode(src_buf, src_size, dest_buf, dest_size);
-  }
+  return pEncoders &&
+         pEncoders->GetFlateModule()->Encode(src_buf, src_size, dest_buf,
+                                             dest_size);
 }
-void FlateEncode(const uint8_t* src_buf,
-                 FX_DWORD src_size,
-                 int predictor,
-                 int Colors,
-                 int BitsPerComponent,
-                 int Columns,
-                 uint8_t*& dest_buf,
-                 FX_DWORD& dest_size) {
+
+bool PngEncode(const uint8_t* src_buf,
+               FX_DWORD src_size,
+               uint8_t** dest_buf,
+               FX_DWORD* dest_size) {
   CCodec_ModuleMgr* pEncoders = CPDF_ModuleMgr::Get()->GetCodecModule();
-  if (pEncoders) {
-    pEncoders->GetFlateModule()->Encode(src_buf, src_size, predictor, Colors,
-                                        BitsPerComponent, Columns, dest_buf,
-                                        dest_size);
-  }
+  return pEncoders &&
+         pEncoders->GetFlateModule()->Encode(src_buf, src_size, dest_buf,
+                                             dest_size);
 }
+
 FX_DWORD FlateDecode(const uint8_t* src_buf,
                      FX_DWORD src_size,
                      uint8_t*& dest_buf,
diff --git a/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode_embeddertest.cpp b/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode_embeddertest.cpp
index c80770366b..810a633c83 100644
--- a/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode_embeddertest.cpp
+++ b/core/src/fpdfapi/fpdf_parser/fpdf_parser_decode_embeddertest.cpp
@@ -47,14 +47,14 @@ TEST_F(FPDFParserDecodeEmbeddertest, FlateEncode) {
 
   for (size_t i = 0; i < FX_ArraySize(flate_encode_cases); ++i) {
     FlateEncodeCase* ptr = &flate_encode_cases[i];
-    unsigned char* result;
-    unsigned int result_size;
-    FlateEncode(ptr->input, ptr->input_size, result, result_size);
-    ASSERT_TRUE(result);
+    unsigned char* buf;
+    unsigned int buf_size;
+    EXPECT_TRUE(FlateEncode(ptr->input, ptr->input_size, &buf, &buf_size));
+    ASSERT_TRUE(buf);
     EXPECT_EQ(std::string((const char*)ptr->expected, ptr->expected_size),
-              std::string((const char*)result, result_size))
+              std::string((const char*)buf, buf_size))
         << " for case " << i;
-    FX_Free(result);
+    FX_Free(buf);
   }
 }
 
diff --git a/core/src/fpdfapi/fpdf_render/fpdf_render_loadimage.cpp b/core/src/fpdfapi/fpdf_render/fpdf_render_loadimage.cpp
index 0306ff0227..c107c002c2 100644
--- a/core/src/fpdfapi/fpdf_render/fpdf_render_loadimage.cpp
+++ b/core/src/fpdfapi/fpdf_render/fpdf_render_loadimage.cpp
@@ -615,11 +615,11 @@ int CPDF_DIBSource::CreateDecoder() {
         src_data, src_size, m_Width, m_Height, m_nComponents,
         pParams ? pParams->GetInteger("ColorTransform", 1) : 1);
     if (!m_pDecoder) {
-      FX_BOOL bTransform = FALSE;
+      bool bTransform = false;
       int comps, bpc;
       ICodec_JpegModule* pJpegModule = CPDF_ModuleMgr::Get()->GetJpegModule();
-      if (pJpegModule->LoadInfo(src_data, src_size, m_Width, m_Height, comps,
-                                bpc, bTransform)) {
+      if (pJpegModule->LoadInfo(src_data, src_size, &m_Width, &m_Height, &comps,
+                                &bpc, &bTransform)) {
         if (m_nComponents != comps) {
           FX_Free(m_pCompData);
           m_nComponents = comps;