Remove type-unsafe void* / uint8_t* usage in fx_crypt.h

Consolidate all of the sha2 contexts while we're at it, the one
with the largest buf is suitable for use by all the others.

Change-Id: Iace6cd8ca4405f75f78842a1559c3a2478910218
Reviewed-on: https://pdfium-review.googlesource.com/4994
Commit-Queue: Tom Sepez <tsepez@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>

3 files changed, 79 insertions, 67 deletions

diff --git a/core/fpdfapi/parser/cpdf_crypto_handler.cpp b/core/fpdfapi/parser/cpdf_crypto_handler.cpp
index ea09601b30..ef84480c23 100644
--- a/core/fpdfapi/parser/cpdf_crypto_handler.cpp
+++ b/core/fpdfapi/parser/cpdf_crypto_handler.cpp
@@ -41,26 +41,29 @@ void CPDF_CryptoHandler::CryptBlock(bool bEncrypt,
     }
   }
   if (m_Cipher == FXCIPHER_AES) {
-    CRYPT_AESSetKey(m_pAESContext, 16, m_KeyLen == 32 ? m_EncryptKey : realkey,
-                    m_KeyLen, bEncrypt);
+    CRYPT_AESSetKey(m_pAESContext.get(), 16,
+                    m_KeyLen == 32 ? m_EncryptKey : realkey, m_KeyLen,
+                    bEncrypt);
     if (bEncrypt) {
       uint8_t iv[16];
       for (int i = 0; i < 16; i++) {
         iv[i] = (uint8_t)rand();
       }
-      CRYPT_AESSetIV(m_pAESContext, iv);
+      CRYPT_AESSetIV(m_pAESContext.get(), iv);
       memcpy(dest_buf, iv, 16);
       int nblocks = src_size / 16;
-      CRYPT_AESEncrypt(m_pAESContext, dest_buf + 16, src_buf, nblocks * 16);
+      CRYPT_AESEncrypt(m_pAESContext.get(), dest_buf + 16, src_buf,
+                       nblocks * 16);
       uint8_t padding[16];
       memcpy(padding, src_buf + nblocks * 16, src_size % 16);
       memset(padding + src_size % 16, 16 - src_size % 16, 16 - src_size % 16);
-      CRYPT_AESEncrypt(m_pAESContext, dest_buf + nblocks * 16 + 16, padding,
-                       16);
+      CRYPT_AESEncrypt(m_pAESContext.get(), dest_buf + nblocks * 16 + 16,
+                       padding, 16);
       dest_size = 32 + nblocks * 16;
     } else {
-      CRYPT_AESSetIV(m_pAESContext, src_buf);
-      CRYPT_AESDecrypt(m_pAESContext, dest_buf, src_buf + 16, src_size - 16);
+      CRYPT_AESSetIV(m_pAESContext.get(), src_buf);
+      CRYPT_AESDecrypt(m_pAESContext.get(), dest_buf, src_buf + 16,
+                       src_size - 16);
       dest_size = src_size - 16;
       dest_size -= dest_buf[dest_size - 1];
     }
@@ -74,10 +77,10 @@ void CPDF_CryptoHandler::CryptBlock(bool bEncrypt,
 }
 
 struct AESCryptContext {
-  uint8_t m_Context[2048];
   bool m_bIV;
   uint8_t m_Block[16];
   uint32_t m_BlockOffset;
+  CRYPT_aes_context m_Context;
 };
 
 void* CPDF_CryptoHandler::CryptStart(uint32_t objnum,
@@ -90,12 +93,12 @@ void* CPDF_CryptoHandler::CryptStart(uint32_t objnum,
     AESCryptContext* pContext = FX_Alloc(AESCryptContext, 1);
     pContext->m_bIV = true;
     pContext->m_BlockOffset = 0;
-    CRYPT_AESSetKey(pContext->m_Context, 16, m_EncryptKey, 32, bEncrypt);
+    CRYPT_AESSetKey(&pContext->m_Context, 16, m_EncryptKey, 32, bEncrypt);
     if (bEncrypt) {
       for (int i = 0; i < 16; i++) {
         pContext->m_Block[i] = (uint8_t)rand();
       }
-      CRYPT_AESSetIV(pContext->m_Context, pContext->m_Block);
+      CRYPT_AESSetIV(&pContext->m_Context, pContext->m_Block);
     }
     return pContext;
   }
@@ -116,12 +119,12 @@ void* CPDF_CryptoHandler::CryptStart(uint32_t objnum,
     AESCryptContext* pContext = FX_Alloc(AESCryptContext, 1);
     pContext->m_bIV = true;
     pContext->m_BlockOffset = 0;
-    CRYPT_AESSetKey(pContext->m_Context, 16, realkey, 16, bEncrypt);
+    CRYPT_AESSetKey(&pContext->m_Context, 16, realkey, 16, bEncrypt);
     if (bEncrypt) {
       for (int i = 0; i < 16; i++) {
         pContext->m_Block[i] = (uint8_t)rand();
       }
-      CRYPT_AESSetIV(pContext->m_Context, pContext->m_Block);
+      CRYPT_AESSetIV(&pContext->m_Context, pContext->m_Block);
     }
     return pContext;
   }
@@ -168,16 +171,16 @@ bool CPDF_CryptoHandler::CryptStream(void* context,
     pContext->m_BlockOffset += copy_size;
     if (pContext->m_BlockOffset == 16) {
       if (!bEncrypt && pContext->m_bIV) {
-        CRYPT_AESSetIV(pContext->m_Context, pContext->m_Block);
+        CRYPT_AESSetIV(&pContext->m_Context, pContext->m_Block);
         pContext->m_bIV = false;
         pContext->m_BlockOffset = 0;
       } else if (src_off < src_size) {
         uint8_t block_buf[16];
         if (bEncrypt) {
-          CRYPT_AESEncrypt(pContext->m_Context, block_buf, pContext->m_Block,
+          CRYPT_AESEncrypt(&pContext->m_Context, block_buf, pContext->m_Block,
                            16);
         } else {
-          CRYPT_AESDecrypt(pContext->m_Context, block_buf, pContext->m_Block,
+          CRYPT_AESDecrypt(&pContext->m_Context, block_buf, pContext->m_Block,
                            16);
         }
         dest_buf.AppendBlock(block_buf, 16);
@@ -207,18 +210,18 @@ bool CPDF_CryptoHandler::CryptFinish(void* context,
   if (bEncrypt) {
     uint8_t block_buf[16];
     if (pContext->m_BlockOffset == 16) {
-      CRYPT_AESEncrypt(pContext->m_Context, block_buf, pContext->m_Block, 16);
+      CRYPT_AESEncrypt(&pContext->m_Context, block_buf, pContext->m_Block, 16);
       dest_buf.AppendBlock(block_buf, 16);
       pContext->m_BlockOffset = 0;
     }
     memset(pContext->m_Block + pContext->m_BlockOffset,
            (uint8_t)(16 - pContext->m_BlockOffset),
            16 - pContext->m_BlockOffset);
-    CRYPT_AESEncrypt(pContext->m_Context, block_buf, pContext->m_Block, 16);
+    CRYPT_AESEncrypt(&pContext->m_Context, block_buf, pContext->m_Block, 16);
     dest_buf.AppendBlock(block_buf, 16);
   } else if (pContext->m_BlockOffset == 16) {
     uint8_t block_buf[16];
-    CRYPT_AESDecrypt(pContext->m_Context, block_buf, pContext->m_Block, 16);
+    CRYPT_AESDecrypt(&pContext->m_Context, block_buf, pContext->m_Block, 16);
     if (block_buf[15] <= 16) {
       dest_buf.AppendBlock(block_buf, 16 - block_buf[15]);
     }
@@ -247,18 +250,18 @@ uint32_t CPDF_CryptoHandler::DecryptGetSize(uint32_t src_size) {
 bool CPDF_CryptoHandler::Init(CPDF_Dictionary* pEncryptDict,
                               CPDF_SecurityHandler* pSecurityHandler) {
   const uint8_t* key;
-  if (!pSecurityHandler->GetCryptInfo(m_Cipher, key, m_KeyLen)) {
+  if (!pSecurityHandler->GetCryptInfo(m_Cipher, key, m_KeyLen))
     return false;
-  }
-  if (m_KeyLen > 32 || m_KeyLen < 0) {
+
+  if (m_KeyLen > 32 || m_KeyLen < 0)
     return false;
-  }
-  if (m_Cipher != FXCIPHER_NONE) {
+
+  if (m_Cipher != FXCIPHER_NONE)
     memcpy(m_EncryptKey, key, m_KeyLen);
-  }
-  if (m_Cipher == FXCIPHER_AES) {
-    m_pAESContext = FX_Alloc(uint8_t, 2048);
-  }
+
+  if (m_Cipher == FXCIPHER_AES)
+    m_pAESContext.reset(FX_Alloc(CRYPT_aes_context, 1));
+
   return true;
 }
 
@@ -288,17 +291,19 @@ bool CPDF_CryptoHandler::Init(int cipher, const uint8_t* key, int keylen) {
   m_Cipher = cipher;
   m_KeyLen = keylen;
   memcpy(m_EncryptKey, key, keylen);
-  if (m_Cipher == FXCIPHER_AES) {
-    m_pAESContext = FX_Alloc(uint8_t, 2048);
-  }
+  if (m_Cipher == FXCIPHER_AES)
+    m_pAESContext.reset(FX_Alloc(CRYPT_aes_context, 1));
+
   return true;
 }
+
 bool CPDF_CryptoHandler::DecryptStream(void* context,
                                        const uint8_t* src_buf,
                                        uint32_t src_size,
                                        CFX_BinaryBuf& dest_buf) {
   return CryptStream(context, src_buf, src_size, dest_buf, false);
 }
+
 bool CPDF_CryptoHandler::DecryptFinish(void* context, CFX_BinaryBuf& dest_buf) {
   return CryptFinish(context, dest_buf, false);
 }
@@ -311,6 +316,7 @@ uint32_t CPDF_CryptoHandler::EncryptGetSize(uint32_t objnum,
   }
   return src_size;
 }
+
 bool CPDF_CryptoHandler::EncryptContent(uint32_t objnum,
                                         uint32_t gennum,
                                         const uint8_t* src_buf,
@@ -320,14 +326,11 @@ bool CPDF_CryptoHandler::EncryptContent(uint32_t objnum,
   CryptBlock(true, objnum, gennum, src_buf, src_size, dest_buf, dest_size);
   return true;
 }
-CPDF_CryptoHandler::CPDF_CryptoHandler() {
-  m_pAESContext = nullptr;
-  m_Cipher = FXCIPHER_NONE;
-  m_KeyLen = 0;
-}
-CPDF_CryptoHandler::~CPDF_CryptoHandler() {
-  FX_Free(m_pAESContext);
-}
+
+CPDF_CryptoHandler::CPDF_CryptoHandler()
+    : m_KeyLen(0), m_Cipher(FXCIPHER_NONE) {}
+
+CPDF_CryptoHandler::~CPDF_CryptoHandler() {}
 
 void CPDF_CryptoHandler::PopulateKey(uint32_t objnum,
                                      uint32_t gennum,
diff --git a/core/fpdfapi/parser/cpdf_crypto_handler.h b/core/fpdfapi/parser/cpdf_crypto_handler.h
index 24caff7081..9d76e67d96 100644
--- a/core/fpdfapi/parser/cpdf_crypto_handler.h
+++ b/core/fpdfapi/parser/cpdf_crypto_handler.h
@@ -7,8 +7,12 @@
 #ifndef CORE_FPDFAPI_PARSER_CPDF_CRYPTO_HANDLER_H_
 #define CORE_FPDFAPI_PARSER_CPDF_CRYPTO_HANDLER_H_
 
+#include <memory>
+
+#include "core/fdrm/crypto/fx_crypt.h"
 #include "core/fxcrt/cfx_retain_ptr.h"
 #include "core/fxcrt/fx_basic.h"
+#include "core/fxcrt/fx_memory.h"
 #include "core/fxcrt/fx_string.h"
 #include "core/fxcrt/fx_system.h"
 
@@ -68,7 +72,7 @@ class CPDF_CryptoHandler : public CFX_Retainable {
   uint8_t m_EncryptKey[32];
   int m_KeyLen;
   int m_Cipher;
-  uint8_t* m_pAESContext;
+  std::unique_ptr<CRYPT_aes_context, FxFreeDeleter> m_pAESContext;
 };
 
 #endif  // CORE_FPDFAPI_PARSER_CPDF_CRYPTO_HANDLER_H_
diff --git a/core/fpdfapi/parser/cpdf_security_handler.cpp b/core/fpdfapi/parser/cpdf_security_handler.cpp
index 4ea5da3497..85a3805eea 100644
--- a/core/fpdfapi/parser/cpdf_security_handler.cpp
+++ b/core/fpdfapi/parser/cpdf_security_handler.cpp
@@ -230,7 +230,7 @@ void Revision6_Hash(const uint8_t* password,
                     const uint8_t* salt,
                     const uint8_t* vector,
                     uint8_t* hash) {
-  CRYPT_sha256_context sha;
+  CRYPT_sha2_context sha;
   CRYPT_SHA256Start(&sha);
   CRYPT_SHA256Update(&sha, password, size);
   CRYPT_SHA256Update(&sha, salt, 8);
@@ -249,7 +249,8 @@ void Revision6_Hash(const uint8_t* password,
   CFX_ByteTextBuf interDigest;
   int i = 0;
   int iBlockSize = 32;
-  uint8_t* aes = FX_Alloc(uint8_t, 2048);
+  CRYPT_aes_context aes;
+  memset(&aes, 0, sizeof(aes));
   while (i < 64 || i < E[iBufLen - 1] + 32) {
     int iRoundSize = size + iBlockSize;
     if (vector) {
@@ -266,9 +267,9 @@ void Revision6_Hash(const uint8_t* password,
         content.AppendBlock(vector, 48);
       }
     }
-    CRYPT_AESSetKey(aes, 16, key, 16, true);
-    CRYPT_AESSetIV(aes, iv);
-    CRYPT_AESEncrypt(aes, E, content.GetBuffer(), iBufLen);
+    CRYPT_AESSetKey(&aes, 16, key, 16, true);
+    CRYPT_AESSetIV(&aes, iv);
+    CRYPT_AESEncrypt(&aes, E, content.GetBuffer(), iBufLen);
     int iHash = 0;
     switch (BigOrder64BitsMod3(E)) {
       case 0:
@@ -297,7 +298,6 @@ void Revision6_Hash(const uint8_t* password,
     iv = input + 16;
     ++i;
   }
-  FX_Free(aes);
   if (hash) {
     memcpy(hash, input, 32);
   }
@@ -319,7 +319,7 @@ bool CPDF_SecurityHandler::AES256_CheckPassword(const uint8_t* password,
     return false;
 
   const uint8_t* pkey = bOwner ? okey.raw_str() : ukey.raw_str();
-  CRYPT_sha256_context sha;
+  CRYPT_sha2_context sha;
   uint8_t digest[32];
   if (m_Revision >= 6) {
     Revision6_Hash(password, size, (const uint8_t*)pkey + 32,
@@ -357,14 +357,15 @@ bool CPDF_SecurityHandler::AES256_CheckPassword(const uint8_t* password,
   if (ekey.GetLength() < 32)
     return false;
 
-  std::vector<uint8_t> aes(2048);
-  CRYPT_AESSetKey(aes.data(), 16, digest, 32, false);
+  CRYPT_aes_context aes;
+  memset(&aes, 0, sizeof(aes));
+  CRYPT_AESSetKey(&aes, 16, digest, 32, false);
   uint8_t iv[16];
   memset(iv, 0, 16);
-  CRYPT_AESSetIV(aes.data(), iv);
-  CRYPT_AESDecrypt(aes.data(), key, ekey.raw_str(), 32);
-  CRYPT_AESSetKey(aes.data(), 16, key, 32, false);
-  CRYPT_AESSetIV(aes.data(), iv);
+  CRYPT_AESSetIV(&aes, iv);
+  CRYPT_AESDecrypt(&aes, key, ekey.raw_str(), 32);
+  CRYPT_AESSetKey(&aes, 16, key, 32, false);
+  CRYPT_AESSetIV(&aes, iv);
   CFX_ByteString perms = m_pEncryptDict->GetStringFor("Perms");
   if (perms.IsEmpty())
     return false;
@@ -375,7 +376,7 @@ bool CPDF_SecurityHandler::AES256_CheckPassword(const uint8_t* password,
       std::min(sizeof(perms_buf), static_cast<size_t>(perms.GetLength()));
   memcpy(perms_buf, perms.raw_str(), copy_len);
   uint8_t buf[16];
-  CRYPT_AESDecrypt(aes.data(), buf, perms_buf, 16);
+  CRYPT_AESDecrypt(&aes, buf, perms_buf, 16);
   if (buf[9] != 'a' || buf[10] != 'd' || buf[11] != 'b')
     return false;
 
@@ -532,7 +533,7 @@ void CPDF_SecurityHandler::OnCreate(CPDF_Dictionary* pEncryptDict,
   }
   if (m_Revision >= 5) {
     int t = (int)time(nullptr);
-    CRYPT_sha256_context sha;
+    CRYPT_sha2_context sha;
     CRYPT_SHA256Start(&sha);
     CRYPT_SHA256Update(&sha, (uint8_t*)&t, sizeof t);
     CRYPT_SHA256Update(&sha, m_EncryptKey, 32);
@@ -640,7 +641,7 @@ void CPDF_SecurityHandler::AES256_SetPassword(CPDF_Dictionary* pEncryptDict,
   CRYPT_SHA1Finish(&sha, digest);
 
   CFX_ByteString ukey = pEncryptDict->GetStringFor("U");
-  CRYPT_sha256_context sha2;
+  CRYPT_sha2_context sha2;
   uint8_t digest1[48];
   if (m_Revision >= 6) {
     Revision6_Hash(password, size, digest, bOwner ? ukey.raw_str() : nullptr,
@@ -669,13 +670,13 @@ void CPDF_SecurityHandler::AES256_SetPassword(CPDF_Dictionary* pEncryptDict,
     }
     CRYPT_SHA256Finish(&sha2, digest1);
   }
-  uint8_t* aes = FX_Alloc(uint8_t, 2048);
-  CRYPT_AESSetKey(aes, 16, digest1, 32, true);
+  CRYPT_aes_context aes;
+  memset(&aes, 0, sizeof(aes));
+  CRYPT_AESSetKey(&aes, 16, digest1, 32, true);
   uint8_t iv[16];
   memset(iv, 0, 16);
-  CRYPT_AESSetIV(aes, iv);
-  CRYPT_AESEncrypt(aes, digest1, key, 32);
-  FX_Free(aes);
+  CRYPT_AESSetIV(&aes, iv);
+  CRYPT_AESEncrypt(&aes, digest1, key, 32);
   pEncryptDict->SetNewFor<CPDF_String>(bOwner ? "OE" : "UE",
                                        CFX_ByteString(digest1, 32), false);
 }
@@ -697,13 +698,17 @@ void CPDF_SecurityHandler::AES256_SetPerms(CPDF_Dictionary* pEncryptDict,
   buf[9] = 'a';
   buf[10] = 'd';
   buf[11] = 'b';
-  uint8_t* aes = FX_Alloc(uint8_t, 2048);
-  CRYPT_AESSetKey(aes, 16, key, 32, true);
-  uint8_t iv[16], buf1[16];
+
+  CRYPT_aes_context aes;
+  memset(&aes, 0, sizeof(aes));
+  CRYPT_AESSetKey(&aes, 16, key, 32, true);
+
+  uint8_t iv[16];
   memset(iv, 0, 16);
-  CRYPT_AESSetIV(aes, iv);
-  CRYPT_AESEncrypt(aes, buf1, buf, 16);
-  FX_Free(aes);
+  CRYPT_AESSetIV(&aes, iv);
+
+  uint8_t buf1[16];
+  CRYPT_AESEncrypt(&aes, buf1, buf, 16);
   pEncryptDict->SetNewFor<CPDF_String>("Perms", CFX_ByteString(buf1, 16),
                                        false);
 }