Avoid use of fixed size buffers for the digest when document signing

The signer object now has an extra method that informs the caller of
the maximum size the digest might be. This is used to allocate space for
the digest within the file and to size some of the buffers used in the
code.

The openssl-based inplementation of the signer object has been updated to
perform a test digest generation so as to find the size needed. We
believe that the digest size is indendent of the hashed data.

1 files changed, 23 insertions, 7 deletions

diff --git a/source/helpers/pkcs7/pkcs7-openssl.c b/source/helpers/pkcs7/pkcs7-openssl.c
index 1468b7e1..58dbf65f 100644
--- a/source/helpers/pkcs7/pkcs7-openssl.c
+++ b/source/helpers/pkcs7/pkcs7-openssl.c
@@ -633,10 +633,12 @@ static int signer_create_digest(pdf_pkcs7_signer *signer, fz_stream *in, unsigne
 	unsigned char *p7_ptr;
 	int p7_len;
 
-	bdata = BIO_new_stream(ctx, in);
-	if (bdata == NULL)
-		goto exit;
-
+	if (in != NULL)
+	{
+		bdata = BIO_new_stream(ctx, in);
+		if (bdata == NULL)
+			goto exit;
+	}
 
 	p7 = PKCS7_new();
 	if (p7 == NULL)
@@ -657,7 +659,7 @@ static int signer_create_digest(pdf_pkcs7_signer *signer, fz_stream *in, unsigne
 	if (bp7in == NULL)
 		goto exit;
 
-	while(1)
+	while(bdata) /* bdata knowingly not changed in the loop */
 	{
 		char buf[4096];
 		int n = BIO_read(bdata, buf, sizeof(buf));
@@ -677,10 +679,12 @@ static int signer_create_digest(pdf_pkcs7_signer *signer, fz_stream *in, unsigne
 		goto exit;
 
 	p7_len = BIO_get_mem_data(bp7, &p7_ptr);
-	if (p7_len > *digest_len)
+	if (digest && p7_len > *digest_len)
 		goto exit;
 
-	memcpy(digest, p7_ptr, p7_len);
+	if (digest)
+		memcpy(digest, p7_ptr, p7_len);
+
 	*digest_len = p7_len;
 	res = 1;
 
@@ -692,6 +696,17 @@ exit:
 	return res;
 }
 
+static int max_digest_size(pdf_pkcs7_signer *signer)
+{
+	/* Perform a test digest generation to find the required size. Size
+	 * is assumed independent of data being hashed */
+	int digest_len = 0;
+
+	signer_create_digest(signer, NULL, NULL, &digest_len);
+
+	return digest_len;
+}
+
 pdf_pkcs7_signer *pkcs7_openssl_read_pfx(fz_context *ctx, const char *pfile, const char *pw)
 {
 	BIO *pfxbio = NULL;
@@ -710,6 +725,7 @@ pdf_pkcs7_signer *pkcs7_openssl_read_pfx(fz_context *ctx, const char *pfile, con
 		signer->base.drop = drop_signer;
 		signer->base.designated_name = signer_designated_name;
 		signer->base.drop_designated_name = signer_drop_designated_name;
+		signer->base.max_digest_size = max_digest_size;
 		signer->base.create_digest = signer_create_digest;
 		signer->ctx = ctx;
 		signer->refs = 1;