Fix 699085: Use at most 16 bytes from MD5 digests.

Previously crypt->len / 8 could be 0..32. In the case
of crypt->len == 256 this meant that when an 16 byte
MD5 was computed, uninitalized key data would be
accessed because 32 bytes of key data was accessed.

Now crypt->len / 8 is limited to 0..16. So when the
16 byte MD5 is computed and later accessed only the
initialized part of the key data is accessed.

Because crypt->len / 8 is now limited to 0..16 the key
data buffer can also be shrunk to 16 bytes without
risking the code stepping over its boundaries.

1 files changed, 11 insertions, 14 deletions

diff --git a/source/pdf/pdf-crypt.c b/source/pdf/pdf-crypt.c
index 7150f66e..2e31f356 100644
--- a/source/pdf/pdf-crypt.c
+++ b/source/pdf/pdf-crypt.c
@@ -353,7 +353,7 @@ pdf_compute_encryption_key(fz_context *ctx, pdf_crypt *crypt, unsigned char *pas
 	int i, n;
 	fz_md5 md5;
 
-	n = crypt->length / 8;
+	n = fz_clampi(crypt->length / 8, 0, 16);
 
 	/* Step 1 - copy and pad password string */
 	if (pwlen > 32)
@@ -569,12 +569,14 @@ pdf_compute_encryption_key_r6(fz_context *ctx, pdf_crypt *crypt, unsigned char *
 static void
 pdf_compute_user_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, size_t pwlen, unsigned char *output)
 {
+	int n = fz_clampi(crypt->length / 8, 0, 16);
+
 	if (crypt->r == 2)
 	{
 		fz_arc4 arc4;
 
 		pdf_compute_encryption_key(ctx, crypt, password, pwlen, crypt->key);
-		fz_arc4_init(&arc4, crypt->key, crypt->length / 8);
+		fz_arc4_init(&arc4, crypt->key, n);
 		fz_arc4_encrypt(&arc4, output, padding, 32);
 	}
 
@@ -584,9 +586,7 @@ pdf_compute_user_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *pass
 		unsigned char digest[16];
 		fz_md5 md5;
 		fz_arc4 arc4;
-		int i, x, n;
-
-		n = crypt->length / 8;
+		int i, x;
 
 		pdf_compute_encryption_key(ctx, crypt, password, pwlen, crypt->key);
 
@@ -649,17 +649,16 @@ pdf_authenticate_user_password(fz_context *ctx, pdf_crypt *crypt, unsigned char
 static int
 pdf_authenticate_owner_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *ownerpass, size_t pwlen)
 {
+	int n = fz_clampi(crypt->length / 8, 0, 16);
+
 	if (crypt->r == 2)
 	{
 		unsigned char pwbuf[32];
-		unsigned char key[32];
+		unsigned char key[16];
 		unsigned char userpass[32];
-		int n;
 		fz_md5 md5;
 		fz_arc4 arc4;
 
-		n = crypt->length / 8;
-
 		if (pwlen > 32)
 			pwlen = 32;
 		memcpy(pwbuf, ownerpass, pwlen);
@@ -678,15 +677,13 @@ pdf_authenticate_owner_password(fz_context *ctx, pdf_crypt *crypt, unsigned char
 	if (crypt->r == 3 || crypt->r == 4)
 	{
 		unsigned char pwbuf[32];
-		unsigned char key[32];
+		unsigned char key[16];
 		unsigned char xor[32];
 		unsigned char userpass[32];
-		int i, n, x;
+		int i, x;
 		fz_md5 md5;
 		fz_arc4 arc4;
 
-		n = crypt->length / 8;
-
 		if (pwlen > 32)
 			pwlen = 32;
 		memcpy(pwbuf, ownerpass, pwlen);
@@ -699,7 +696,7 @@ pdf_authenticate_owner_password(fz_context *ctx, pdf_crypt *crypt, unsigned char
 		for (i = 0; i < 50; i++)
 		{
 			fz_md5_init(&md5);
-			fz_md5_update(&md5, key, 16);
+			fz_md5_update(&md5, key, n);
 			fz_md5_final(&md5, key);
 		}