Add support for AESv3 encryption from SumatraPDF.

6 files changed, 320 insertions, 17 deletions

diff --git a/Makefile b/Makefile
index 19ca9adf..7d4254df 100644
--- a/Makefile
+++ b/Makefile
@@ -85,6 +85,7 @@ FITZ_SRC := \
 	fitz/crypt_aes.c \
 	fitz/crypt_arc4.c \
 	fitz/crypt_md5.c \
+	fitz/crypt_sha2.c \
 	fitz/dev_bbox.c \
 	fitz/dev_draw.c \
 	fitz/dev_list.c \
diff --git a/fitz/crypt_sha2.c b/fitz/crypt_sha2.c
new file mode 100644
index 00000000..6796c5ab
--- /dev/null
+++ b/fitz/crypt_sha2.c
@@ -0,0 +1,182 @@
+/*
+This code is based on the code found from 7-Zip, which has a modified
+version of the SHA-256 found from Crypto++ <http://www.cryptopp.com/>.
+The code was modified a little to fit into liblzma and fitz.
+
+This file has been put into the public domain.
+You can do whatever you want with this file.
+*/
+
+#include "fitz.h"
+
+static inline int isbigendian(void)
+{
+	static const int one = 1;
+	return *(char*)&one == 0;
+}
+
+static inline unsigned int bswap32(unsigned int num)
+{
+	if (!isbigendian())
+	{
+		return	( (((num) << 24))
+			| (((num) << 8) & 0x00FF0000)
+			| (((num) >> 8) & 0x0000FF00)
+			| (((num) >> 24)) );
+	}
+	return num;
+}
+
+/* At least on x86, GCC is able to optimize this to a rotate instruction. */
+#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount)))
+
+#define blk0(i) (W[i] = data[i])
+#define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \
+		+ s0(W[(i - 15) & 15]))
+
+#define Ch(x, y, z) (z ^ (x & (y ^ z)))
+#define Maj(x, y, z) ((x & y) | (z & (x | y)))
+
+#define a(i) T[(0 - i) & 7]
+#define b(i) T[(1 - i) & 7]
+#define c(i) T[(2 - i) & 7]
+#define d(i) T[(3 - i) & 7]
+#define e(i) T[(4 - i) & 7]
+#define f(i) T[(5 - i) & 7]
+#define g(i) T[(6 - i) & 7]
+#define h(i) T[(7 - i) & 7]
+
+#define R(i) \
+	h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \
+		+ (j ? blk2(i) : blk0(i)); \
+	d(i) += h(i); \
+	h(i) += S0(a(i)) + Maj(a(i), b(i), c(i))
+
+#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22))
+#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25))
+#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3))
+#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10))
+
+static const unsigned int SHA256_K[64] = {
+	0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
+	0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
+	0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
+	0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
+	0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
+	0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
+	0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
+	0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
+	0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
+	0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
+	0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
+	0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
+	0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
+	0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
+	0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
+	0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
+};
+
+static void
+transform(unsigned int state[8], const unsigned int data_xe[16])
+{
+	unsigned int data[16];
+	unsigned int W[16];
+	unsigned int T[8];
+	unsigned int j;
+
+	/* ensure big-endian integers */
+	for (j = 0; j < 16; j++)
+		data[j] = bswap32(data_xe[j]);
+
+	/* Copy state[] to working vars. */
+	memcpy(T, state, sizeof(T));
+
+	/* 64 operations, partially loop unrolled */
+	for (j = 0; j < 64; j += 16) {
+		R( 0); R( 1); R( 2); R( 3);
+		R( 4); R( 5); R( 6); R( 7);
+		R( 8); R( 9); R(10); R(11);
+		R(12); R(13); R(14); R(15);
+	}
+
+	/* Add the working vars back into state[]. */
+	state[0] += a(0);
+	state[1] += b(0);
+	state[2] += c(0);
+	state[3] += d(0);
+	state[4] += e(0);
+	state[5] += f(0);
+	state[6] += g(0);
+	state[7] += h(0);
+}
+
+void fz_sha256init(fz_sha256 *context)
+{
+	context->count[0] = context->count[1] = 0;
+
+	context->state[0] = 0x6A09E667;
+	context->state[1] = 0xBB67AE85;
+	context->state[2] = 0x3C6EF372;
+	context->state[3] = 0xA54FF53A;
+	context->state[4] = 0x510E527F;
+	context->state[5] = 0x9B05688C;
+	context->state[6] = 0x1F83D9AB;
+	context->state[7] = 0x5BE0CD19;
+}
+
+void fz_sha256update(fz_sha256 *context, const unsigned char *input, unsigned int inlen)
+{
+	/* Copy the input data into a properly aligned temporary buffer.
+	 * This way we can be called with arbitrarily sized buffers
+	 * (no need to be multiple of 64 bytes), and the code works also
+	 * on architectures that don't allow unaligned memory access. */
+	while (inlen > 0)
+	{
+		const unsigned int copy_start = context->count[0] & 0x3F;
+		unsigned int copy_size = 64 - copy_start;
+		if (copy_size > inlen)
+			copy_size = inlen;
+
+		memcpy(context->buffer.u8 + copy_start, input, copy_size);
+
+		input += copy_size;
+		inlen -= copy_size;
+		context->count[0] += copy_size;
+		/* carry overflow from low to high */
+		if (context->count[0] < copy_size)
+			context->count[1]++;
+
+		if ((context->count[0] & 0x3F) == 0)
+			transform(context->state, context->buffer.u32);
+	}
+}
+
+void fz_sha256final(fz_sha256 *context, unsigned char digest[32])
+{
+	/* Add padding as described in RFC 3174 (it describes SHA-1 but
+	 * the same padding style is used for SHA-256 too). */
+	unsigned int j = context->count[0] & 0x3F;
+	context->buffer.u8[j++] = 0x80;
+
+	while (j != 56)
+	{
+		if (j == 64)
+		{
+			transform(context->state, context->buffer.u32);
+			j = 0;
+		}
+		context->buffer.u8[j++] = 0x00;
+	}
+
+	/* Convert the message size from bytes to bits. */
+	context->count[1] = (context->count[1] << 3) + (context->count[0] >> 29);
+	context->count[0] = context->count[0] << 3;
+
+	context->buffer.u32[14] = bswap32(context->count[1]);
+	context->buffer.u32[15] = bswap32(context->count[0]);
+	transform(context->state, context->buffer.u32);
+
+	for (j = 0; j < 8; j++)
+		((unsigned int *)digest)[j] = bswap32(context->state[j]);
+	memset(context, 0, sizeof(fz_sha256));
+}
diff --git a/fitz/fitz.h b/fitz/fitz.h
index 8f756895..6a339303 100644
--- a/fitz/fitz.h
+++ b/fitz/fitz.h
@@ -285,6 +285,24 @@ void fz_md5init(fz_md5 *state);
 void fz_md5update(fz_md5 *state, const unsigned char *input, const unsigned inlen);
 void fz_md5final(fz_md5 *state, unsigned char digest[16]);
 
+/* sha-256 digests */
+
+typedef struct fz_sha256_s fz_sha256;
+
+struct fz_sha256_s
+{
+	unsigned int state[8];
+	unsigned int count[2];
+	union {
+		unsigned char u8[64];
+		unsigned int u32[16];
+	} buffer;
+};
+
+void fz_sha256init(fz_sha256 *state);
+void fz_sha256update(fz_sha256 *state, const unsigned char *input, unsigned int inlen);
+void fz_sha256final(fz_sha256 *state, unsigned char digest[32]);
+
 /* arc4 crypto */
 
 typedef struct fz_arc4_s fz_arc4;
diff --git a/mupdf/mupdf.h b/mupdf/mupdf.h
index e53ea90a..4bbea33b 100644
--- a/mupdf/mupdf.h
+++ b/mupdf/mupdf.h
@@ -70,6 +70,7 @@ enum
 	PDF_CRYPT_NONE,
 	PDF_CRYPT_RC4,
 	PDF_CRYPT_AESV2,
+	PDF_CRYPT_AESV3,
 	PDF_CRYPT_UNKNOWN,
 };
 
@@ -77,7 +78,6 @@ struct pdf_cryptfilter_s
 {
 	int method;
 	int length;
-	unsigned char key[16];
 };
 
 struct pdf_crypt_s
@@ -92,8 +92,10 @@ struct pdf_crypt_s
 	pdf_cryptfilter strf;
 
 	int r;
-	unsigned char o[32];
-	unsigned char u[32];
+	unsigned char o[48];
+	unsigned char u[48];
+	unsigned char oe[32];
+	unsigned char ue[32];
 	int p;
 	int encryptmetadata;
 
diff --git a/mupdf/pdf_crypt.c b/mupdf/pdf_crypt.c
index 019f889c..25cb1d8d 100644
--- a/mupdf/pdf_crypt.c
+++ b/mupdf/pdf_crypt.c
@@ -35,7 +35,7 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 	obj = fz_dictgets(dict, "V");
 	if (fz_isint(obj))
 		crypt->v = fz_toint(obj);
-	if (crypt->v != 1 && crypt->v != 2 && crypt->v != 4)
+	if (crypt->v != 1 && crypt->v != 2 && crypt->v != 4 && crypt->v != 5)
 	{
 		pdf_freecrypt(crypt);
 		return fz_throw("unknown encryption version");
@@ -64,6 +64,9 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 		}
 	}
 
+	if (crypt->v == 5)
+		crypt->length = 256;
+
 	if (crypt->v == 1 || crypt->v == 2)
 	{
 		crypt->stmf.method = PDF_CRYPT_RC4;
@@ -73,7 +76,7 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 		crypt->strf.length = crypt->length;
 	}
 
-	if (crypt->v == 4)
+	if (crypt->v == 4 || crypt->v == 5)
 	{
 		crypt->stmf.method = PDF_CRYPT_NONE;
 		crypt->stmf.length = crypt->length;
@@ -134,6 +137,9 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 	obj = fz_dictgets(dict, "O");
 	if (fz_isstring(obj) && fz_tostrlen(obj) == 32)
 		memcpy(crypt->o, fz_tostrbuf(obj), 32);
+	/* /O and /U are supposed to be 48 bytes long for revision 5, they're often longer, though */
+	else if (crypt->r == 5 && fz_isstring(obj) && fz_tostrlen(obj) >= 48)
+		memcpy(crypt->o, fz_tostrbuf(obj), 48);
 	else
 	{
 		pdf_freecrypt(crypt);
@@ -143,6 +149,8 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 	obj = fz_dictgets(dict, "U");
 	if (fz_isstring(obj) && fz_tostrlen(obj) == 32)
 		memcpy(crypt->u, fz_tostrbuf(obj), 32);
+	else if (fz_isstring(obj) && fz_tostrlen(obj) >= 48 && crypt->r == 5)
+		memcpy(crypt->u, fz_tostrbuf(obj), 48);
 	else
 	{
 		pdf_freecrypt(crypt);
@@ -158,6 +166,25 @@ pdf_newcrypt(pdf_crypt **cryptp, fz_obj *dict, fz_obj *id)
 		return fz_throw("encryption dictionary missing permissions value");
 	}
 
+	if (crypt->r == 5)
+	{
+		obj = fz_dictgets(dict, "OE");
+		if (!fz_isstring(obj) || fz_tostrlen(obj) != 32)
+		{
+			pdf_freecrypt(crypt);
+			return fz_throw("encryption dictionary missing owner encryption key");
+		}
+		memcpy(crypt->oe, fz_tostrbuf(obj), 32);
+
+		obj = fz_dictgets(dict, "UE");
+		if (!fz_isstring(obj) || fz_tostrlen(obj) != 32)
+		{
+			pdf_freecrypt(crypt);
+			return fz_throw("encryption dictionary missing user encryption key");
+		}
+		memcpy(crypt->ue, fz_tostrbuf(obj), 32);
+	}
+
 	crypt->encryptmetadata = 1;
 	obj = fz_dictgets(dict, "EncryptMetadata");
 	if (fz_isbool(obj))
@@ -214,6 +241,8 @@ pdf_parsecryptfilter(pdf_cryptfilter *cf, fz_obj *dict, int defaultlength)
 			cf->method = PDF_CRYPT_RC4;
 		else if (!strcmp(fz_toname(obj), "AESV2"))
 			cf->method = PDF_CRYPT_AESV2;
+		else if (!strcmp(fz_toname(obj), "AESV3"))
+			cf->method = PDF_CRYPT_AESV3;
 		else
 			fz_throw("unknown encryption method: %s", fz_toname(obj));
 	}
@@ -306,6 +335,51 @@ pdf_computeencryptionkey(pdf_crypt *crypt, unsigned char *password, int pwlen, u
 }
 
 /*
+ * Compute an encryption key (PDF 1.7 ExtensionLevel 3 algorithm 3.2a)
+ */
+
+static void
+pdf_computeencryptionkey_r5(pdf_crypt *crypt, unsigned char *password, int pwlen, int ownerkey, unsigned char *validationkey)
+{
+	unsigned char buffer[128 + 8 + 48];
+	fz_sha256 sha256;
+	fz_aes aes;
+
+	/* Step 2 - truncate UTF-8 password to 127 characters */
+
+	if (pwlen > 127)
+		pwlen = 127;
+
+	/* Step 3/4 - test password against owner/user key and compute encryption key */
+
+	memcpy(buffer, password, pwlen);
+	if (ownerkey)
+	{
+		memcpy(buffer + pwlen, crypt->o + 32, 8);
+		memcpy(buffer + pwlen + 8, crypt->u, 48);
+	}
+	else
+		memcpy(buffer + pwlen, crypt->u + 32, 8);
+
+	fz_sha256init(&sha256);
+	fz_sha256update(&sha256, buffer, pwlen + 8 + (ownerkey ? 48 : 0));
+	fz_sha256final(&sha256, validationkey);
+
+	/* Step 3.5/4.5 - compute file encryption key from OE/UE */
+
+	memcpy(buffer + pwlen, crypt->u + 40, 8);
+
+	fz_sha256init(&sha256);
+	fz_sha256update(&sha256, buffer, pwlen + 8);
+	fz_sha256final(&sha256, buffer);
+
+	// clear password buffer and use it as iv
+	memset(buffer + 32, 0, sizeof(buffer) - 32);
+	aes_setkey_dec(&aes, buffer, crypt->length);
+	aes_crypt_cbc(&aes, AES_DECRYPT, 32, buffer + 32, ownerkey ? crypt->oe : crypt->ue, crypt->key);
+}
+
+/*
  * Computing the user password (PDF 1.7 algorithm 3.4 and 3.5)
  * Also save the generated key for decrypting objects and streams in crypt->key.
  */
@@ -322,7 +396,7 @@ pdf_computeuserpassword(pdf_crypt *crypt, unsigned char *password, int pwlen, un
 		fz_arc4encrypt(&arc4, output, padding, 32);
 	}
 
-	if (crypt->r >= 3)
+	if (crypt->r == 3 || crypt->r == 4)
 	{
 		unsigned char xor[32];
 		unsigned char digest[16];
@@ -352,10 +426,16 @@ pdf_computeuserpassword(pdf_crypt *crypt, unsigned char *password, int pwlen, un
 
 		memcpy(output + 16, padding, 16);
 	}
+
+	if (crypt->r == 5)
+	{
+		pdf_computeencryptionkey_r5(crypt, password, pwlen, 0, output);
+	}
 }
 
 /*
- * Authenticating the user password (PDF 1.7 algorithm 3.6)
+ * Authenticating the user password (PDF 1.7 algorithm 3.6
+ * and ExtensionLevel 3 algorithm 3.11)
  * This also has the side effect of saving a key generated
  * from the password for decrypting objects and streams.
  */
@@ -365,15 +445,16 @@ pdf_authenticateuserpassword(pdf_crypt *crypt, unsigned char *password, int pwle
 {
 	unsigned char output[32];
 	pdf_computeuserpassword(crypt, password, pwlen, output);
-	if (crypt->r == 2)
+	if (crypt->r == 2 || crypt->r == 5)
 		return memcmp(output, crypt->u, 32) == 0;
-	if (crypt->r >= 3)
+	if (crypt->r == 3 || crypt->r == 4)
 		return memcmp(output, crypt->u, 16) == 0;
 	return 0;
 }
 
 /*
- * Authenticating the owner password (PDF 1.7 algorithm 3.7)
+ * Authenticating the owner password (PDF 1.7 algorithm 3.7
+ * and ExtensionLevel 3 algorithm 3.12)
  * Generates the user password from the owner password
  * and calls pdf_authenticateuserpassword.
  */
@@ -389,6 +470,15 @@ pdf_authenticateownerpassword(pdf_crypt *crypt, unsigned char *ownerpass, int pw
 	fz_md5 md5;
 	fz_arc4 arc4;
 
+	if (crypt->r == 5)
+	{
+		/* PDF 1.7 ExtensionLevel 3 algorithm 3.12 */
+
+		pdf_computeencryptionkey_r5(crypt, ownerpass, pwlen, 1, key);
+
+		return !memcmp(key, crypt->o, 32);
+	}
+
 	n = crypt->length / 8;
 
 	/* Step 1 -- steps 1 to 4 of PDF 1.7 algorithm 3.3 */
@@ -463,7 +553,7 @@ pdf_needspassword(pdf_xref *xref)
 }
 
 /*
- * PDF 1.7 algorithm 3.1
+ * PDF 1.7 algorithm 3.1 and ExtensionLevel 3 algorithm 3.1a
  *
  * Using the global encryption key that was generated from the
  * password, create a new key that is used to decrypt indivual
@@ -477,6 +567,12 @@ pdf_computeobjectkey(pdf_crypt *crypt, pdf_cryptfilter *cf, int num, int gen, un
 	fz_md5 md5;
 	unsigned char message[5];
 
+	if (cf->method == PDF_CRYPT_AESV3)
+	{
+		memcpy(key, crypt->key, crypt->length / 8);
+		return crypt->length / 8;
+	}
+
 	fz_md5init(&md5);
 	fz_md5update(&md5, crypt->key, crypt->length / 8);
 	message[0] = (num) & 0xFF;
@@ -497,7 +593,7 @@ pdf_computeobjectkey(pdf_crypt *crypt, pdf_cryptfilter *cf, int num, int gen, un
 }
 
 /*
- * PDF 1.7 algorithm 3.1
+ * PDF 1.7 algorithm 3.1 and ExtensionLevel 3 algorithm 3.1a
  *
  * Decrypt all strings in obj modifying the data in-place.
  * Recurse through arrays and dictionaries, but do not follow
@@ -525,7 +621,7 @@ pdf_cryptobjimp(pdf_crypt *crypt, fz_obj *obj, unsigned char *key, int keylen)
 			fz_arc4encrypt(&arc4, s, s, n);
 		}
 
-		if (crypt->strf.method == PDF_CRYPT_AESV2)
+		if (crypt->strf.method == PDF_CRYPT_AESV2 || crypt->strf.method == PDF_CRYPT_AESV3)
 		{
 			if (n >= 32)
 			{
@@ -562,7 +658,7 @@ pdf_cryptobjimp(pdf_crypt *crypt, fz_obj *obj, unsigned char *key, int keylen)
 void
 pdf_cryptobj(pdf_crypt *crypt, fz_obj *obj, int num, int gen)
 {
-	unsigned char key[16];
+	unsigned char key[32];
 	int len;
 
 	len = pdf_computeobjectkey(crypt, &crypt->strf, num, gen, key);
@@ -571,14 +667,14 @@ pdf_cryptobj(pdf_crypt *crypt, fz_obj *obj, int num, int gen)
 }
 
 /*
- * PDF 1.7 algorithm 3.1
+ * PDF 1.7 algorithm 3.1 and ExtensionLevel 3 algorithm 3.1a
  *
  * Create filter suitable for de/encrypting a stream.
  */
 fz_stream *
 pdf_opencrypt(fz_stream *chain, pdf_crypt *crypt, pdf_cryptfilter *stmf, int num, int gen)
 {
-	unsigned char key[16];
+	unsigned char key[32];
 	int len;
 
 	len = pdf_computeobjectkey(crypt, stmf, num, gen, key);
@@ -586,7 +682,7 @@ pdf_opencrypt(fz_stream *chain, pdf_crypt *crypt, pdf_cryptfilter *stmf, int num
 	if (stmf->method == PDF_CRYPT_RC4)
 		return fz_openarc4(chain, key, len);
 
-	if (stmf->method == PDF_CRYPT_AESV2)
+	if (stmf->method == PDF_CRYPT_AESV2 || stmf->method == PDF_CRYPT_AESV3)
 		return fz_openaesd(chain, key, len);
 
 	return fz_opencopy(chain);
diff --git a/win32/mupdf/mupdf.vcproj b/win32/mupdf/mupdf.vcproj
index ac0af2e9..8292a2a8 100644
--- a/win32/mupdf/mupdf.vcproj
+++ b/win32/mupdf/mupdf.vcproj
@@ -325,6 +325,10 @@
 				>

 			</File>

 			<File

+				RelativePath="..\..\fitz\crypt_sha2.c"

+				>

+			</File>

+			<File

 				RelativePath="..\..\fitz\dev_bbox.c"

 				>

 			</File>