move the profxjc things out of pdf_new_crypt

1 files changed, 166 insertions, 162 deletions

diff --git a/source/pdf/pdf-crypt.c b/source/pdf/pdf-crypt.c
index 566a80d5..d0ec5055 100644
--- a/source/pdf/pdf-crypt.c
+++ b/source/pdf/pdf-crypt.c
@@ -169,6 +169,119 @@ struct pdf_crypt_s
 
 static void pdf_parse_crypt_filter(fz_context *ctx, pdf_crypt_filter *cf, pdf_crypt *crypt, pdf_obj *name);
 
+static pdf_crypt *
+pdf_new_profxjc_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
+{
+	fz_sha256 sha256;
+	fz_sha256 xml_hash;
+	fz_sha256 final_hash;
+	unsigned char xml_hash_sum[32];
+	unsigned char *r_buf = NULL;
+	size_t r_size = 0;
+	mt19937 mtctx;
+	unsigned char crypt_bytes[4] = { 3, 0, 0, 0 }; /* AESV3 */
+	unsigned char len_bytes[4] = { 0, 0, 0, 0 };
+	unsigned char idstr[32];
+
+	obj = pdf_dict_get(ctx, dict, PDF_NAME(Length));
+	if (pdf_is_int(ctx, obj))
+		crypt->length = pdf_to_int(ctx, obj);
+
+	crypt->stmf.method = PDF_CRYPT_AESV3;
+	crypt->stmf.length = crypt->length;
+	crypt->strf.method = PDF_CRYPT_AESV3;
+	crypt->strf.length = crypt->length;
+	len_bytes[0] = crypt->length / 8;
+
+	fz_sha256_init(&sha256);
+	fz_sha256_update(&sha256, "ProfXJC.General", 15);
+	fz_sha256_update(&sha256, crypt_bytes, 4);
+	fz_sha256_update(&sha256, len_bytes, 4);
+	fz_sha256_update(&sha256, &crypt->p, 4); /* TODO: may not work for big endian */
+
+	if (pdf_is_array(ctx, id) && pdf_array_len(ctx, id) == 2)
+	{
+		const char *buf;
+		size_t osz;
+
+		/* the original id setter */
+		obj = pdf_array_get(ctx, id, 0);
+		if (pdf_is_string(ctx, obj))
+			crypt->id = pdf_keep_obj(ctx, obj);
+
+		/* TODO: what if id is not 16*2 bytes? */
+		obj = pdf_array_get(ctx, id, 0);
+		buf = pdf_to_string(ctx, obj, &osz);
+		if (osz == 16) {
+			memcpy(idstr, buf, 16);
+		} else {
+			fz_warn(ctx, "length of id[0] is %ld", osz);
+		}
+		obj = pdf_array_get(ctx, id, 1);
+		buf = pdf_to_string(ctx, obj, &osz);
+		if (osz == 16) {
+			memcpy(idstr + 16, buf, 16);
+		} else {
+			fz_warn(ctx, "length of id[1] is %ld", osz);
+		}
+	}
+
+	obj = pdf_dict_gets(ctx, dict, "Recipients");
+	if (pdf_is_string(ctx, obj)) {
+		r_size = pdf_to_str_len(ctx, obj);
+
+		r_buf = fz_malloc(ctx, r_size);
+		memcpy(r_buf, pdf_to_str_buf(ctx, obj), r_size);
+		unsigned char tmpbyte = 0xff;
+		int ididx = 0;
+		for (int i = 0; i < r_size; i++) {
+			tmpbyte ^= idstr[ididx];
+			ididx = (ididx + 1) % 32;
+			r_buf[i] ^= tmpbyte;
+		}
+	}
+
+	const char *sseed = strstr(r_buf, "Seed=\"");
+	if (!sseed) {
+		fz_throw(ctx, FZ_ERROR_GENERIC, "Cannot find the seed");
+	}
+	uint32_t seed = strtoll(sseed + 6, NULL, 10);
+	mt19937_init(&mtctx, seed);
+
+	const char *userpass = strstr(r_buf, "<xjc:UserPassword>");
+	if (!userpass) {
+		fz_throw(ctx, FZ_ERROR_GENERIC, "Cannot find UserPassword");
+	}
+	userpass += 18;
+
+	uint32_t tmpword = 0xffffffff;
+	uint32_t pass[0x20];
+	for (int i = 0; i < 0x20; i++) {
+		tmpword ^= mt19937_next(&mtctx);
+		uint32_t dw = hex_to_dword(userpass);
+		userpass += 8;
+		pass[i] = dw ^ tmpword;
+	}
+
+	fz_sha256_update(&sha256, pass, 0x80);
+	fz_sha256_update(&sha256, r_buf, r_size);
+
+	fz_sha256_init(&xml_hash);
+	fz_sha256_update(&xml_hash, r_buf, r_size);
+	fz_sha256_final(&xml_hash, xml_hash_sum);
+
+	fz_sha256_update(&sha256, xml_hash_sum, 32);
+	fz_sha256_final(&sha256, crypt->key);
+
+	for (int i = 0; i < 43; i++) {
+		fz_sha256_init(&final_hash);
+		fz_sha256_update(&final_hash, crypt->key, 32);
+		fz_sha256_final(&final_hash, crypt->key);
+	}
+	fz_free(ctx, r_buf);
+	return crypt;
+}
+
 /*
  * Create crypt object for decrypting strings and streams
  * given the Encryption and ID objects.
@@ -194,7 +307,8 @@ pdf_new_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
 	if (!strcmp("ProfXJC", pdf_to_name(ctx, obj)))
 	{
 		crypt->xjc = 1;
-	} else
+		return pdf_new_profxjc_crypt(ctx, dict, id);
+	}
 	if (!pdf_name_eq(ctx, PDF_NAME(Standard), obj))
 	{
 		pdf_drop_crypt(ctx, crypt);
@@ -205,7 +319,7 @@ pdf_new_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
 	obj = pdf_dict_get(ctx, dict, PDF_NAME(V));
 	if (pdf_is_int(ctx, obj))
 		crypt->v = pdf_to_int(ctx, obj);
-	if (!crypt->xjc && crypt->v != 1 && crypt->v != 2 && crypt->v != 4 && crypt->v != 5)
+	if (crypt->v != 1 && crypt->v != 2 && crypt->v != 4 && crypt->v != 5)
 	{
 		pdf_drop_crypt(ctx, crypt);
 		fz_throw(ctx, FZ_ERROR_GENERIC, "unknown encryption version");
@@ -213,60 +327,58 @@ pdf_new_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
 
 	/* Standard security handler (PDF 1.7 table 3.19) */
 
-	if (!crypt->xjc) {
-		obj = pdf_dict_get(ctx, dict, PDF_NAME(R));
-		if (pdf_is_int(ctx, obj))
-			crypt->r = pdf_to_int(ctx, obj);
-		else if (crypt->v <= 4)
-		{
-			fz_warn(ctx, "encryption dictionary missing revision value, guessing...");
-			if (crypt->v < 2)
-				crypt->r = 2;
-			else if (crypt->v == 2)
-				crypt->r = 3;
-			else if (crypt->v == 4)
-				crypt->r = 4;
-		}
-		else
-		{
-			pdf_drop_crypt(ctx, crypt);
-			fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing version and revision value");
-		}
-		if (crypt->r < 1 || crypt->r > 6)
-		{
-			int r = crypt->r;
-			pdf_drop_crypt(ctx, crypt);
-			fz_throw(ctx, FZ_ERROR_GENERIC, "unknown crypt revision %d", r);
-		}
+	obj = pdf_dict_get(ctx, dict, PDF_NAME(R));
+	if (pdf_is_int(ctx, obj))
+		crypt->r = pdf_to_int(ctx, obj);
+	else if (crypt->v <= 4)
+	{
+		fz_warn(ctx, "encryption dictionary missing revision value, guessing...");
+		if (crypt->v < 2)
+			crypt->r = 2;
+		else if (crypt->v == 2)
+			crypt->r = 3;
+		else if (crypt->v == 4)
+			crypt->r = 4;
+	}
+	else
+	{
+		pdf_drop_crypt(ctx, crypt);
+		fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing version and revision value");
+	}
+	if (crypt->r < 1 || crypt->r > 6)
+	{
+		int r = crypt->r;
+		pdf_drop_crypt(ctx, crypt);
+		fz_throw(ctx, FZ_ERROR_GENERIC, "unknown crypt revision %d", r);
+	}
 
-		obj = pdf_dict_get(ctx, dict, PDF_NAME(O));
-		if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
-			memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 32);
-		/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
-		else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
-			memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 48);
-		else
-		{
-			pdf_drop_crypt(ctx, crypt);
-			fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing owner password");
-		}
+	obj = pdf_dict_get(ctx, dict, PDF_NAME(O));
+	if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
+		memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 32);
+	/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
+	else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
+		memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 48);
+	else
+	{
+		pdf_drop_crypt(ctx, crypt);
+		fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing owner password");
+	}
 
-		obj = pdf_dict_get(ctx, dict, PDF_NAME(U));
-		if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
-			memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 32);
-		/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
-		else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
-			memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 48);
-		else if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) < 32)
-		{
-			fz_warn(ctx, "encryption password key too short (%d)", pdf_to_str_len(ctx, obj));
-			memcpy(crypt->u, pdf_to_str_buf(ctx, obj), pdf_to_str_len(ctx, obj));
-		}
-		else
-		{
-			pdf_drop_crypt(ctx, crypt);
-			fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing user password");
-		}
+	obj = pdf_dict_get(ctx, dict, PDF_NAME(U));
+	if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
+		memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 32);
+	/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
+	else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
+		memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 48);
+	else if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) < 32)
+	{
+		fz_warn(ctx, "encryption password key too short (%d)", pdf_to_str_len(ctx, obj));
+		memcpy(crypt->u, pdf_to_str_buf(ctx, obj), pdf_to_str_len(ctx, obj));
+	}
+	else
+	{
+		pdf_drop_crypt(ctx, crypt);
+		fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing user password");
 	}
 
 	obj = pdf_dict_get(ctx, dict, PDF_NAME(P));
@@ -389,114 +501,6 @@ pdf_new_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
 			crypt->length = crypt->stmf.length;
 	}
 
-	if (crypt->xjc) {
-		fz_sha256 sha256;
-		fz_sha256 xml_hash;
-		fz_sha256 final_hash;
-		unsigned char xml_hash_sum[32];
-		unsigned char *r_buf = NULL;
-		size_t r_size = 0;
-		mt19937 mtctx;
-
-		obj = pdf_dict_get(ctx, dict, PDF_NAME(Length));
-		if (pdf_is_int(ctx, obj))
-			crypt->length = pdf_to_int(ctx, obj);
-
-		crypt->stmf.method = PDF_CRYPT_AESV3;
-		crypt->stmf.length = crypt->length;
-		crypt->strf.method = PDF_CRYPT_AESV3;
-		crypt->strf.length = crypt->length;
-
-		fz_warn(ctx, "crypt method: %d, crypt length: %d, crypt perm: %x",
-				crypt->stmf.method, crypt->length, crypt->p);
-
-		fz_sha256_init(&sha256);
-		fz_sha256_update(&sha256, "ProfXJC.General", 15);
-		fz_sha256_update(&sha256, &crypt->stmf.method, 4);
-		unsigned char lenbytes[4] = { crypt->length / 8, 0, 0, 0 };
-		fz_sha256_update(&sha256, lenbytes, 4);
-		fz_sha256_update(&sha256, &crypt->p, 4);
-
-		obj = pdf_dict_gets(ctx, dict, "Recipients");
-		if (pdf_is_string(ctx, obj)) {
-			unsigned char idstr[32];
-			r_size = pdf_to_str_len(ctx, obj);
-
-			if (pdf_is_array(ctx, id) && pdf_array_len(ctx, id) == 2)
-			{
-				crypt->id = id;
-			}
-			if (pdf_is_array(ctx, crypt->id)) {
-				pdf_obj *oid;
-				size_t osz;
-				const char *buf;
-
-				oid = pdf_array_get(ctx, crypt->id, 0);
-				buf = pdf_to_string(ctx, oid, &osz);
-				if (osz == 16) {
-					memcpy(idstr, buf, 16);
-				} else {
-					fz_warn(ctx, "length of id[0] is %ld", osz);
-				}
-				oid = pdf_array_get(ctx, crypt->id, 1);
-				buf = pdf_to_string(ctx, oid, &osz);
-				if (osz == 16) {
-					memcpy(idstr + 16, buf, 16);
-				} else {
-					fz_warn(ctx, "length of id[1] is %ld", osz);
-				}
-			}
-			r_buf = fz_malloc(ctx, r_size);
-			memcpy(r_buf, pdf_to_str_buf(ctx, obj), r_size);
-			unsigned char tmpbyte = 0xff;
-			int ididx = 0;
-			for (int i = 0; i < r_size; i++) {
-				tmpbyte ^= idstr[ididx];
-				ididx = (ididx + 1) % 32;
-				r_buf[i] ^= tmpbyte;
-			}
-		}
-
-		const char *sseed = strstr(r_buf, "Seed=\"");
-		if (!sseed) {
-			fz_throw(ctx, FZ_ERROR_GENERIC, "Cannot find the seed");
-		}
-		uint32_t seed = strtoll(sseed + 6, NULL, 10);
-		mt19937_init(&mtctx, seed);
-
-		const char *userpass = strstr(r_buf, "<xjc:UserPassword>");
-		if (!userpass) {
-			fz_throw(ctx, FZ_ERROR_GENERIC, "Cannot find UserPassword");
-		}
-		userpass += 18;
-
-		uint32_t tmpword = 0xffffffff;
-		uint32_t pass[0x20];
-		for (int i = 0; i < 0x20; i++) {
-			tmpword ^= mt19937_next(&mtctx);
-			uint32_t dw = hex_to_dword(userpass);
-			userpass += 8;
-			pass[i] = dw ^ tmpword;
-		}
-
-		fz_sha256_update(&sha256, pass, 0x80);
-		fz_sha256_update(&sha256, r_buf, r_size);
-
-		fz_sha256_init(&xml_hash);
-		fz_sha256_update(&xml_hash, r_buf, r_size);
-		fz_sha256_final(&xml_hash, xml_hash_sum);
-
-		fz_sha256_update(&sha256, xml_hash_sum, 32);
-		fz_sha256_final(&sha256, crypt->key);
-
-		for (int i = 0; i < 43; i++) {
-			fz_sha256_init(&final_hash);
-			fz_sha256_update(&final_hash, crypt->key, 32);
-			fz_sha256_final(&final_hash, crypt->key);
-		}
-		fz_free(ctx, r_buf);
-		return crypt;
-	}
 	return crypt;
 }