Improve return codes from pdf_authenticate_password.

Customer request to enable finer control based on which
password authenticates.

3 files changed, 45 insertions, 15 deletions

diff --git a/include/mupdf/fitz/document.h b/include/mupdf/fitz/document.h
index c66dc2ca..90b54b26 100644
--- a/include/mupdf/fitz/document.h
+++ b/include/mupdf/fitz/document.h
@@ -195,6 +195,15 @@ int fz_needs_password(fz_context *ctx, fz_document *doc);
 	specifications do not specify any particular text encoding, so
 	neither do we.
 
+	Returns 0 for failure to authenticate, non-zero for success.
+
+	For PDF documents, further information can be given by examining
+	the bits in the return code.
+
+		Bit 0 => No password required
+		Bit 1 => User password authenticated
+		Bit 2 => Owner password authenticated
+
 	Does not throw exceptions.
 */
 int fz_authenticate_password(fz_context *ctx, fz_document *doc, const char *password);
diff --git a/include/mupdf/pdf/document.h b/include/mupdf/pdf/document.h
index 56d16ebf..a84ee872 100644
--- a/include/mupdf/pdf/document.h
+++ b/include/mupdf/pdf/document.h
@@ -113,6 +113,18 @@ pdf_page *pdf_page_from_fz_page(fz_context *ctx, fz_page *ptr);
 pdf_annot *pdf_annot_from_fz_annot(fz_context *ctx, fz_annot *ptr);
 
 int pdf_needs_password(fz_context *ctx, pdf_document *doc);
+
+/*
+	pdf_authenticate_password: Attempt to authenticate a
+	password.
+
+	Returns 0 for failure, non-zero for success.
+
+	In the non-zero case:
+		bit 0 set => no password required
+		bit 1 set => user password authenticated
+		bit 2 set => owner password authenticated
+*/
 int pdf_authenticate_password(fz_context *ctx, pdf_document *doc, const char *pw);
 
 int pdf_has_permission(fz_context *ctx, pdf_document *doc, fz_permission p);
diff --git a/source/pdf/pdf-crypt.c b/source/pdf/pdf-crypt.c
index 1499e39e..aad7324e 100644
--- a/source/pdf/pdf-crypt.c
+++ b/source/pdf/pdf-crypt.c
@@ -759,25 +759,34 @@ int
 pdf_authenticate_password(fz_context *ctx, pdf_document *doc, const char *pwd_utf8)
 {
 	char password[2048];
+	int auth;
 
-	if (doc->crypt)
+	if (!doc->crypt)
+		return 1; /* No password required */
+
+	password[0] = 0;
+	if (pwd_utf8)
 	{
-		password[0] = 0;
-		if (pwd_utf8)
-		{
-			if (doc->crypt->r <= 4)
-				pdf_docenc_from_utf8(password, pwd_utf8, sizeof password);
-			else
-				pdf_saslprep_from_utf8(password, pwd_utf8, sizeof password);
-		}
+		if (doc->crypt->r <= 4)
+			pdf_docenc_from_utf8(password, pwd_utf8, sizeof password);
+		else
+			pdf_saslprep_from_utf8(password, pwd_utf8, sizeof password);
+	}
 
-		if (pdf_authenticate_user_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
-			return 1;
-		if (pdf_authenticate_owner_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
-			return 1;
-		return 0;
+	auth = 0;
+	if (pdf_authenticate_user_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
+		auth = 2;
+	if (pdf_authenticate_owner_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
+		auth |= 4;
+	else if (auth & 2)
+	{
+		/* We need to reauthenticate the user password,
+		 * because the failed attempt to authenticate
+		 * the owner password will have invalidated the
+		 * stored keys. */
+		(void)pdf_authenticate_user_password(ctx, doc->crypt, (unsigned char *)password, strlen(password));
 	}
-	return 1;
+	return auth;
 }
 
 int