Bug 696803: Fix pam output.

PAM output was broken in the absence of alpha. Fixed now.

3 files changed, 41 insertions, 33 deletions

diff --git a/include/mupdf/fitz/output-pnm.h b/include/mupdf/fitz/output-pnm.h
index 42259780..ec6cdbc9 100644
--- a/include/mupdf/fitz/output-pnm.h
+++ b/include/mupdf/fitz/output-pnm.h
@@ -13,8 +13,8 @@
 void fz_save_pixmap_as_pnm(fz_context *ctx, fz_pixmap *pixmap, char *filename);
 
 void fz_write_pixmap_as_pnm(fz_context *ctx, fz_output *out, fz_pixmap *pixmap);
-void fz_write_pnm_header(fz_context *ctx, fz_output *out, int w, int h, int n);
-void fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stride, int band, int bandheight, unsigned char *p);
+void fz_write_pnm_header(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha);
+void fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int stride, int band, int bandheight, unsigned char *p);
 
 /*
 	fz_save_pixmap_as_pam: Save a pixmap as a PAM image file.
@@ -22,8 +22,8 @@ void fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int
 void fz_save_pixmap_as_pam(fz_context *ctx, fz_pixmap *pixmap, char *filename, int savealpha);
 
 void fz_write_pixmap_as_pam(fz_context *ctx, fz_output *out, fz_pixmap *pixmap, int savealpha);
-void fz_write_pam_header(fz_context *ctx, fz_output *out, int w, int h, int n, int savealpha);
-void fz_write_pam_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stride, int band, int bandheight, unsigned char *sp, int savealpha);
+void fz_write_pam_header(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int savealpha);
+void fz_write_pam_band(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int stride, int band, int bandheight, unsigned char *sp, int savealpha);
 
 /*
 	fz_save_bitmap_as_pbm: Save a bitmap as a PBM image file.
diff --git a/source/fitz/pixmap.c b/source/fitz/pixmap.c
index 1d985fa9..e2564c98 100644
--- a/source/fitz/pixmap.c
+++ b/source/fitz/pixmap.c
@@ -845,27 +845,31 @@ fz_gamma_pixmap(fz_context *ctx, fz_pixmap *pix, float gamma)
  */
 
 void
-fz_write_pnm_header(fz_context *ctx, fz_output *out, int w, int h, int n)
+fz_write_pnm_header(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha)
 {
-	if (n < 1 || n > 4)
+	n -= alpha;
+	if (n != 1 && n != 3)
 		fz_throw(ctx, FZ_ERROR_GENERIC, "pixmap must be grayscale or rgb to write as pnm");
 
-	if (n == 1 || n == 2)
+	if (n == 1)
 		fz_printf(ctx, out, "P5\n");
-	if (n == 3 || n == 4)
+	if (n == 3)
 		fz_printf(ctx, out, "P6\n");
 	fz_printf(ctx, out, "%d %d\n", w, h);
 	fz_printf(ctx, out, "255\n");
 }
 
 void
-fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stride, int band, int bandheight, unsigned char *p)
+fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int stride, int band, int bandheight, unsigned char *p)
 {
 	char buffer[2*3*4*5*6]; /* Buffer must be a multiple of 2 and 3 at least. */
 	int len;
 	int start = band * bandheight;
 	int end = start + bandheight;
 
+	if (n-alpha != 1 && n-alpha != 3)
+		fz_throw(ctx, FZ_ERROR_GENERIC, "pixmap must be grayscale or rgb to write as pnm");
+
 	if (!out)
 		return;
 
@@ -936,16 +940,16 @@ fz_write_pnm_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stri
 void
 fz_write_pixmap_as_pnm(fz_context *ctx, fz_output *out, fz_pixmap *pixmap)
 {
-	fz_write_pnm_header(ctx, out, pixmap->w, pixmap->h, pixmap->n);
-	fz_write_pnm_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->stride, 0, pixmap->h, pixmap->samples);
+	fz_write_pnm_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha);
+	fz_write_pnm_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, pixmap->stride, 0, pixmap->h, pixmap->samples);
 }
 
 void
 fz_save_pixmap_as_pnm(fz_context *ctx, fz_pixmap *pixmap, char *filename)
 {
 	fz_output *out = fz_new_output_with_path(ctx, filename, 0);
-	fz_write_pnm_header(ctx, out, pixmap->w, pixmap->h, pixmap->n);
-	fz_write_pnm_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->stride, 0, pixmap->h, pixmap->samples);
+	fz_write_pnm_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha);
+	fz_write_pnm_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, pixmap->stride, 0, pixmap->h, pixmap->samples);
 	fz_drop_output(ctx, out);
 }
 
@@ -954,12 +958,14 @@ fz_save_pixmap_as_pnm(fz_context *ctx, fz_pixmap *pixmap, char *filename)
  */
 
 void
-fz_write_pam_header(fz_context *ctx, fz_output *out, int w, int h, int n, int savealpha)
+fz_write_pam_header(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int savealpha)
 {
 	int sn = n;
-	int dn = n;
-	if (!savealpha && dn > 1)
-		dn--;
+	int dn = n - alpha;
+	if (!alpha)
+		savealpha = 0;
+
+	dn += savealpha;
 
 	fz_printf(ctx, out, "P7\n");
 	fz_printf(ctx, out, "WIDTH %d\n", w);
@@ -968,23 +974,25 @@ fz_write_pam_header(fz_context *ctx, fz_output *out, int w, int h, int n, int sa
 	fz_printf(ctx, out, "MAXVAL 255\n");
 	if (dn == 1) fz_printf(ctx, out, "TUPLTYPE GRAYSCALE\n");
 	else if (dn == 2 && sn == 2) fz_printf(ctx, out, "TUPLTYPE GRAYSCALE_ALPHA\n");
-	else if (dn == 3 && sn == 4) fz_printf(ctx, out, "TUPLTYPE RGB\n");
-	else if (dn == 4 && sn == 4) fz_printf(ctx, out, "TUPLTYPE RGB_ALPHA\n");
-	else if (dn == 4 && sn == 5) fz_printf(ctx, out, "TUPLTYPE CMYK\n");
-	else if (dn == 5 && sn == 5) fz_printf(ctx, out, "TUPLTYPE CMYK_ALPHA\n");
+	else if (dn == 3 && sn == 3) fz_printf(ctx, out, "TUPLTYPE RGB\n");
+	else if (dn == 4 && sn == 3) fz_printf(ctx, out, "TUPLTYPE RGB_ALPHA\n");
+	else if (dn == 4 && sn == 4) fz_printf(ctx, out, "TUPLTYPE CMYK\n");
+	else if (dn == 5 && sn == 4) fz_printf(ctx, out, "TUPLTYPE CMYK_ALPHA\n");
 	fz_printf(ctx, out, "ENDHDR\n");
 }
 
 void
-fz_write_pam_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stride, int band, int bandheight, unsigned char *sp, int savealpha)
+fz_write_pam_band(fz_context *ctx, fz_output *out, int w, int h, int n, int alpha, int stride, int band, int bandheight, unsigned char *sp, int savealpha)
 {
 	int y, x;
 	int start = band * bandheight;
 	int end = start + bandheight;
 	int sn = n;
-	int dn = n;
-	if (!savealpha && dn > 1)
-		dn--;
+	int dn = n - alpha;
+	if (!alpha)
+		savealpha = 0;
+
+	dn += savealpha;
 
 	if (!out)
 		return;
@@ -1008,8 +1016,8 @@ fz_write_pam_band(fz_context *ctx, fz_output *out, int w, int h, int n, int stri
 void
 fz_write_pixmap_as_pam(fz_context *ctx, fz_output *out, fz_pixmap *pixmap, int savealpha)
 {
-	fz_write_pam_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, savealpha);
-	fz_write_pam_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->stride, 0, pixmap->h, pixmap->samples, savealpha);
+	fz_write_pam_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, savealpha);
+	fz_write_pam_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, pixmap->stride, 0, pixmap->h, pixmap->samples, savealpha);
 }
 
 void
@@ -1018,8 +1026,8 @@ fz_save_pixmap_as_pam(fz_context *ctx, fz_pixmap *pixmap, char *filename, int sa
 	fz_output *out = fz_new_output_with_path(ctx, filename, 0);
 	fz_try(ctx)
 	{
-		fz_write_pam_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, savealpha);
-		fz_write_pam_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->stride, 0, pixmap->h, pixmap->samples, savealpha);
+		fz_write_pam_header(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, savealpha);
+		fz_write_pam_band(ctx, out, pixmap->w, pixmap->h, pixmap->n, pixmap->alpha, pixmap->stride, 0, pixmap->h, pixmap->samples, savealpha);
 	}
 	fz_always(ctx)
 		fz_drop_output(ctx, out);
diff --git a/source/tools/mudraw.c b/source/tools/mudraw.c
index b1ab97e9..4b3a6df2 100644
--- a/source/tools/mudraw.c
+++ b/source/tools/mudraw.c
@@ -854,9 +854,9 @@ static void dodrawpage(fz_context *ctx, fz_page *page, fz_display_list *list, in
 			if (output)
 			{
 				if (output_format == OUT_PGM || output_format == OUT_PPM || output_format == OUT_PNM)
-					fz_write_pnm_header(ctx, out, pix->w, totalheight, pix->n);
+					fz_write_pnm_header(ctx, out, pix->w, totalheight, pix->n, pix->alpha);
 				else if (output_format == OUT_PAM)
-					fz_write_pam_header(ctx, out, pix->w, totalheight, pix->n, savealpha);
+					fz_write_pam_header(ctx, out, pix->w, totalheight, pix->n, pix->alpha, savealpha);
 				else if (output_format == OUT_PNG)
 					poc = fz_write_png_header(ctx, out, pix->w, totalheight, pix->n, pix->alpha, savealpha);
 				else if (output_format == OUT_PBM)
@@ -890,9 +890,9 @@ static void dodrawpage(fz_context *ctx, fz_page *page, fz_display_list *list, in
 				if (output)
 				{
 					if (output_format == OUT_PGM || output_format == OUT_PPM || output_format == OUT_PNM)
-						fz_write_pnm_band(ctx, out, pix->w, totalheight, pix->n, pix->stride, band, drawheight, pix->samples);
+						fz_write_pnm_band(ctx, out, pix->w, totalheight, pix->n, pix->alpha, pix->stride, band, drawheight, pix->samples);
 					else if (output_format == OUT_PAM)
-						fz_write_pam_band(ctx, out, pix->w, totalheight, pix->n, pix->stride, band, drawheight, pix->samples, savealpha);
+						fz_write_pam_band(ctx, out, pix->w, totalheight, pix->n, pix->alpha, pix->stride, band, drawheight, pix->samples, savealpha);
 					else if (output_format == OUT_PNG)
 						fz_write_png_band(ctx, out, poc, pix->stride, band, drawheight, pix->samples);
 					else if (output_format == OUT_PWG)