Common code to ensure we only premultiply pixmaps with additive colors.

10 files changed, 119 insertions, 101 deletions

diff --git a/include/mupdf/fitz/colorspace.h b/include/mupdf/fitz/colorspace.h
index 398503d1..8a1c5caf 100644
--- a/include/mupdf/fitz/colorspace.h
+++ b/include/mupdf/fitz/colorspace.h
@@ -29,6 +29,13 @@ int fz_colorspace_is_indexed(fz_context *ctx, fz_colorspace *cs);
 int fz_colorspace_is_lab(fz_context *ctx, fz_colorspace *cs);
 
 /*
+	fz_colorspace_is_subtractive: Return true if a colorspace is subtractive.
+
+	True for CMYK, Separation and DeviceN colorspaces.
+*/
+int fz_colorspace_is_subtractive(fz_context *ctx, fz_colorspace *pix);
+
+/*
 	fz_device_gray: Get colorspace representing device specific gray.
 */
 fz_colorspace *fz_device_gray(fz_context *ctx);
@@ -77,7 +84,7 @@ typedef void (fz_colorspace_convert_fn)(fz_context *ctx, fz_colorspace *cs, cons
 
 typedef void (fz_colorspace_destruct_fn)(fz_context *ctx, fz_colorspace *cs);
 
-fz_colorspace *fz_new_colorspace(fz_context *ctx, char *name, int n, fz_colorspace_convert_fn *to_rgb, fz_colorspace_convert_fn *from_rgb, fz_colorspace_destruct_fn *destruct, void *data, size_t size);
+fz_colorspace *fz_new_colorspace(fz_context *ctx, char *name, int n, int is_subtractive, fz_colorspace_convert_fn *to_rgb, fz_colorspace_convert_fn *from_rgb, fz_colorspace_destruct_fn *destruct, void *data, size_t size);
 fz_colorspace *fz_new_indexed_colorspace(fz_context *ctx, fz_colorspace *base, int high, unsigned char *lookup);
 fz_colorspace *fz_keep_colorspace(fz_context *ctx, fz_colorspace *colorspace);
 void fz_drop_colorspace(fz_context *ctx, fz_colorspace *colorspace);
diff --git a/include/mupdf/fitz/pixmap.h b/include/mupdf/fitz/pixmap.h
index b2a47407..6dfd4bae 100644
--- a/include/mupdf/fitz/pixmap.h
+++ b/include/mupdf/fitz/pixmap.h
@@ -262,6 +262,13 @@ void fz_gamma_pixmap(fz_context *ctx, fz_pixmap *pix, float gamma);
 void fz_unmultiply_pixmap(fz_context *ctx, fz_pixmap *pix);
 
 /*
+	fz_ensure_pixmap_is_additive: Convert a pixmap in a subtractive colorspace to an additive colorspace.
+
+	Pixmaps with premultiplied alpha must be in an additive colorspace.
+*/
+fz_pixmap *fz_ensure_pixmap_is_additive(fz_context *ctx, fz_pixmap *pix);
+
+/*
 	fz_convert_pixmap: Convert an existing pixmap to a desired
 	colorspace. Other properties of the pixmap, such as resolution
 	and position are are copied to the converted pixmap.
diff --git a/source/fitz/colorspace-imp.h b/source/fitz/colorspace-imp.h
index 4d3f8825..bf8f3fef 100644
--- a/source/fitz/colorspace-imp.h
+++ b/source/fitz/colorspace-imp.h
@@ -7,6 +7,7 @@ struct fz_colorspace_s
 	size_t size;
 	char name[16];
 	int n;
+	int is_subtractive;
 	fz_colorspace_convert_fn *to_rgb;
 	fz_colorspace_convert_fn *from_rgb;
 	fz_colorspace_destruct_fn *free_data;
diff --git a/source/fitz/colorspace.c b/source/fitz/colorspace.c
index 1a32258b..63032951 100644
--- a/source/fitz/colorspace.c
+++ b/source/fitz/colorspace.c
@@ -15,13 +15,14 @@ fz_drop_colorspace_imp(fz_context *ctx, fz_storable *cs_)
 }
 
 fz_colorspace *
-fz_new_colorspace(fz_context *ctx, char *name, int n, fz_colorspace_convert_fn *to_rgb, fz_colorspace_convert_fn *from_rgb, fz_colorspace_destruct_fn *destruct, void *data, size_t size)
+fz_new_colorspace(fz_context *ctx, char *name, int n, int is_subtractive, fz_colorspace_convert_fn *to_rgb, fz_colorspace_convert_fn *from_rgb, fz_colorspace_destruct_fn *destruct, void *data, size_t size)
 {
 	fz_colorspace *cs = fz_malloc_struct(ctx, fz_colorspace);
 	FZ_INIT_STORABLE(cs, 1, fz_drop_colorspace_imp);
 	cs->size = sizeof(fz_colorspace) + size;
 	fz_strlcpy(cs->name, name, sizeof cs->name);
 	cs->n = n;
+	cs->is_subtractive = is_subtractive;
 	cs->to_rgb = to_rgb;
 	cs->from_rgb = from_rgb;
 	cs->free_data = destruct;
@@ -215,11 +216,17 @@ fz_colorspace_is_lab(fz_context *ctx, fz_colorspace *cs)
 	return (cs && cs->to_rgb == lab_to_rgb);
 }
 
-static fz_colorspace k_default_gray = { {-1, fz_drop_colorspace_imp}, 0, "DeviceGray", 1, gray_to_rgb, rgb_to_gray };
-static fz_colorspace k_default_rgb = { {-1, fz_drop_colorspace_imp}, 0, "DeviceRGB", 3, rgb_to_rgb, rgb_to_rgb };
-static fz_colorspace k_default_bgr = { {-1, fz_drop_colorspace_imp}, 0, "DeviceBGR", 3, bgr_to_rgb, rgb_to_bgr };
-static fz_colorspace k_default_cmyk = { {-1, fz_drop_colorspace_imp}, 0, "DeviceCMYK", 4, cmyk_to_rgb, rgb_to_cmyk };
-static fz_colorspace k_default_lab = { {-1, fz_drop_colorspace_imp}, 0, "Lab", 3, lab_to_rgb, rgb_to_lab };
+int
+fz_colorspace_is_subtractive(fz_context *ctx, fz_colorspace *cs)
+{
+	return (cs && cs->is_subtractive);
+}
+
+static fz_colorspace k_default_gray = { {-1, fz_drop_colorspace_imp}, 0, "DeviceGray", 1, 0, gray_to_rgb, rgb_to_gray };
+static fz_colorspace k_default_rgb = { {-1, fz_drop_colorspace_imp}, 0, "DeviceRGB", 3, 0, rgb_to_rgb, rgb_to_rgb };
+static fz_colorspace k_default_bgr = { {-1, fz_drop_colorspace_imp}, 0, "DeviceBGR", 3, 0, bgr_to_rgb, rgb_to_bgr };
+static fz_colorspace k_default_cmyk = { {-1, fz_drop_colorspace_imp}, 0, "DeviceCMYK", 4, 1, cmyk_to_rgb, rgb_to_cmyk };
+static fz_colorspace k_default_lab = { {-1, fz_drop_colorspace_imp}, 0, "Lab", 3, 0, lab_to_rgb, rgb_to_lab };
 
 static fz_colorspace *fz_default_gray = &k_default_gray;
 static fz_colorspace *fz_default_rgb = &k_default_rgb;
@@ -2011,7 +2018,7 @@ fz_new_indexed_colorspace(fz_context *ctx, fz_colorspace *base, int high, unsign
 
 	fz_try(ctx)
 	{
-		cs = fz_new_colorspace(ctx, "Indexed", 1, indexed_to_rgb, NULL, free_indexed, idx, sizeof(*idx) + (base->n * (idx->high + 1)) + base->size);
+		cs = fz_new_colorspace(ctx, "Indexed", 1, 0, indexed_to_rgb, NULL, free_indexed, idx, sizeof(*idx) + (base->n * (idx->high + 1)) + base->size);
 	}
 	fz_catch(ctx)
 	{
diff --git a/source/fitz/load-jpx.c b/source/fitz/load-jpx.c
index c32f66a4..7f9e9734 100644
--- a/source/fitz/load-jpx.c
+++ b/source/fitz/load-jpx.c
@@ -400,16 +400,11 @@ jpx_read_image(fz_context *ctx, fz_jpxd *state, unsigned char *data, size_t size
 
 		if (state->pix->alpha && ! (HAS_PALETTE(colorspace) && !state->expand_indexed))
 		{
-			/* CMYK is a subtractive colorspace, we want additive for premul alpha */
-			if (state->pix->n == 5)
-			{
-				fz_pixmap *rgb = fz_convert_pixmap(ctx, state->pix, fz_device_rgb(ctx), 1);
-				fz_drop_pixmap(ctx, state->pix);
-				state->pix = rgb;
-			}
-
 			if (alphas > 0 && prealphas == 0)
+			{
+				state->pix = fz_ensure_pixmap_is_additive(ctx, state->pix);
 				fz_premultiply_pixmap(ctx, state->pix);
+			}
 		}
 	}
 	fz_always(ctx)
@@ -674,6 +669,8 @@ jpx_read_image(fz_context *ctx, fz_jpxd *state, unsigned char *data, size_t size
 	int sub_h[FZ_MAX_COLORS];
 	int upsample_required = 0;
 
+	fz_var(img);
+
 	if (size < 2)
 		fz_throw(ctx, FZ_ERROR_GENERIC, "not enough data to determine image format");
 
@@ -865,13 +862,7 @@ jpx_read_image(fz_context *ctx, fz_jpxd *state, unsigned char *data, size_t size
 
 		if (a)
 		{
-			/* CMYK is a subtractive colorspace, we want additive for premul alpha */
-			if (n == 4)
-			{
-				fz_pixmap *rgb = fz_convert_pixmap(ctx, img, fz_device_rgb(ctx), 1);
-				fz_drop_pixmap(ctx, img);
-				img = rgb;
-			}
+			img = fz_ensure_pixmap_is_additive(ctx, img);
 			fz_premultiply_pixmap(ctx, img);
 		}
 	}
diff --git a/source/fitz/load-jxr.c b/source/fitz/load-jxr.c
index 8bff2833..4ec857dc 100644
--- a/source/fitz/load-jxr.c
+++ b/source/fitz/load-jxr.c
@@ -394,7 +394,9 @@ fz_pixmap *
 fz_load_jxr(fz_context *ctx, unsigned char *data, size_t size)
 {
 	struct info info = { 0 };
-	fz_pixmap *image;
+	fz_pixmap *image = NULL;
+
+	fz_var(image);
 
 	jxr_read_image(ctx, data, size, &info, 0);
 
@@ -405,18 +407,10 @@ fz_load_jxr(fz_context *ctx, unsigned char *data, size_t size)
 
 	fz_try(ctx)
 	{
-
 		fz_unpack_tile(ctx, image, info.samples, fz_colorspace_n(ctx, info.cspace) + 1, 8, info.stride, 0);
-
 		if (info.has_alpha && !info.has_premul)
 		{
-			/* CMYK is a subtractive colorspace, we want additive for premul alpha */
-			if (info.comps >= 4)
-			{
-				fz_pixmap *rgb = fz_convert_pixmap(ctx, image, fz_device_rgb(ctx), 1);
-				fz_drop_pixmap(ctx, image);
-				image = rgb;
-			}
+			image = fz_ensure_pixmap_is_additive(ctx, image);
 			fz_premultiply_pixmap(ctx, image);
 		}
 	}
diff --git a/source/fitz/load-pnm.c b/source/fitz/load-pnm.c
index 2874689a..5c8fad61 100644
--- a/source/fitz/load-pnm.c
+++ b/source/fitz/load-pnm.c
@@ -427,6 +427,8 @@ pam_binary_read_image(fz_context *ctx, struct info *pnm, unsigned char *p, unsig
 	int minval = 1;
 	int maxval = 65535;
 
+	fz_var(img);
+
 	p = pam_binary_read_header(ctx, pnm, p, e);
 
 	if (pnm->tupletype == PAM_UNKNOWN)
@@ -504,75 +506,76 @@ pam_binary_read_image(fz_context *ctx, struct info *pnm, unsigned char *p, unsig
 		int w, h, n;
 
 		img = fz_new_pixmap(ctx, pnm->cs, pnm->width, pnm->height, pnm->alpha);
-		dp = img->samples;
+		fz_try(ctx)
+		{
+			dp = img->samples;
 
-		w = img->w;
-		h = img->h;
-		n = img->n;
+			w = img->w;
+			h = img->h;
+			n = img->n;
 
-		if (pnm->maxval == 1 && e - p < w * h * n)
-		{
-			if (e - p < w * h * n / 8)
+			if (pnm->maxval == 1 && e - p < w * h * n)
+			{
+				if (e - p < w * h * n / 8)
+					fz_throw(ctx, FZ_ERROR_GENERIC, "truncated image");
+				packed = 1;
+			}
+			else if (e - p < w * h * n * (pnm->maxval < 256 ? 1 : 2))
 				fz_throw(ctx, FZ_ERROR_GENERIC, "truncated image");
-			packed = 1;
-		}
-		else if (e - p < w * h * n * (pnm->maxval < 256 ? 1 : 2))
-			fz_throw(ctx, FZ_ERROR_GENERIC, "truncated image");
 
-		if (pnm->maxval == 255)
-			memcpy(dp, p, w * h * n);
-		else if (bitmap && packed)
-		{
-			/* some encoders incorrectly pack bits into bytes and inverts the image */
-			for (y = 0; y < h; y++)
-				for (x = 0; x < w; x++)
-				{
-					for (k = 0; k < n; k++)
+			if (pnm->maxval == 255)
+				memcpy(dp, p, w * h * n);
+			else if (bitmap && packed)
+			{
+				/* some encoders incorrectly pack bits into bytes and inverts the image */
+				for (y = 0; y < h; y++)
+					for (x = 0; x < w; x++)
 					{
-						*dp++ = (*p & (1 << (7 - (x & 0x7)))) ? 0x00 : 0xff;
-						if ((x & 0x7) == 7)
+						for (k = 0; k < n; k++)
+						{
+							*dp++ = (*p & (1 << (7 - (x & 0x7)))) ? 0x00 : 0xff;
+							if ((x & 0x7) == 7)
+								p++;
+						}
+						if (w & 0x7)
 							p++;
 					}
-					if (w & 0x7)
-						p++;
-				}
-		}
-		else if (bitmap)
-		{
-			for (y = 0; y < h; y++)
-				for (x = 0; x < w; x++)
-					for (k = 0; k < n; k++)
-						*dp++ = *p++ ? 0xff : 0x00;
-		}
-		else if (pnm->maxval < 255)
-		{
-			for (y = 0; y < h; y++)
-				for (x = 0; x < w; x++)
-					for (k = 0; k < n; k++)
-						*dp++ = map_color(ctx, *p++, pnm->maxval, 255);
-		}
-		else
-		{
-			for (y = 0; y < h; y++)
-				for (x = 0; x < w; x++)
-					for (k = 0; k < n; k++)
-					{
-						*dp++ = map_color(ctx, (p[0] << 8) | p[1], pnm->maxval, 255);
-						p += 2;
-					}
-		}
-
-		if (pnm->alpha)
-		{
-			/* CMYK is a subtractive colorspace, we want additive for premul alpha */
-			if (img->n > 4)
+			}
+			else if (bitmap)
+			{
+				for (y = 0; y < h; y++)
+					for (x = 0; x < w; x++)
+						for (k = 0; k < n; k++)
+							*dp++ = *p++ ? 0xff : 0x00;
+			}
+			else if (pnm->maxval < 255)
 			{
-				fz_pixmap *rgb = fz_convert_pixmap(ctx, img, fz_device_rgb(ctx), 1);
-				fz_drop_pixmap(ctx, img);
-				img = rgb;
+				for (y = 0; y < h; y++)
+					for (x = 0; x < w; x++)
+						for (k = 0; k < n; k++)
+							*dp++ = map_color(ctx, *p++, pnm->maxval, 255);
+			}
+			else
+			{
+				for (y = 0; y < h; y++)
+					for (x = 0; x < w; x++)
+						for (k = 0; k < n; k++)
+						{
+							*dp++ = map_color(ctx, (p[0] << 8) | p[1], pnm->maxval, 255);
+							p += 2;
+						}
 			}
 
-			fz_premultiply_pixmap(ctx, img);
+			if (pnm->alpha)
+			{
+				img = fz_ensure_pixmap_is_additive(ctx, img);
+				fz_premultiply_pixmap(ctx, img);
+			}
+		}
+		fz_catch(ctx)
+		{
+			fz_drop_pixmap(ctx, img);
+			fz_rethrow(ctx);
 		}
 	}
 
diff --git a/source/fitz/load-tiff.c b/source/fitz/load-tiff.c
index 7262b23e..8d759254 100644
--- a/source/fitz/load-tiff.c
+++ b/source/fitz/load-tiff.c
@@ -1283,14 +1283,7 @@ fz_load_tiff_subimage(fz_context *ctx, unsigned char *buf, size_t len, int subim
 		/* We should only do this on non-pre-multiplied images, but files in the wild are bad */
 		if (tiff.extrasamples /* == 2 */)
 		{
-			/* CMYK is a subtractive colorspace, we want additive for premul alpha */
-			if (image->n == 5)
-			{
-				fz_pixmap *rgb = fz_convert_pixmap(ctx, image, fz_device_rgb(ctx), 1);
-				fz_drop_pixmap(ctx, image);
-				image = rgb;
-			}
-
+			image = fz_ensure_pixmap_is_additive(ctx, image);
 			fz_premultiply_pixmap(ctx, image);
 		}
 	}
diff --git a/source/fitz/pixmap.c b/source/fitz/pixmap.c
index 1d080576..156206f9 100644
--- a/source/fitz/pixmap.c
+++ b/source/fitz/pixmap.c
@@ -670,6 +670,9 @@ fz_premultiply_pixmap(fz_context *ctx, fz_pixmap *pix)
 	if (!pix->alpha)
 		return;
 
+	if (fz_colorspace_is_subtractive(ctx, pix->colorspace))
+		fz_throw(ctx, FZ_ERROR_GENERIC, "cannot pre-multiply subtractive colors");
+
 	for (y = 0; y < pix->h; y++)
 	{
 		for (x = 0; x < pix->w; x++)
@@ -709,6 +712,18 @@ fz_unmultiply_pixmap(fz_context *ctx, fz_pixmap *pix)
 }
 
 fz_pixmap *
+fz_ensure_pixmap_is_additive(fz_context *ctx, fz_pixmap *pix)
+{
+	if (fz_colorspace_is_subtractive(ctx, pix->colorspace))
+	{
+		fz_pixmap *rgb = fz_convert_pixmap(ctx, pix, fz_device_rgb(ctx), 1);
+		fz_drop_pixmap(ctx, pix);
+		return rgb;
+	}
+	return pix;
+}
+
+fz_pixmap *
 fz_alpha_from_gray(fz_context *ctx, fz_pixmap *gray)
 {
 	fz_pixmap *alpha;
diff --git a/source/pdf/pdf-colorspace.c b/source/pdf/pdf-colorspace.c
index f9a95335..18a4aa17 100644
--- a/source/pdf/pdf-colorspace.c
+++ b/source/pdf/pdf-colorspace.c
@@ -108,7 +108,7 @@ load_separation(fz_context *ctx, pdf_document *doc, pdf_obj *array)
 		sep->base = base;
 		sep->tint = tint;
 
-		cs = fz_new_colorspace(ctx, n == 1 ? "Separation" : "DeviceN", n, separation_to_rgb, NULL, free_separation, sep,
+		cs = fz_new_colorspace(ctx, n == 1 ? "Separation" : "DeviceN", n, 1, separation_to_rgb, NULL, free_separation, sep,
 			sizeof(struct separation) + (base ? base->size : 0) + fz_function_size(ctx, tint));
 	}
 	fz_catch(ctx)