Move to be more like Ghostscript in the handling of shapes.

When pushing a transparency group, MuPDF used to always create a shape
plane if there was one before. Ghostscript works differently, and
only creates a shape plane if we are in a non-isolated group.

This means that when blending an isolated group back to a non
isolated group, GS uses the "source alpha" in place of the "shape
alpha" to modify the non-isolated groups shape plane.

We update MuPDF to do the same here - it requires some new (small)
routines to do this new type of shape update, but means we carry
smaller amounts of data around overall.

Also, for what little remains of my sanities sake, this helps by
making it easier to compare the debug output from the 2 different
renderers.

3 files changed, 94 insertions, 12 deletions

diff --git a/source/fitz/draw-device.c b/source/fitz/draw-device.c
index 85c0b7e3..27229108 100644
--- a/source/fitz/draw-device.c
+++ b/source/fitz/draw-device.c
@@ -2119,12 +2119,8 @@ fz_draw_begin_group(fz_context *ctx, fz_device *devp, const fz_rect *rect, fz_co
 			fz_copy_pixmap_rect(ctx, dest, state[0].dest, &bbox, dev->default_cs);
 		}
 
-		if (blendmode == 0 && alpha == 1.0f && isolated)
-		{
-			/* We can render direct to any existing shape plane.
-			 * If there isn't one, we don't need to make one. */
-			state[1].shape = state[0].shape;
-		}
+		if (isolated)
+			state[1].shape = NULL;
 		else
 		{
 			state[1].shape = fz_new_pixmap_with_bbox(ctx, NULL, &bbox, NULL, 1);
@@ -2212,17 +2208,21 @@ fz_draw_end_group(fz_context *ctx, fz_device *devp)
 	else
 		fz_blend_pixmap(ctx, state[0].dest, state[1].dest, alpha * 255, blendmode, isolated, state[1].shape);
 
+	assert(state[0].shape == NULL || state[0].shape != state[1].shape);
+	if (state[0].shape && state[0].shape != state[1].shape)
+	{
+		if (state[1].shape)
+			fz_paint_pixmap(state[0].shape, state[1].shape, alpha * 255);
+		else
+			fz_paint_pixmap_alpha(state[0].shape, state[1].dest, alpha * 255);
+	}
+	fz_drop_pixmap(ctx, state[1].shape);
 	/* The following test should not be required, but just occasionally
 	 * errors can cause the stack to get out of sync, and this might save
 	 * our bacon. */
+	assert(state[0].dest != state[1].dest);
 	if (state[0].dest != state[1].dest)
 		fz_drop_pixmap(ctx, state[1].dest);
-	if (state[0].shape != state[1].shape)
-	{
-		if (state[0].shape)
-			fz_paint_pixmap(state[0].shape, state[1].shape, alpha * 255);
-		fz_drop_pixmap(ctx, state[1].shape);
-	}
 #ifdef DUMP_GROUP_BLENDS
 	fz_dump_blend(ctx, " to get ", state[0].dest);
 	if (state[0].shape)
diff --git a/source/fitz/draw-imp.h b/source/fitz/draw-imp.h
index c14947c8..d2d50d2e 100644
--- a/source/fitz/draw-imp.h
+++ b/source/fitz/draw-imp.h
@@ -458,6 +458,7 @@ void fz_paint_image(fz_pixmap * restrict dst, const fz_irect * restrict scissor,
 void fz_paint_image_with_color(fz_pixmap * restrict dst, const fz_irect * restrict scissor, fz_pixmap *restrict shape, const fz_pixmap * restrict img, const fz_matrix * restrict ctm, const unsigned char * restrict colorbv, int lerp_allowed, int gridfit_as_tiled, const fz_overprint * restrict eop);
 
 void fz_paint_pixmap(fz_pixmap * restrict dst, const fz_pixmap * restrict src, int alpha);
+void fz_paint_pixmap_alpha(fz_pixmap * restrict dst, const fz_pixmap * restrict src, int alpha);
 void fz_paint_pixmap_with_mask(fz_pixmap * restrict dst, const fz_pixmap * restrict src, const fz_pixmap * restrict msk);
 void fz_paint_pixmap_with_bbox(fz_pixmap * restrict dst, const fz_pixmap * restrict src, int alpha, fz_irect bbox);
 void fz_paint_pixmap_with_overprint(fz_pixmap * restrict dst, const fz_pixmap * restrict src, const fz_overprint *op);
diff --git a/source/fitz/draw-paint.c b/source/fitz/draw-paint.c
index aaa1c890..0a58fd58 100644
--- a/source/fitz/draw-paint.c
+++ b/source/fitz/draw-paint.c
@@ -2256,6 +2256,87 @@ fz_paint_pixmap(fz_pixmap * restrict dst, const fz_pixmap * restrict src, int al
 	}
 }
 
+static inline void
+paint_span_alpha_solid(byte * restrict dp, const byte * restrict sp, int n, int w)
+{
+	TRACK_FN();
+	sp += n-1;
+	do
+	{
+		int s = *sp;
+		int t = FZ_EXPAND(255 - s);
+		sp += n;
+		*dp = s + FZ_COMBINE(*dp, t);
+		dp ++;
+	}
+	while (--w);
+}
+
+static inline void
+paint_span_alpha_not_solid(byte * restrict dp, const byte * restrict sp, int n, int w, int alpha)
+{
+	TRACK_FN();
+	sp += n-1;
+	alpha = FZ_EXPAND(alpha);
+	do
+	{
+		int masa = FZ_COMBINE(sp[0], alpha);
+		sp += n;
+		*dp = FZ_BLEND(*sp, *dp, masa);
+		dp++;
+	}
+	while (--w);
+}
+
+void
+fz_paint_pixmap_alpha(fz_pixmap * restrict dst, const fz_pixmap * restrict src, int alpha)
+{
+	const unsigned char *sp;
+	unsigned char *dp;
+	fz_irect bbox;
+	fz_irect bbox2;
+	int x, y, w, h, n;
+
+	if (alpha == 0)
+		return;
+
+	assert(dst->n == 1 && dst->alpha == 1 && src->n >= 1 && src->alpha == 1);
+
+	fz_pixmap_bbox_no_ctx(dst, &bbox);
+	fz_pixmap_bbox_no_ctx(src, &bbox2);
+	fz_intersect_irect(&bbox, &bbox2);
+
+	x = bbox.x0;
+	y = bbox.y0;
+	w = bbox.x1 - bbox.x0;
+	h = bbox.y1 - bbox.y0;
+	if (w == 0 || h == 0)
+		return;
+
+	n = src->n;
+	sp = src->samples + (unsigned int)((y - src->y) * src->stride + (x - src->x) * src->n);
+	dp = dst->samples + (unsigned int)((y - dst->y) * dst->stride + (x - dst->x) * dst->n);
+
+	if (alpha == 255)
+	{
+		while (h--)
+		{
+			paint_span_alpha_solid(dp, sp, n, w);
+			sp += src->stride;
+			dp += dst->stride;
+		}
+	}
+	else
+	{
+		while (h--)
+		{
+			paint_span_alpha_not_solid(dp, sp, n, w, alpha);
+			sp += src->stride;
+			dp += dst->stride;
+		}
+	}
+}
+
 void
 fz_paint_pixmap_with_overprint(fz_pixmap * restrict dst, const fz_pixmap * restrict src, const fz_overprint *op)
 {