More changes to blending of groups.

In an effort to solve bug 692377, examine what ghostscript does,
and move mupdf into line with it.

Firstly, it seems that rather than collecting a pure 'shape' plane
ghostscript keeps a 'shape-with-alpha' plane. Only a tiny change is
required to move mupdf to the do the same thing, so done here.

Secondly, it seems that ghostscript 'uncomposites' the result of
non-isolated groups, before applying the blend mode. It's not clear
to me that this is entirely correct; this 'uncomposite' operation
assumes that all compositing has been done internally with the
'normal' blend mode. Nonetheless, I do the same here, and it seems
to work.

This 'uncomposite' operation may yet turn out to be a source of bugs
if I have muddled the use of premultiplied and non-premultiplied
components, but it seems to work on the testfiles I have.

1 files changed, 70 insertions, 14 deletions

diff --git a/draw/draw_blend.c b/draw/draw_blend.c
index 20ad8a28..2f72ec9c 100644
--- a/draw/draw_blend.c
+++ b/draw/draw_blend.c
@@ -335,22 +335,57 @@ fz_blend_separable_nonisolated(byte * restrict bp, byte * restrict sp, int n, in
 {
 	int k;
 	int n1 = n - 1;
+
+	if (alpha == 255 && blendmode == 0)
+	{
+		/* In this case, the uncompositing and the recompositing
+		 * cancel one another out, and it's just a simple copy. */
+		/* FIXME: Maybe we can avoid using the shape plane entirely
+		 * and just copy? */
+		while (w--)
+		{
+			int ha = fz_mul255(*hp++, alpha); /* ha = shape_alpha */
+			/* If ha == 0 then leave everything unchanged */
+			if (ha != 0)
+			{
+				for (k = 0; k < n; k++)
+				{
+					bp[k] = sp[k];
+				}
+			}
+
+			sp += n;
+			bp += n;
+		}
+		return;
+	}
 	while (w--)
 	{
-		int ha = fz_mul255(*hp++, alpha); /* ha = shape_alpha */
-		/* If ha == 0 then leave everything unchanged */
-		if (ha != 0)
+		int ha = *hp++;
+		int haa = fz_mul255(ha, alpha); /* ha = shape_alpha */
+		/* If haa == 0 then leave everything unchanged */
+		if (haa != 0)
 		{
 			int sa = sp[n1];
 			int ba = bp[n1];
-			int baha = fz_mul255(ba, ha);
+			int baha = fz_mul255(ba, haa);
 
 			/* ugh, division to get non-premul components */
 			int invsa = sa ? 255 * 256 / sa : 0;
 			int invba = ba ? 255 * 256 / ba : 0;
 
 			/* Calculate result_alpha */
-			int ra = bp[n1] = ba - baha + ha;
+			int ra = bp[n1] = ba - baha + haa;
+
+			/* Because we are a non-isolated group, we need to
+			 * 'uncomposite' before we blend (recomposite).
+			 * We assume that normal blending has been done inside
+			 * the group, so:   ra.rc = (1-ha).bc + ha.sc
+			 * A bit of rearrangement, and that gives us that:
+			 *  sc = (ra.rc - bc)/ha + bc
+			 * Now, the result of the blend was stored in src, so:
+			 */
+			int invha = ha ? 255 * 256 / ha : 0;
 
 			if (ra != 0) for (k = 0; k < n1; k++)
 			{
@@ -358,6 +393,11 @@ fz_blend_separable_nonisolated(byte * restrict bp, byte * restrict sp, int n, in
 				int bc = (bp[k] * invba) >> 8;
 				int rc;
 
+				/* Uncomposite */
+				sc = (((sc-bc)*invha)>>8) + bc;
+				if (sc < 0) sc = 0;
+				if (sc > 255) sc = 255;
+
 				switch (blendmode)
 				{
 				default:
@@ -374,7 +414,7 @@ fz_blend_separable_nonisolated(byte * restrict bp, byte * restrict sp, int n, in
 				case FZ_BLEND_DIFFERENCE: rc = fz_difference_byte(bc, sc); break;
 				case FZ_BLEND_EXCLUSION: rc = fz_exclusion_byte(bc, sc); break;
 				}
-				rc = fz_mul255(255 - ha, bc) + fz_mul255(255 - ba, sc) + fz_mul255(baha, rc);
+				rc = fz_mul255(255 - haa, bc) + fz_mul255(fz_mul255(255 - ba, sc), haa) + fz_mul255(baha, rc);
 				if (rc < 0) rc = 0;
 				if (rc > 255) rc = 255;
 				bp[k] = fz_mul255(rc, ra);
@@ -391,18 +431,29 @@ fz_blend_nonseparable_nonisolated(byte * restrict bp, byte * restrict sp, int w,
 {
 	while (w--)
 	{
-		int ha = fz_mul255(*hp++, alpha);
-		if (ha != 0)
+		int ha = *hp++;
+		int haa = fz_mul255(ha, alpha);
+		if (haa != 0)
 		{
 			int sa = sp[3];
 			int ba = bp[3];
-			int baha = fz_mul255(ba, ha);
+			int baha = fz_mul255(ba, haa);
 
 			/* Calculate result_alpha */
-			int ra = bp[3] = ba - baha + ha;
-
+			int ra = bp[3] = ba - baha + haa;
 			if (ra != 0)
 			{
+				/* Because we are a non-isolated group, we
+				 * need to 'uncomposite' before we blend
+				 * (recomposite). We assume that normal
+				 * blending has been done inside the group,
+				 * so:     ra.rc = (1-ha).bc + ha.sc
+				 * A bit of rearrangement, and that gives us
+				 * that:   sc = (ra.rc - bc)/ha + bc
+				 * Now, the result of the blend was stored in
+				 * src, so: */
+				int invha = ha ? 255 * 256 / ha : 0;
+
 				int rr, rg, rb;
 
 				/* ugh, division to get non-premul components */
@@ -417,6 +468,11 @@ fz_blend_nonseparable_nonisolated(byte * restrict bp, byte * restrict sp, int w,
 				int bg = (bp[1] * invba) >> 8;
 				int bb = (bp[2] * invba) >> 8;
 
+				/* Uncomposite */
+				sr = (((sr-br)*invha)>>8) + br;
+				sg = (((sg-bg)*invha)>>8) + bg;
+				sb = (((sb-bb)*invha)>>8) + bb;
+
 				switch (blendmode)
 				{
 				default:
@@ -434,9 +490,9 @@ fz_blend_nonseparable_nonisolated(byte * restrict bp, byte * restrict sp, int w,
 					break;
 				}
 
-				rr = fz_mul255(255 - ha, bp[0]) + fz_mul255(255 - ba, sp[0]) + fz_mul255(baha, rr);
-				rg = fz_mul255(255 - ha, bp[1]) + fz_mul255(255 - ba, sp[1]) + fz_mul255(baha, rg);
-				rb = fz_mul255(255 - ha, bp[2]) + fz_mul255(255 - ba, sp[2]) + fz_mul255(baha, rb);
+				rr = fz_mul255(255 - haa, bp[0]) + fz_mul255(fz_mul255(255 - ba, sr), haa) + fz_mul255(baha, rr);
+				rg = fz_mul255(255 - haa, bp[1]) + fz_mul255(fz_mul255(255 - ba, sg), haa) + fz_mul255(baha, rg);
+				rb = fz_mul255(255 - haa, bp[2]) + fz_mul255(fz_mul255(255 - ba, sb), haa) + fz_mul255(baha, rb);
 				bp[0] = fz_mul255(ra, rr);
 				bp[1] = fz_mul255(ra, rg);
 				bp[2] = fz_mul255(ra, rb);