Bug 692681: Pull SoftMask invocation logic together

Softmasks can be applied in 2 places in our code; once when starting a
group, once when running an XObject. The two implementations had
drifted apart. To avoid this in future, pull the two together.

This solves the bug, apart from the issue of transfer functions not
working.

Also, fix another issue seen in cluster testing. For luminance smasks
the bbox is only used to clip the contents drawn - the background color
extends into the surrounding area. Fix the code to respect this.

And another problem; text in soft masks would upset text outside the
SMasks - fix this by storing/restoring the text settings in the
interpreter state around the smask rendering.

1 files changed, 97 insertions, 71 deletions

diff --git a/pdf/pdf_interpret.c b/pdf/pdf_interpret.c
index 2f160eef..1c077901 100644
--- a/pdf/pdf_interpret.c
+++ b/pdf/pdf_interpret.c
@@ -323,58 +323,104 @@ pdf_is_hidden_ocg(pdf_obj *ocg, pdf_csi *csi, pdf_obj *rdb)
  * Emit graphics calls to device.
  */
 
-static void
-pdf_begin_group(pdf_csi *csi, const fz_rect *bbox)
+typedef struct softmask_save_s softmask_save;
+
+struct softmask_save_s
+{
+	pdf_xobject *softmask;
+	fz_matrix ctm;
+};
+
+static pdf_gstate *
+begin_softmask(pdf_csi * csi, softmask_save *save)
 {
 	pdf_gstate *gstate = csi->gstate + csi->gtop;
-	fz_context *ctx = csi->dev->ctx;
+	pdf_xobject *softmask = gstate->softmask;
+	fz_rect mask_bbox;
+	fz_context *ctx;
+	fz_matrix save_tm, save_tlm, save_ctm;
+	int save_in_text;
 
-	if (gstate->softmask)
-	{
-		pdf_xobject *softmask = gstate->softmask;
-		fz_rect mask_bbox = softmask->bbox;
-		fz_matrix save_ctm = gstate->ctm;
+	save->softmask = softmask;
+	if (softmask == NULL)
+		return gstate;
+	save->ctm = gstate->softmask_ctm;
+	save_ctm = gstate->ctm;
 
+	mask_bbox = softmask->bbox;
+	ctx = csi->dev->ctx;
+	save_tm = csi->tm;
+	save_tlm = csi->tlm;
+	save_in_text = csi->in_text;
+
+	csi->in_text = 0;
+	if (gstate->luminosity)
+		mask_bbox = fz_infinite_rect;
+	else
+	{
 		fz_transform_rect(&mask_bbox, &softmask->matrix);
 		fz_transform_rect(&mask_bbox, &gstate->softmask_ctm);
-		gstate->softmask = NULL;
-		gstate->ctm = gstate->softmask_ctm;
+	}
+	gstate->softmask = NULL;
+	gstate->ctm = gstate->softmask_ctm;
 
-		fz_begin_mask(csi->dev, &mask_bbox, gstate->luminosity,
+	fz_begin_mask(csi->dev, &mask_bbox, gstate->luminosity,
 			softmask->colorspace, gstate->softmask_bc);
-		fz_try(ctx)
-		{
-			pdf_run_xobject(csi, NULL, softmask, &fz_identity);
-		}
-		fz_catch(ctx)
-		{
-			/* FIXME: Ignore error - nasty, but if we throw from
-			 * here the clip stack would be messed up. */
-			if (csi->cookie)
-				csi->cookie->errors++;
-		}
+	fz_try(ctx)
+	{
+		pdf_run_xobject(csi, NULL, softmask, &fz_identity);
+	}
+	fz_catch(ctx)
+	{
+		/* FIXME: Ignore error - nasty, but if we throw from
+		 * here the clip stack would be messed up. */
+		if (csi->cookie)
+			csi->cookie->errors++;
+	}
 
-		fz_end_mask(csi->dev);
+	fz_end_mask(csi->dev);
 
-		gstate = csi->gstate + csi->gtop;
-		gstate->softmask = softmask;
-		gstate->ctm = save_ctm;
-	}
+	csi->tm = save_tm;
+	csi->tlm = save_tlm;
+	csi->in_text = save_in_text;
+
+	gstate = csi->gstate + csi->gtop;
+	gstate->ctm = save_ctm;
+
+	return gstate;
+}
+
+static void
+end_softmask(pdf_csi *csi, softmask_save *save)
+{
+	pdf_gstate *gstate = csi->gstate + csi->gtop;
+
+	if (save->softmask == NULL)
+		return;
+
+	gstate->softmask = save->softmask;
+	gstate->softmask_ctm = save->ctm;
+	fz_pop_clip(csi->dev);
+}
+
+static void
+pdf_begin_group(pdf_csi *csi, const fz_rect *bbox, softmask_save *softmask)
+{
+	pdf_gstate *gstate = begin_softmask(csi, softmask);
 
 	if (gstate->blendmode)
 		fz_begin_group(csi->dev, bbox, 1, 0, gstate->blendmode, 1);
 }
 
 static void
-pdf_end_group(pdf_csi *csi)
+pdf_end_group(pdf_csi *csi, softmask_save *softmask)
 {
 	pdf_gstate *gstate = csi->gstate + csi->gtop;
 
 	if (gstate->blendmode)
 		fz_end_group(csi->dev);
 
-	if (gstate->softmask)
-		fz_pop_clip(csi->dev);
+	end_softmask(csi, softmask);
 }
 
 static void
@@ -383,17 +429,18 @@ pdf_show_shade(pdf_csi *csi, fz_shade *shd)
 	fz_context *ctx = csi->dev->ctx;
 	pdf_gstate *gstate = csi->gstate + csi->gtop;
 	fz_rect bbox;
+	softmask_save softmask = { NULL };
 
 	if (csi->in_hidden_ocg > 0)
 		return;
 
 	fz_bound_shade(ctx, shd, &gstate->ctm, &bbox);
 
-	pdf_begin_group(csi, &bbox);
+	pdf_begin_group(csi, &bbox, &softmask);
 
 	fz_fill_shade(csi->dev, shd, &gstate->ctm, gstate->fill.alpha);
 
-	pdf_end_group(csi);
+	pdf_end_group(csi, &softmask);
 }
 
 static void
@@ -402,6 +449,7 @@ pdf_show_image(pdf_csi *csi, fz_image *image)
 	pdf_gstate *gstate = csi->gstate + csi->gtop;
 	fz_matrix image_ctm;
 	fz_rect bbox;
+	softmask_save softmask = { NULL };
 
 	if (csi->in_hidden_ocg > 0)
 		return;
@@ -421,7 +469,7 @@ pdf_show_image(pdf_csi *csi, fz_image *image)
 		fz_clip_image_mask(csi->dev, image->mask, &bbox, &image_ctm);
 	}
 	else
-		pdf_begin_group(csi, &bbox);
+		pdf_begin_group(csi, &bbox, &softmask);
 
 	if (!image->colorspace)
 	{
@@ -464,7 +512,7 @@ pdf_show_image(pdf_csi *csi, fz_image *image)
 			fz_end_group(csi->dev);
 	}
 	else
-		pdf_end_group(csi);
+		pdf_end_group(csi, &softmask);
 }
 
 static void
@@ -474,6 +522,7 @@ pdf_show_path(pdf_csi *csi, int doclose, int dofill, int dostroke, int even_odd)
 	pdf_gstate *gstate = csi->gstate + csi->gtop;
 	fz_path *path;
 	fz_rect bbox;
+	softmask_save softmask = { NULL };
 
 	if (dostroke) {
 		if (csi->dev->flags & (FZ_DEVFLAG_STROKECOLOR_UNDEFINED | FZ_DEVFLAG_LINEJOIN_UNDEFINED | FZ_DEVFLAG_LINEWIDTH_UNDEFINED))
@@ -509,7 +558,7 @@ pdf_show_path(pdf_csi *csi, int doclose, int dofill, int dostroke, int even_odd)
 			dostroke = dofill = 0;
 
 		if (dofill || dostroke)
-			pdf_begin_group(csi, &bbox);
+			pdf_begin_group(csi, &bbox, &softmask);
 
 		if (dofill)
 		{
@@ -570,7 +619,7 @@ pdf_show_path(pdf_csi *csi, int doclose, int dofill, int dostroke, int even_odd)
 		}
 
 		if (dofill || dostroke)
-			pdf_end_group(csi);
+			pdf_end_group(csi, &softmask);
 	}
 	fz_always(ctx)
 	{
@@ -596,6 +645,7 @@ pdf_flush_text(pdf_csi *csi)
 	int doclip;
 	int doinvisible;
 	fz_context *ctx = csi->dev->ctx;
+	softmask_save softmask = { NULL };
 
 	if (!csi->text)
 		return;
@@ -628,7 +678,7 @@ pdf_flush_text(pdf_csi *csi)
 		if (text->len == 0)
 			break;
 
-		pdf_begin_group(csi, &tb);
+		pdf_begin_group(csi, &tb, &softmask);
 
 		if (doinvisible)
 			fz_ignore_text(csi->dev, text, &gstate->ctm);
@@ -699,7 +749,7 @@ pdf_flush_text(pdf_csi *csi)
 			csi->accumulate = 2;
 		}
 
-		pdf_end_group(csi);
+		pdf_end_group(csi, &softmask);
 	}
 	fz_always(ctx)
 	{
@@ -1391,6 +1441,7 @@ pdf_run_xobject(pdf_csi *csi, pdf_obj *resources, pdf_xobject *xobj, const fz_ma
 	int oldtop = 0;
 	int popmask;
 	fz_matrix local_transform = *transform;
+	softmask_save softmask = { NULL };
 
 	/* Avoid infinite recursion */
 	if (xobj == NULL || pdf_obj_mark(xobj->me))
@@ -1398,7 +1449,6 @@ pdf_run_xobject(pdf_csi *csi, pdf_obj *resources, pdf_xobject *xobj, const fz_ma
 
 	fz_var(gstate);
 	fz_var(oldtop);
-	fz_var(popmask);
 
 	fz_try(ctx)
 	{
@@ -1417,31 +1467,7 @@ pdf_run_xobject(pdf_csi *csi, pdf_obj *resources, pdf_xobject *xobj, const fz_ma
 		{
 			fz_rect bbox = xobj->bbox;
 			fz_transform_rect(&bbox, &gstate->ctm);
-			if (gstate->softmask)
-			{
-				pdf_xobject *softmask = gstate->softmask;
-
-				gstate->softmask = NULL;
-				popmask = 1;
-
-				fz_begin_mask(csi->dev, &bbox, gstate->luminosity,
-					softmask->colorspace, gstate->softmask_bc);
-				fz_try(ctx)
-				{
-					pdf_run_xobject(csi, resources, softmask, &fz_identity);
-				}
-				fz_catch(ctx)
-				{
-					/* FIXME: Ignore error - nasty, but if
-					 * we throw from here the clip stack
-					 * would be messed up */
-					if (csi->cookie)
-						csi->cookie->errors++;
-				}
-				fz_end_mask(csi->dev);
-
-				pdf_drop_xobject(ctx, softmask);
-			}
+			gstate = begin_softmask(csi, &softmask);
 
 			fz_begin_group(csi->dev, &bbox,
 				xobj->isolated, xobj->knockout, gstate->blendmode, gstate->fill.alpha);
@@ -1484,19 +1510,19 @@ pdf_run_xobject(pdf_csi *csi, pdf_obj *resources, pdf_xobject *xobj, const fz_ma
 		}
 
 		pdf_obj_unmark(xobj->me);
+
+		/* wrap up transparency stacks */
+		if (xobj->transparency)
+		{
+			fz_end_group(csi->dev);
+			end_softmask(csi, &softmask);
+		}
 	}
 	fz_catch(ctx)
 	{
 		fz_rethrow(ctx);
 	}
 
-	/* wrap up transparency stacks */
-	if (xobj->transparency)
-	{
-		fz_end_group(csi->dev);
-		if (popmask)
-			fz_pop_clip(csi->dev);
-	}
 }
 
 static void