Bug 695843: Tweak bboxes of type3 fonts; honour the d1 values.

The example file for this bug has an invalid font bbox. The current
code uses this bbox (or some multiple of it) to clip the glyphs
size.

In the new code, when we convert the glyphs to display lists we
watch for the bbox given in any d1 operator used. If we find one,
we gather the rectangle specified and store it as the glyph rectangle
in the fz_font.

If we then attempt to bound a glyph that used d1, it happens instantly
without needing to run the list. This seems to match acrobats behaviour.

Tests indicate that Acrobat never clips d0 glyphs, so our behaviour
is still different here, but I am not changing this at the moment.

Also, I note that t3flags should be a un unsigned short but are currently
just a char. Fix that too.

Also fix some missing code in fz_new_font that would cause leaks if mallocs
failed.

5 files changed, 38 insertions, 13 deletions

diff --git a/include/mupdf/fitz/device.h b/include/mupdf/fitz/device.h
index e8098502..55674abd 100644
--- a/include/mupdf/fitz/device.h
+++ b/include/mupdf/fitz/device.h
@@ -40,6 +40,7 @@ enum
 	/* Arguably we should have a bit for the dash pattern itself being
 	 * undefined, but that causes problems; do we assume that it should
 	 * always be set to non-dashing at the start of every glyph? */
+	FZ_DEVFLAG_BBOX_DEFINED = 2048,
 };
 
 enum
@@ -130,6 +131,8 @@ struct fz_device_s
 	int (*begin_tile)(fz_context *, fz_device *, const fz_rect *area, const fz_rect *view, float xstep, float ystep, const fz_matrix *ctm, int id);
 	void (*end_tile)(fz_context *, fz_device *);
 
+	fz_rect d1_rect;
+
 	int error_depth;
 	char errmess[256];
 
diff --git a/include/mupdf/fitz/font.h b/include/mupdf/fitz/font.h
index 6c02bc60..3a4dccf0 100644
--- a/include/mupdf/fitz/font.h
+++ b/include/mupdf/fitz/font.h
@@ -44,7 +44,7 @@ struct fz_font_s
 	fz_buffer **t3procs; /* has 256 entries if used */
 	struct fz_display_list_s **t3lists; /* has 256 entries if used */
 	float *t3widths; /* has 256 entries if used */
-	char *t3flags; /* has 256 entries if used */
+	unsigned short *t3flags; /* has 256 entries if used */
 	void *t3doc; /* a pdf_document for the callback */
 	void (*t3run)(fz_context *ctx, void *doc, void *resources, fz_buffer *contents, struct fz_device_s *dev, const fz_matrix *ctm, void *gstate, int nestedDepth);
 	void (*t3freeres)(fz_context *ctx, void *doc, void *resources);
diff --git a/source/fitz/font.c b/source/fitz/font.c
index 5f2f85e1..3d4a06e5 100644
--- a/source/fitz/font.c
+++ b/source/fitz/font.c
@@ -1007,10 +1007,18 @@ fz_new_type3_font(fz_context *ctx, const char *name, const fz_matrix *matrix)
 	int i;
 
 	font = fz_new_font(ctx, name, 1, 256);
-	font->t3procs = fz_malloc_array(ctx, 256, sizeof(fz_buffer*));
-	font->t3lists = fz_malloc_array(ctx, 256, sizeof(fz_display_list*));
-	font->t3widths = fz_malloc_array(ctx, 256, sizeof(float));
-	font->t3flags = fz_malloc_array(ctx, 256, sizeof(char));
+	fz_try(ctx)
+	{
+		font->t3procs = fz_malloc_array(ctx, 256, sizeof(fz_buffer*));
+		font->t3lists = fz_malloc_array(ctx, 256, sizeof(fz_display_list*));
+		font->t3widths = fz_malloc_array(ctx, 256, sizeof(float));
+		font->t3flags = fz_malloc_array(ctx, 256, sizeof(unsigned short));
+	}
+	fz_catch(ctx)
+	{
+		fz_drop_font(ctx, font);
+		fz_rethrow(ctx);
+	}
 
 	font->t3matrix = *matrix;
 	for (i = 0; i < 256; i++)
@@ -1050,29 +1058,35 @@ fz_prepare_t3_glyph(fz_context *ctx, fz_font *font, int gid, int nested_depth)
 			FZ_DEVFLAG_LINEWIDTH_UNDEFINED;
 	font->t3run(ctx, font->t3doc, font->t3resources, contents, dev, &fz_identity, NULL, 0);
 	font->t3flags[gid] = dev->flags;
+	if (dev->flags & FZ_DEVFLAG_BBOX_DEFINED)
+	{
+		assert(font->bbox_table != NULL);
+		assert(font->bbox_count > gid);
+		font->bbox_table[gid] = dev->d1_rect;
+		fz_transform_rect(&font->bbox_table[gid], &font->t3matrix);
+	}
 	fz_drop_device(ctx, dev);
 }
 
 static fz_rect *
-fz_bound_t3_glyph(fz_context *ctx, fz_font *font, int gid, const fz_matrix *trm, fz_rect *bounds)
+fz_bound_t3_glyph(fz_context *ctx, fz_font *font, int gid, fz_rect *bounds)
 {
 	fz_display_list *list;
-	fz_matrix ctm;
 	fz_device *dev;
 	fz_rect big;
+	float m;
 
 	list = font->t3lists[gid];
 	if (!list)
 	{
 		*bounds = fz_empty_rect;
-		return fz_transform_rect(bounds, trm);
+		return bounds;
 	}
 
-	fz_concat(&ctm, &font->t3matrix, trm);
 	dev = fz_new_bbox_device(ctx, bounds);
 	fz_try(ctx)
 	{
-		fz_run_display_list(ctx, list, dev, &ctm, &fz_infinite_rect, NULL);
+		fz_run_display_list(ctx, list, dev, &font->t3matrix, &fz_infinite_rect, NULL);
 	}
 	fz_always(ctx)
 	{
@@ -1085,7 +1099,8 @@ fz_bound_t3_glyph(fz_context *ctx, fz_font *font, int gid, const fz_matrix *trm,
 
 	/* clip the bbox size to a reasonable maximum for degenerate glyphs */
 	big = font->bbox;
-	fz_expand_rect(&big, fz_max(fz_matrix_expansion(&ctm) * 2, fz_max(big.x1 - big.x0, big.y1 - big.y0)));
+	m = fz_max(fz_abs(big.x1 - big.x0), fz_abs(big.y1 - big.y0));
+	fz_expand_rect(&big, fz_max(fz_matrix_expansion(&font->t3matrix) * 2, m));
 	fz_intersect_rect(bounds, &big);
 
 	return bounds;
@@ -1256,7 +1271,7 @@ fz_bound_glyph(fz_context *ctx, fz_font *font, int gid, const fz_matrix *trm, fz
 			if (font->ft_face)
 				fz_bound_ft_glyph(ctx, font, gid, &font->bbox_table[gid]);
 			else if (font->t3lists)
-				fz_bound_t3_glyph(ctx, font, gid, &fz_identity, &font->bbox_table[gid]);
+				fz_bound_t3_glyph(ctx, font, gid, &font->bbox_table[gid]);
 			else
 				font->bbox_table[gid] = fz_empty_rect;
 		}
diff --git a/source/pdf/pdf-interpret-imp.h b/source/pdf/pdf-interpret-imp.h
index 817a7d2c..9c72bfe2 100644
--- a/source/pdf/pdf-interpret-imp.h
+++ b/source/pdf/pdf-interpret-imp.h
@@ -129,6 +129,8 @@ struct pdf_csi_s
 	int xbalance;
 	int in_text;
 
+	fz_rect d1_rect;
+
 	/* cookie support */
 	fz_cookie *cookie;
 };
diff --git a/source/pdf/pdf-op-run.c b/source/pdf/pdf-op-run.c
index c09abb45..204d3c23 100644
--- a/source/pdf/pdf-op-run.c
+++ b/source/pdf/pdf-op-run.c
@@ -2367,7 +2367,7 @@ static void pdf_run_d1(pdf_csi *csi, void *state)
 
 	if (pr->nested_depth > 1)
 		return;
-	pr->dev->flags |= FZ_DEVFLAG_MASK;
+	pr->dev->flags |= FZ_DEVFLAG_MASK | FZ_DEVFLAG_BBOX_DEFINED;
 	pr->dev->flags &= ~(FZ_DEVFLAG_FILLCOLOR_UNDEFINED |
 				FZ_DEVFLAG_STROKECOLOR_UNDEFINED |
 				FZ_DEVFLAG_STARTCAP_UNDEFINED |
@@ -2376,6 +2376,11 @@ static void pdf_run_d1(pdf_csi *csi, void *state)
 				FZ_DEVFLAG_LINEJOIN_UNDEFINED |
 				FZ_DEVFLAG_MITERLIMIT_UNDEFINED |
 				FZ_DEVFLAG_LINEWIDTH_UNDEFINED);
+
+	pr->dev->d1_rect.x0 = fz_min(csi->stack[2], csi->stack[4]);
+	pr->dev->d1_rect.y0 = fz_min(csi->stack[3], csi->stack[5]);
+	pr->dev->d1_rect.x1 = fz_max(csi->stack[2], csi->stack[4]);
+	pr->dev->d1_rect.y1 = fz_max(csi->stack[3], csi->stack[5]);
 }
 
 static void pdf_run_f(pdf_csi *csi, void *state)