Bug 593603: Fix problems with tiling.

Two problems with tiling are fixed here.

Firstly, if the tiling bounds are huge, the 'patch' region (the region
we are writing into), can overflow, causing a SEGV due to the paint code
being very confused by pixmaps that go from just under INT_MAX to just
over INT_MIN. Fix this by checking explicitly for overflow in these
bounds.

If the tiles are stupidly huge, but the scissor is small, we can end up
looping many more times than we need to. We fix mapping the scissor
region back through the inverse transform, and intersecting this
with the pattern area.

Problem found in 4201.pdf.SIGSEGV.622.3560, a test file supplied by
Mateusz "j00ru" Jurczyk and Gynvael Coldwind of the Google Security
Team using Address Sanitizer. Many thanks!

3 files changed, 69 insertions, 2 deletions

diff --git a/draw/draw_device.c b/draw/draw_device.c
index 605a335e..2b6f62d8 100644
--- a/draw/draw_device.c
+++ b/draw/draw_device.c
@@ -1647,8 +1647,9 @@ fz_draw_end_tile(fz_device *devp)
 {
 	fz_draw_device *dev = devp->user;
 	float xstep, ystep;
-	fz_matrix ctm, ttm, shapectm;
-	fz_rect area;
+	fz_matrix ttm, ctm, shapectm;
+	fz_rect area, scissor;
+	fz_bbox scissor_box;
 	int x0, y0, x1, y1, x, y;
 	fz_context *ctx = dev->ctx;
 	fz_draw_state *state;
@@ -1665,6 +1666,16 @@ fz_draw_end_tile(fz_device *devp)
 	area = state[1].area;
 	ctm = state[1].ctm;
 
+	/* Fudge the scissor bbox a little to allow for inaccuracies in the
+	 * matrix inversion. */
+	scissor_box = fz_expand_bbox(state[0].scissor, 1);
+	scissor.x0 = (float)scissor_box.x0;
+	scissor.y0 = (float)scissor_box.y0;
+	scissor.x1 = (float)scissor_box.x1;
+	scissor.y1 = (float)scissor_box.y1;
+	scissor = fz_transform_rect(fz_invert_matrix(ctm), scissor);
+	area = fz_intersect_rect(area, scissor);
+
 	x0 = floorf(area.x0 / xstep);
 	y0 = floorf(area.y0 / ystep);
 	x1 = ceilf(area.x1 / xstep);
@@ -1696,6 +1707,10 @@ fz_draw_end_tile(fz_device *devp)
 			ttm = fz_concat(fz_translate(x * xstep, y * ystep), ctm);
 			state[1].dest->x = ttm.e;
 			state[1].dest->y = ttm.f;
+			if (state[1].dest->x > 0 && state[1].dest->x + state[1].dest->w < 0)
+				continue;
+			if (state[1].dest->y > 0 && state[1].dest->y + state[1].dest->h < 0)
+				continue;
 			fz_paint_pixmap_with_rect(state[0].dest, state[1].dest, 255, state[0].scissor);
 			if (state[1].shape)
 			{
diff --git a/fitz/base_geometry.c b/fitz/base_geometry.c
index 70ba814c..3c48d4c6 100644
--- a/fitz/base_geometry.c
+++ b/fitz/base_geometry.c
@@ -351,3 +351,48 @@ fz_transform_bbox(fz_matrix m, fz_bbox b)
 	return b;
 
 }
+
+fz_bbox
+fz_expand_bbox(fz_bbox a, int expand)
+{
+	fz_bbox b;
+
+	if (fz_is_infinite_bbox(a))
+		return a;
+
+	b.x0 = a.x0 - expand;
+	b.y0 = a.y0 - expand;
+	b.x1 = a.x1 + expand;
+	b.y1 = a.y1 + expand;
+	/* Check for overflow */
+	if (((~a.x0^-expand)&(a.x0^b.x0)) < 0)
+	{
+		if (-expand < 0)
+			b.x0 = INT_MIN;
+		else
+			b.x0 = INT_MAX;
+	}
+	if (((~a.x1^expand)&(a.x1^b.x1)) < 0)
+	{
+		if (expand < 0)
+			b.x1 = INT_MIN;
+		else
+			b.x1 = INT_MAX;
+	}
+	if (((~a.y0^-expand)&(a.y0^b.y0)) < 0)
+	{
+		if (-expand < 0)
+			b.y0 = INT_MIN;
+		else
+			b.y0 = INT_MAX;
+	}
+	if (((~a.y1^expand)&(a.y1^b.y1)) < 0)
+	{
+		if (expand < 0)
+			b.y1 = INT_MIN;
+		else
+			b.y1 = INT_MAX;
+	}
+	return b;
+
+}
diff --git a/fitz/fitz.h b/fitz/fitz.h
index 19732e42..1bbedd05 100644
--- a/fitz/fitz.h
+++ b/fitz/fitz.h
@@ -1029,6 +1029,13 @@ fz_rect fz_union_rect(fz_rect a, fz_rect b);
 fz_bbox fz_union_bbox(fz_bbox a, fz_bbox b);
 
 /*
+	fz_expand_bbox: Expand a bbox by a given amount in all directions.
+
+	Does not throw exceptions.
+*/
+fz_bbox fz_expand_bbox(fz_bbox b, int expand);
+
+/*
 	fz_translate_bbox: Translate bounding box.
 
 	Translate a bbox by a given x and y offset. Allows for overflow.