Bug 693503: Fix SEGV in glyph painting due to bbox overflow.

When calculating the bbox for draw_glyph, if the x and y origins of
the glyph are extreme (too large to fit in an int), we get overflows
of the bbox; empty bboxes are transformed to large ones.

The fix is to introduce an fz_translate_bbox function that checks for
such things.

Also, we update various bbox/rect functions to check for empty bboxes
before they check for infinite ones (as a bbox of x0=0 x1=0 y0=0 y1=-1
will be detected both as infinite and empty).

Problem found in 2485.pdf.SIGSEGV.2a.1652, a test file supplied by
Mateusz "j00ru" Jurczyk and Gynvael Coldwind of the Google Security
Team using Address Sanitizer. Many thanks!

1 files changed, 52 insertions, 8 deletions

diff --git a/fitz/base_geometry.c b/fitz/base_geometry.c
index 27cc3cf3..70ba814c 100644
--- a/fitz/base_geometry.c
+++ b/fitz/base_geometry.c
@@ -205,10 +205,11 @@ fz_rect
 fz_intersect_rect(fz_rect a, fz_rect b)
 {
 	fz_rect r;
-	if (fz_is_infinite_rect(a)) return b;
-	if (fz_is_infinite_rect(b)) return a;
+	/* Check for empty box before infinite box */
 	if (fz_is_empty_rect(a)) return fz_empty_rect;
 	if (fz_is_empty_rect(b)) return fz_empty_rect;
+	if (fz_is_infinite_rect(a)) return b;
+	if (fz_is_infinite_rect(b)) return a;
 	r.x0 = fz_max(a.x0, b.x0);
 	r.y0 = fz_max(a.y0, b.y0);
 	r.x1 = fz_min(a.x1, b.x1);
@@ -220,10 +221,11 @@ fz_rect
 fz_union_rect(fz_rect a, fz_rect b)
 {
 	fz_rect r;
-	if (fz_is_infinite_rect(a)) return a;
-	if (fz_is_infinite_rect(b)) return b;
+	/* Check for empty box before infinite box */
 	if (fz_is_empty_rect(a)) return b;
 	if (fz_is_empty_rect(b)) return a;
+	if (fz_is_infinite_rect(a)) return a;
+	if (fz_is_infinite_rect(b)) return b;
 	r.x0 = fz_min(a.x0, b.x0);
 	r.y0 = fz_min(a.y0, b.y0);
 	r.x1 = fz_max(a.x1, b.x1);
@@ -235,10 +237,11 @@ fz_bbox
 fz_intersect_bbox(fz_bbox a, fz_bbox b)
 {
 	fz_bbox r;
-	if (fz_is_infinite_rect(a)) return b;
-	if (fz_is_infinite_rect(b)) return a;
+	/* Check for empty box before infinite box */
 	if (fz_is_empty_rect(a)) return fz_empty_bbox;
 	if (fz_is_empty_rect(b)) return fz_empty_bbox;
+	if (fz_is_infinite_rect(a)) return b;
+	if (fz_is_infinite_rect(b)) return a;
 	r.x0 = fz_maxi(a.x0, b.x0);
 	r.y0 = fz_maxi(a.y0, b.y0);
 	r.x1 = fz_mini(a.x1, b.x1);
@@ -247,13 +250,54 @@ fz_intersect_bbox(fz_bbox a, fz_bbox b)
 }
 
 fz_bbox
+fz_translate_bbox(fz_bbox a, int xoff, int yoff)
+{
+	fz_bbox b;
+	b.x0 = a.x0 + xoff;
+	b.y0 = a.y0 + yoff;
+	b.x1 = a.x1 + xoff;
+	b.y1 = a.y1 + yoff;
+	/* Check for overflow */
+	if (((~a.x0^xoff)&(a.x0^b.x0)) < 0)
+	{
+		if (xoff < 0)
+			b.x0 = INT_MIN;
+		else
+			b.x0 = INT_MAX;
+	}
+	if (((~a.x1^xoff)&(a.x1^b.x1)) < 0)
+	{
+		if (xoff < 0)
+			b.x1 = INT_MIN;
+		else
+			b.x1 = INT_MAX;
+	}
+	if (((~a.y0^yoff)&(a.y0^b.y0)) < 0)
+	{
+		if (yoff < 0)
+			b.y0 = INT_MIN;
+		else
+			b.y0 = INT_MAX;
+	}
+	if (((~a.y1^yoff)&(a.y1^b.y1)) < 0)
+	{
+		if (yoff < 0)
+			b.y1 = INT_MIN;
+		else
+			b.y1 = INT_MAX;
+	}
+	return b;
+}
+
+fz_bbox
 fz_union_bbox(fz_bbox a, fz_bbox b)
 {
 	fz_bbox r;
-	if (fz_is_infinite_rect(a)) return a;
-	if (fz_is_infinite_rect(b)) return b;
+	/* Check for empty box before infinite box */
 	if (fz_is_empty_rect(a)) return b;
 	if (fz_is_empty_rect(b)) return a;
+	if (fz_is_infinite_rect(a)) return a;
+	if (fz_is_infinite_rect(b)) return b;
 	r.x0 = fz_mini(a.x0, b.x0);
 	r.y0 = fz_mini(a.y0, b.y0);
 	r.x1 = fz_maxi(a.x1, b.x1);