From 4fddb35e247a2d81b9b78ca3543b97da9e9fce45 Mon Sep 17 00:00:00 2001
From: Robin Watts <robin@peeves.(none)>
Date: Mon, 11 Jun 2012 11:49:31 -0700
Subject: Fix Bug 693102: Overflows in large pixmap indexing.

When we allocate a pixmap > 2G, but < 4G, the index into that
pixmap, when calculated as an int can be negative. Fix this with
various casts to unsigned int.

If we ever move to support >4G images we'll need to rejig the
casting to cast each part of the element to ptrdiff_t first.
---
 fitz/image_png.c    | 10 +++++-----
 fitz/image_tiff.c   |  8 ++++----
 fitz/res_font.c     |  8 ++++----
 fitz/res_halftone.c |  2 +-
 fitz/res_pixmap.c   | 16 +++++++++-------
 5 files changed, 23 insertions(+), 21 deletions(-)

(limited to 'fitz')

diff --git a/fitz/image_png.c b/fitz/image_png.c
index aeb33137..9e3f39c5 100644
--- a/fitz/image_png.c
+++ b/fitz/image_png.c
@@ -88,8 +88,8 @@ png_predict(unsigned char *samples, int width, int height, int n, int depth)
 
 	for (row = 0; row < height; row ++)
 	{
-		unsigned char *src = samples + (stride + 1) * row;
-		unsigned char *dst = samples + stride * row;
+		unsigned char *src = samples + (unsigned int)((stride + 1) * row);
+		unsigned char *dst = samples + (unsigned int)(stride * row);
 
 		unsigned char *a = dst;
 		unsigned char *b = dst - stride;
@@ -192,7 +192,7 @@ png_deinterlace(struct info *info, int *passw, int *passh, int *passofs)
 
 	for (p = 0; p < 7; p++)
 	{
-		unsigned char *sp = info->samples + passofs[p];
+	  unsigned char *sp = info->samples + (unsigned int)(passofs[p]);
 		int w = passw[p];
 		int h = passh[p];
 
@@ -521,8 +521,8 @@ png_mask_transparency(struct info *info, fz_pixmap *dst)
 
 	for (y = 0; y < info->height; y++)
 	{
-		unsigned char *sp = info->samples + y * stride;
-		unsigned char *dp = dst->samples + y * dst->w * dst->n;
+		unsigned char *sp = info->samples + (unsigned int)(y * stride);
+		unsigned char *dp = dst->samples + (unsigned int)(y * dst->w * dst->n);
 		for (x = 0; x < info->width; x++)
 		{
 			t = 1;
diff --git a/fitz/image_tiff.c b/fitz/image_tiff.c
index de4c0732..f79e8672 100644
--- a/fitz/image_tiff.c
+++ b/fitz/image_tiff.c
@@ -286,8 +286,8 @@ fz_expand_tiff_colormap(struct tiff *tiff)
 
 	for (y = 0; y < tiff->imagelength; y++)
 	{
-		src = tiff->samples + (tiff->stride * y);
-		dst = samples + (stride * y);
+		src = tiff->samples + (unsigned int)(tiff->stride * y);
+		dst = samples + (unsigned int)(stride * y);
 
 		for (x = 0; x < tiff->imagewidth; x++)
 		{
@@ -403,8 +403,8 @@ fz_decode_tiff_strips(struct tiff *tiff)
 		unsigned wlen = tiff->stride * tiff->rowsperstrip;
 		unsigned char *rp = tiff->bp + offset;
 
-		if (wp + wlen > tiff->samples + tiff->stride * tiff->imagelength)
-			wlen = tiff->samples + tiff->stride * tiff->imagelength - wp;
+		if (wp + wlen > tiff->samples + (unsigned int)(tiff->stride * tiff->imagelength))
+			wlen = tiff->samples + (unsigned int)(tiff->stride * tiff->imagelength) - wp;
 
 		if (rp + rlen > tiff->ep)
 			fz_throw(tiff->ctx, "strip extends beyond the end of the file");
diff --git a/fitz/res_font.c b/fitz/res_font.c
index 966cbc6e..25939d02 100644
--- a/fitz/res_font.c
+++ b/fitz/res_font.c
@@ -356,8 +356,8 @@ fz_copy_ft_bitmap(fz_context *ctx, int left, int top, FT_Bitmap *bitmap)
 	{
 		for (y = 0; y < pixmap->h; y++)
 		{
-			unsigned char *out = pixmap->samples + y * pixmap->w;
-			unsigned char *in = bitmap->buffer + (pixmap->h - y - 1) * bitmap->pitch;
+			unsigned char *out = pixmap->samples + (unsigned int)(y * pixmap->w);
+			unsigned char *in = bitmap->buffer + (unsigned int)((pixmap->h - y - 1) * bitmap->pitch);
 			unsigned char bit = 0x80;
 			int w = pixmap->w;
 			while (w--)
@@ -376,8 +376,8 @@ fz_copy_ft_bitmap(fz_context *ctx, int left, int top, FT_Bitmap *bitmap)
 	{
 		for (y = 0; y < pixmap->h; y++)
 		{
-			memcpy(pixmap->samples + y * pixmap->w,
-				bitmap->buffer + (pixmap->h - y - 1) * bitmap->pitch,
+			memcpy(pixmap->samples + (unsigned int)(y * pixmap->w),
+			       bitmap->buffer + (unsigned int)((pixmap->h - y - 1) * bitmap->pitch),
 				pixmap->w);
 		}
 	}
diff --git a/fitz/res_halftone.c b/fitz/res_halftone.c
index 3b5e0e51..f8bef236 100644
--- a/fitz/res_halftone.c
+++ b/fitz/res_halftone.c
@@ -99,7 +99,7 @@ static void make_ht_line(unsigned char *buf, fz_halftone *ht, int x, int y, int
 		assert(tile->n == 1);
 
 		/* Left hand section; from x to tile width */
-		tbase = tile->samples + py * tw;
+		tbase = tile->samples + (unsigned int)(py * tw);
 		t = tbase + px;
 		len = tw - px;
 		if (len > w2)
diff --git a/fitz/res_pixmap.c b/fitz/res_pixmap.c
index 29dd95a0..b7cf5f58 100644
--- a/fitz/res_pixmap.c
+++ b/fitz/res_pixmap.c
@@ -136,14 +136,16 @@ fz_pixmap_height(fz_context *ctx, fz_pixmap *pix)
 void
 fz_clear_pixmap(fz_context *ctx, fz_pixmap *pix)
 {
-	memset(pix->samples, 0, pix->w * pix->h * pix->n);
+  memset(pix->samples, 0, (unsigned int)(pix->w * pix->h * pix->n));
 }
 
 void
 fz_clear_pixmap_with_value(fz_context *ctx, fz_pixmap *pix, int value)
 {
 	if (value == 255)
-		memset(pix->samples, 255, pix->w * pix->h * pix->n);
+	{
+		memset(pix->samples, 255, (unsigned int)(pix->w * pix->h * pix->n));
+	}
 	else
 	{
 		int k, x, y;
@@ -175,9 +177,9 @@ fz_copy_pixmap_rect(fz_context *ctx, fz_pixmap *dest, fz_pixmap *src, fz_bbox r)
 		return;
 
 	srcspan = src->w * src->n;
-	srcp = src->samples + srcspan * (r.y0 - src->y) + src->n * (r.x0 - src->x);
+	srcp = src->samples + (unsigned int)(srcspan * (r.y0 - src->y) + src->n * (r.x0 - src->x));
 	destspan = dest->w * dest->n;
-	destp = dest->samples + destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x);
+	destp = dest->samples + (unsigned int)(destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x));
 
 	if (src->n == dest->n)
 	{
@@ -273,11 +275,11 @@ fz_clear_pixmap_rect_with_value(fz_context *ctx, fz_pixmap *dest, int value, fz_
 		return;
 
 	destspan = dest->w * dest->n;
-	destp = dest->samples + destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x);
+	destp = dest->samples + (unsigned int)(destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x));
 	if (value == 255)
 		do
 		{
-			memset(destp, 255, w * dest->n);
+			memset(destp, 255, (unsigned int)(w * dest->n));
 			destp += destspan;
 		}
 		while (--y);
@@ -389,7 +391,7 @@ void fz_invert_pixmap_rect(fz_pixmap *image, fz_bbox rect)
 
 	for (y = y0; y < y1; y++)
 	{
-		p = image->samples + (y * image->w + x0) * image->n;
+		p = image->samples + (unsigned int)((y * image->w + x0) * image->n);
 		for (x = x0; x < x1; x++)
 		{
 			for (n = image->n; n > 0; n--, p++)
-- 
cgit v1.2.3