Fix Bug 693102: Overflows in large pixmap indexing.

When we allocate a pixmap > 2G, but < 4G, the index into that
pixmap, when calculated as an int can be negative. Fix this with
various casts to unsigned int.

If we ever move to support >4G images we'll need to rejig the
casting to cast each part of the element to ptrdiff_t first.

12 files changed, 44 insertions, 42 deletions

diff --git a/draw/draw_affine.c b/draw/draw_affine.c
index 52b8a847..67dbd37d 100644
--- a/draw/draw_affine.c
+++ b/draw/draw_affine.c
@@ -674,7 +674,7 @@ fz_paint_image_imp(fz_pixmap *dst, fz_bbox scissor, fz_pixmap *shape, fz_pixmap
 		v -= 32768;
 	}
 
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * dst->n;
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * dst->n);
 	n = dst->n;
 	sp = img->samples;
 	sw = img->w;
@@ -682,7 +682,7 @@ fz_paint_image_imp(fz_pixmap *dst, fz_bbox scissor, fz_pixmap *shape, fz_pixmap
 	if (shape)
 	{
 		hw = shape->w;
-		hp = shape->samples + ((y - shape->y) * hw) + x - shape->x;
+		hp = shape->samples + (unsigned int)(((y - shape->y) * hw) + x - shape->x);
 	}
 	else
 	{
diff --git a/draw/draw_blend.c b/draw/draw_blend.c
index 7c74df79..8a751c08 100644
--- a/draw/draw_blend.c
+++ b/draw/draw_blend.c
@@ -598,14 +598,14 @@ fz_blend_pixmap(fz_pixmap *dst, fz_pixmap *src, int alpha, int blendmode, int is
 	h = bbox.y1 - bbox.y0;
 
 	n = src->n;
-	sp = src->samples + ((y - src->y) * src->w + (x - src->x)) * n;
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * n;
+	sp = src->samples + (unsigned int)(((y - src->y) * src->w + (x - src->x)) * n);
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * n);
 
 	assert(src->n == dst->n);
 
 	if (!isolated)
 	{
-		unsigned char *hp = shape->samples + (y - shape->y) * shape->w + (x - shape->x);
+		unsigned char *hp = shape->samples + (unsigned int)((y - shape->y) * shape->w + (x - shape->x));
 
 		while (h--)
 		{
diff --git a/draw/draw_device.c b/draw/draw_device.c
index 3df440ef..6f74eabf 100644
--- a/draw/draw_device.c
+++ b/draw/draw_device.c
@@ -439,8 +439,8 @@ draw_glyph(unsigned char *colorbv, fz_pixmap *dst, fz_pixmap *msk,
 	w = bbox.x1 - bbox.x0;
 	h = bbox.y1 - bbox.y0;
 
-	mp = msk->samples + ((y - msk->y - yorig) * msk->w + (x - msk->x - xorig));
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * dst->n;
+	mp = msk->samples + (unsigned int)((y - msk->y - yorig) * msk->w + (x - msk->x - xorig));
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * dst->n);
 
 	assert(msk->n == 1);
 
@@ -783,7 +783,7 @@ fz_draw_fill_shade(fz_device *devp, fz_shade *shade, fz_matrix ctm, float alpha)
 		n = dest->n;
 		for (y = scissor.y0; y < scissor.y1; y++)
 		{
-			s = dest->samples + ((scissor.x0 - dest->x) + (y - dest->y) * dest->w) * dest->n;
+			s = dest->samples + (unsigned int)(((scissor.x0 - dest->x) + (y - dest->y) * dest->w) * dest->n);
 			for (x = scissor.x0; x < scissor.x1; x++)
 			{
 				for (i = 0; i < n; i++)
@@ -794,7 +794,7 @@ fz_draw_fill_shade(fz_device *devp, fz_shade *shade, fz_matrix ctm, float alpha)
 		{
 			for (y = scissor.y0; y < scissor.y1; y++)
 			{
-				s = shape->samples + (scissor.x0 - shape->x) + (y - shape->y) * shape->w;
+				s = shape->samples + (unsigned int)((scissor.x0 - shape->x) + (y - shape->y) * shape->w);
 				for (x = scissor.x0; x < scissor.x1; x++)
 				{
 					*s++ = 255;
diff --git a/draw/draw_edge.c b/draw/draw_edge.c
index 53b67b73..eac7cfba 100644
--- a/draw/draw_edge.c
+++ b/draw/draw_edge.c
@@ -624,7 +624,7 @@ static inline void blit_aa(fz_pixmap *dst, int x, int y,
 	unsigned char *mp, int w, unsigned char *color)
 {
 	unsigned char *dp;
-	dp = dst->samples + ( (y - dst->y) * dst->w + (x - dst->x) ) * dst->n;
+	dp = dst->samples + (unsigned int)(( (y - dst->y) * dst->w + (x - dst->x) ) * dst->n);
 	if (color)
 		fz_paint_span_with_color(dp, mp, dst->n, w, color);
 	else
@@ -725,7 +725,7 @@ static inline void blit_sharp(int x0, int x1, int y,
 	x1 = CLAMP(x1, dst->x, dst->x + dst->w);
 	if (x0 < x1)
 	{
-		dp = dst->samples + ( (y - dst->y) * dst->w + (x0 - dst->x) ) * dst->n;
+		dp = dst->samples + (unsigned int)(( (y - dst->y) * dst->w + (x0 - dst->x) ) * dst->n);
 		if (color)
 			fz_paint_solid_color(dp, dst->n, x1 - x0, color);
 		else
diff --git a/draw/draw_mesh.c b/draw/draw_mesh.c
index 025ec366..924bd34d 100644
--- a/draw/draw_mesh.c
+++ b/draw/draw_mesh.c
@@ -269,7 +269,7 @@ static int clip_poly(float src[MAXV][MAXN],
 
 static void paint_scan(fz_pixmap *pix, int y, int x1, int x2, int *v1, int *v2, int n)
 {
-	unsigned char *p = pix->samples + ((y - pix->y) * pix->w + (x1 - pix->x)) * pix->n;
+	unsigned char *p = pix->samples + (unsigned int)(((y - pix->y) * pix->w + (x1 - pix->x)) * pix->n);
 	int v[FZ_MAX_COLORS];
 	int dv[FZ_MAX_COLORS];
 	int w = x2 - x1;
diff --git a/draw/draw_paint.c b/draw/draw_paint.c
index 2a5f9607..57e69748 100644
--- a/draw/draw_paint.c
+++ b/draw/draw_paint.c
@@ -393,8 +393,8 @@ fz_paint_pixmap_with_rect(fz_pixmap *dst, fz_pixmap *src, int alpha, fz_bbox bbo
 		return;
 
 	n = src->n;
-	sp = src->samples + ((y - src->y) * src->w + (x - src->x)) * src->n;
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * dst->n;
+	sp = src->samples + (unsigned int)(((y - src->y) * src->w + (x - src->x)) * src->n);
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * dst->n);
 
 	while (h--)
 	{
@@ -424,8 +424,8 @@ fz_paint_pixmap(fz_pixmap *dst, fz_pixmap *src, int alpha)
 		return;
 
 	n = src->n;
-	sp = src->samples + ((y - src->y) * src->w + (x - src->x)) * src->n;
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * dst->n;
+	sp = src->samples + (unsigned int)(((y - src->y) * src->w + (x - src->x)) * src->n);
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * dst->n);
 
 	while (h--)
 	{
@@ -457,9 +457,9 @@ fz_paint_pixmap_with_mask(fz_pixmap *dst, fz_pixmap *src, fz_pixmap *msk)
 		return;
 
 	n = src->n;
-	sp = src->samples + ((y - src->y) * src->w + (x - src->x)) * src->n;
-	mp = msk->samples + ((y - msk->y) * msk->w + (x - msk->x)) * msk->n;
-	dp = dst->samples + ((y - dst->y) * dst->w + (x - dst->x)) * dst->n;
+	sp = src->samples + (unsigned int)(((y - src->y) * src->w + (x - src->x)) * src->n);
+	mp = msk->samples + (unsigned int)(((y - msk->y) * msk->w + (x - msk->x)) * msk->n);
+	dp = dst->samples + (unsigned int)(((y - dst->y) * dst->w + (x - dst->x)) * dst->n);
 
 	while (h--)
 	{
diff --git a/draw/draw_unpack.c b/draw/draw_unpack.c
index f988dcf9..39c287c6 100644
--- a/draw/draw_unpack.c
+++ b/draw/draw_unpack.c
@@ -70,8 +70,8 @@ fz_unpack_tile(fz_pixmap *dst, unsigned char * restrict src, int n, int depth, i
 
 	for (y = 0; y < dst->h; y++)
 	{
-		unsigned char *sp = src + y * stride;
-		unsigned char *dp = dst->samples + y * (dst->w * dst->n);
+		unsigned char *sp = src + (unsigned int)(y * stride);
+		unsigned char *dp = dst->samples + (unsigned int)(y * dst->w * dst->n);
 
 		/* Specialized loops */
 
diff --git a/fitz/image_png.c b/fitz/image_png.c
index aeb33137..9e3f39c5 100644
--- a/fitz/image_png.c
+++ b/fitz/image_png.c
@@ -88,8 +88,8 @@ png_predict(unsigned char *samples, int width, int height, int n, int depth)
 
 	for (row = 0; row < height; row ++)
 	{
-		unsigned char *src = samples + (stride + 1) * row;
-		unsigned char *dst = samples + stride * row;
+		unsigned char *src = samples + (unsigned int)((stride + 1) * row);
+		unsigned char *dst = samples + (unsigned int)(stride * row);
 
 		unsigned char *a = dst;
 		unsigned char *b = dst - stride;
@@ -192,7 +192,7 @@ png_deinterlace(struct info *info, int *passw, int *passh, int *passofs)
 
 	for (p = 0; p < 7; p++)
 	{
-		unsigned char *sp = info->samples + passofs[p];
+	  unsigned char *sp = info->samples + (unsigned int)(passofs[p]);
 		int w = passw[p];
 		int h = passh[p];
 
@@ -521,8 +521,8 @@ png_mask_transparency(struct info *info, fz_pixmap *dst)
 
 	for (y = 0; y < info->height; y++)
 	{
-		unsigned char *sp = info->samples + y * stride;
-		unsigned char *dp = dst->samples + y * dst->w * dst->n;
+		unsigned char *sp = info->samples + (unsigned int)(y * stride);
+		unsigned char *dp = dst->samples + (unsigned int)(y * dst->w * dst->n);
 		for (x = 0; x < info->width; x++)
 		{
 			t = 1;
diff --git a/fitz/image_tiff.c b/fitz/image_tiff.c
index de4c0732..f79e8672 100644
--- a/fitz/image_tiff.c
+++ b/fitz/image_tiff.c
@@ -286,8 +286,8 @@ fz_expand_tiff_colormap(struct tiff *tiff)
 
 	for (y = 0; y < tiff->imagelength; y++)
 	{
-		src = tiff->samples + (tiff->stride * y);
-		dst = samples + (stride * y);
+		src = tiff->samples + (unsigned int)(tiff->stride * y);
+		dst = samples + (unsigned int)(stride * y);
 
 		for (x = 0; x < tiff->imagewidth; x++)
 		{
@@ -403,8 +403,8 @@ fz_decode_tiff_strips(struct tiff *tiff)
 		unsigned wlen = tiff->stride * tiff->rowsperstrip;
 		unsigned char *rp = tiff->bp + offset;
 
-		if (wp + wlen > tiff->samples + tiff->stride * tiff->imagelength)
-			wlen = tiff->samples + tiff->stride * tiff->imagelength - wp;
+		if (wp + wlen > tiff->samples + (unsigned int)(tiff->stride * tiff->imagelength))
+			wlen = tiff->samples + (unsigned int)(tiff->stride * tiff->imagelength) - wp;
 
 		if (rp + rlen > tiff->ep)
 			fz_throw(tiff->ctx, "strip extends beyond the end of the file");
diff --git a/fitz/res_font.c b/fitz/res_font.c
index 966cbc6e..25939d02 100644
--- a/fitz/res_font.c
+++ b/fitz/res_font.c
@@ -356,8 +356,8 @@ fz_copy_ft_bitmap(fz_context *ctx, int left, int top, FT_Bitmap *bitmap)
 	{
 		for (y = 0; y < pixmap->h; y++)
 		{
-			unsigned char *out = pixmap->samples + y * pixmap->w;
-			unsigned char *in = bitmap->buffer + (pixmap->h - y - 1) * bitmap->pitch;
+			unsigned char *out = pixmap->samples + (unsigned int)(y * pixmap->w);
+			unsigned char *in = bitmap->buffer + (unsigned int)((pixmap->h - y - 1) * bitmap->pitch);
 			unsigned char bit = 0x80;
 			int w = pixmap->w;
 			while (w--)
@@ -376,8 +376,8 @@ fz_copy_ft_bitmap(fz_context *ctx, int left, int top, FT_Bitmap *bitmap)
 	{
 		for (y = 0; y < pixmap->h; y++)
 		{
-			memcpy(pixmap->samples + y * pixmap->w,
-				bitmap->buffer + (pixmap->h - y - 1) * bitmap->pitch,
+			memcpy(pixmap->samples + (unsigned int)(y * pixmap->w),
+			       bitmap->buffer + (unsigned int)((pixmap->h - y - 1) * bitmap->pitch),
 				pixmap->w);
 		}
 	}
diff --git a/fitz/res_halftone.c b/fitz/res_halftone.c
index 3b5e0e51..f8bef236 100644
--- a/fitz/res_halftone.c
+++ b/fitz/res_halftone.c
@@ -99,7 +99,7 @@ static void make_ht_line(unsigned char *buf, fz_halftone *ht, int x, int y, int
 		assert(tile->n == 1);
 
 		/* Left hand section; from x to tile width */
-		tbase = tile->samples + py * tw;
+		tbase = tile->samples + (unsigned int)(py * tw);
 		t = tbase + px;
 		len = tw - px;
 		if (len > w2)
diff --git a/fitz/res_pixmap.c b/fitz/res_pixmap.c
index 29dd95a0..b7cf5f58 100644
--- a/fitz/res_pixmap.c
+++ b/fitz/res_pixmap.c
@@ -136,14 +136,16 @@ fz_pixmap_height(fz_context *ctx, fz_pixmap *pix)
 void
 fz_clear_pixmap(fz_context *ctx, fz_pixmap *pix)
 {
-	memset(pix->samples, 0, pix->w * pix->h * pix->n);
+  memset(pix->samples, 0, (unsigned int)(pix->w * pix->h * pix->n));
 }
 
 void
 fz_clear_pixmap_with_value(fz_context *ctx, fz_pixmap *pix, int value)
 {
 	if (value == 255)
-		memset(pix->samples, 255, pix->w * pix->h * pix->n);
+	{
+		memset(pix->samples, 255, (unsigned int)(pix->w * pix->h * pix->n));
+	}
 	else
 	{
 		int k, x, y;
@@ -175,9 +177,9 @@ fz_copy_pixmap_rect(fz_context *ctx, fz_pixmap *dest, fz_pixmap *src, fz_bbox r)
 		return;
 
 	srcspan = src->w * src->n;
-	srcp = src->samples + srcspan * (r.y0 - src->y) + src->n * (r.x0 - src->x);
+	srcp = src->samples + (unsigned int)(srcspan * (r.y0 - src->y) + src->n * (r.x0 - src->x));
 	destspan = dest->w * dest->n;
-	destp = dest->samples + destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x);
+	destp = dest->samples + (unsigned int)(destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x));
 
 	if (src->n == dest->n)
 	{
@@ -273,11 +275,11 @@ fz_clear_pixmap_rect_with_value(fz_context *ctx, fz_pixmap *dest, int value, fz_
 		return;
 
 	destspan = dest->w * dest->n;
-	destp = dest->samples + destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x);
+	destp = dest->samples + (unsigned int)(destspan * (r.y0 - dest->y) + dest->n * (r.x0 - dest->x));
 	if (value == 255)
 		do
 		{
-			memset(destp, 255, w * dest->n);
+			memset(destp, 255, (unsigned int)(w * dest->n));
 			destp += destspan;
 		}
 		while (--y);
@@ -389,7 +391,7 @@ void fz_invert_pixmap_rect(fz_pixmap *image, fz_bbox rect)
 
 	for (y = y0; y < y1; y++)
 	{
-		p = image->samples + (y * image->w + x0) * image->n;
+		p = image->samples + (unsigned int)((y * image->w + x0) * image->n);
 		for (x = x0; x < x1; x++)
 		{
 			for (n = image->n; n > 0; n--, p++)