summaryrefslogtreecommitdiff
path: root/source/fitz
diff options
context:
space:
mode:
authorRobin Watts <robin.watts@artifex.com>2016-06-06 15:29:16 +0100
committerRobin Watts <robin.watts@artifex.com>2016-06-06 15:30:21 +0100
commitef6bc91b369a89a3da77f93ed6cc56fef22a193b (patch)
treeae715a36c000b117f65e8bf55ce1dced9ca0cc45 /source/fitz
parent6df3706d77b2db7368313693e9a45307320e1253 (diff)
downloadmupdf-ef6bc91b369a89a3da77f93ed6cc56fef22a193b.tar.xz
Bug 696810: Avoid integer overflows.
When blanking pixmaps (or converting pixmap colors), watch out for integer overflows.
Diffstat (limited to 'source/fitz')
-rw-r--r--source/fitz/colorspace.c130
-rw-r--r--source/fitz/pixmap.c23
2 files changed, 98 insertions, 55 deletions
diff --git a/source/fitz/colorspace.c b/source/fitz/colorspace.c
index c07d322b..89f16e30 100644
--- a/source/fitz/colorspace.c
+++ b/source/fitz/colorspace.c
@@ -289,11 +289,14 @@ static void fast_gray_to_rgb(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 3);
int s_line_inc = src->stride - w * (src->alpha + 1);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -306,7 +309,7 @@ static void fast_gray_to_rgb(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[0];
@@ -324,7 +327,7 @@ static void fast_gray_to_rgb(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[0];
@@ -345,7 +348,7 @@ static void fast_gray_to_rgb(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[0];
@@ -364,11 +367,14 @@ static void fast_gray_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 4);
int s_line_inc = src->stride - w * (src->alpha + 1);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -381,7 +387,7 @@ static void fast_gray_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = 0;
@@ -400,7 +406,7 @@ static void fast_gray_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = 0;
@@ -422,7 +428,7 @@ static void fast_gray_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = 0;
@@ -442,11 +448,14 @@ static void fast_rgb_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 1);
int s_line_inc = src->stride - w * (src->alpha + 3);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -459,7 +468,7 @@ static void fast_rgb_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 77 + (s[1]+1) * 150 + (s[2]+1) * 28) >> 8;
@@ -475,7 +484,7 @@ static void fast_rgb_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 77 + (s[1]+1) * 150 + (s[2]+1) * 28) >> 8;
@@ -494,7 +503,7 @@ static void fast_rgb_to_gray(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 77 + (s[1]+1) * 150 + (s[2]+1) * 28) >> 8;
@@ -511,11 +520,14 @@ static void fast_bgr_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 1);
int s_line_inc = src->stride - w * (src->alpha + 3);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -528,7 +540,7 @@ static void fast_bgr_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 28 + (s[1]+1) * 150 + (s[2]+1) * 77) >> 8;
@@ -544,7 +556,7 @@ static void fast_bgr_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 28 + (s[1]+1) * 150 + (s[2]+1) * 77) >> 8;
@@ -563,7 +575,7 @@ static void fast_bgr_to_gray(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = ((s[0]+1) * 28 + (s[1]+1) * 150 + (s[2]+1) * 77) >> 8;
@@ -580,11 +592,14 @@ static void fast_rgb_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 4);
int s_line_inc = src->stride - w * (src->alpha + 3);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -597,7 +612,7 @@ static void fast_rgb_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[0];
@@ -620,7 +635,7 @@ static void fast_rgb_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[0];
@@ -646,7 +661,7 @@ static void fast_rgb_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[0];
@@ -670,11 +685,14 @@ static void fast_bgr_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 4);
int s_line_inc = src->stride - w * (src->alpha + 3);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -687,7 +705,7 @@ static void fast_bgr_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[2];
@@ -710,7 +728,7 @@ static void fast_bgr_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[2];
@@ -736,7 +754,7 @@ static void fast_bgr_to_cmyk(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = 255 - s[2];
@@ -760,11 +778,14 @@ static void fast_cmyk_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 1);
int s_line_inc = src->stride - w * (src->alpha + 4);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -777,7 +798,7 @@ static void fast_cmyk_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = fz_mul255(s[0], 77);
@@ -796,7 +817,7 @@ static void fast_cmyk_to_gray(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = fz_mul255(s[0], 77);
@@ -817,7 +838,7 @@ static void fast_cmyk_to_gray(fz_pixmap *dst, fz_pixmap *src)
int si = 4 + src->alpha;
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
unsigned char c = fz_mul255(s[0], 77);
@@ -1164,13 +1185,16 @@ static void fast_cmyk_to_rgb(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 3);
int s_line_inc = src->stride - w * (src->alpha + 4);
unsigned int C,M,Y,K;
unsigned char r,g,b;
+ if ((int)w < 0 || h < 0)
+ return;
+
C = 0;
M = 0;
Y = 0;
@@ -1198,7 +1222,7 @@ static void fast_cmyk_to_rgb(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
#endif
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1217,7 +1241,7 @@ static void fast_cmyk_to_rgb(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1240,7 +1264,7 @@ static void fast_cmyk_to_rgb(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1260,13 +1284,16 @@ static void fast_cmyk_to_bgr(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 3);
int s_line_inc = src->stride - w * (src->alpha + 4);
unsigned int C,M,Y,K;
unsigned char r,g,b;
+ if ((int)w < 0 || h < 0)
+ return;
+
C = 0;
M = 0;
Y = 0;
@@ -1287,7 +1314,7 @@ static void fast_cmyk_to_bgr(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1306,7 +1333,7 @@ static void fast_cmyk_to_bgr(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1329,7 +1356,7 @@ static void fast_cmyk_to_bgr(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
cached_cmyk_conv(&r, &g, &b, &C, &M, &Y, &K, s[0], s[1], s[2], s[3]);
@@ -1349,11 +1376,14 @@ static void fast_rgb_to_bgr(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * (dst->alpha + 3);
int s_line_inc = src->stride - w * (src->alpha + 3);
+ if ((int)w < 0 || h < 0)
+ return;
+
if (d_line_inc == 0 && s_line_inc == 0)
{
w *= h;
@@ -1366,7 +1396,7 @@ static void fast_rgb_to_bgr(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[2];
@@ -1382,7 +1412,7 @@ static void fast_rgb_to_bgr(fz_pixmap *dst, fz_pixmap *src)
{
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[2];
@@ -1402,7 +1432,7 @@ static void fast_rgb_to_bgr(fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
d[0] = s[2];
@@ -1422,7 +1452,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
float dstv[FZ_MAX_COLORS];
int srcn, dstn;
int k, i;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int d_line_inc = dst->stride - w * dst->n;
int s_line_inc = src->stride - w * src->n;
@@ -1435,6 +1465,9 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
+ if ((int)w < 0 || h < 0)
+ return;
+
srcn = ss->n;
dstn = ds->n;
@@ -1456,7 +1489,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
fz_lookup_color_converter(ctx, &cc, ds, ss);
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
srcv[0] = *s++ / 255.0f * 100;
@@ -1484,7 +1517,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
fz_lookup_color_converter(ctx, &cc, ds, ss);
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
for (k = 0; k < srcn; k++)
@@ -1520,7 +1553,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
i = *s++;
@@ -1550,7 +1583,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *dst, fz_pixmap *src)
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
if (*s == *sold && memcmp(sold,s,srcn) == 0)
@@ -1605,12 +1638,15 @@ static void fast_copy_alpha(fz_pixmap *dst, fz_pixmap *src)
{
unsigned char *s = src->samples;
unsigned char *d = dst->samples;
- int w = src->w;
+ size_t w = src->w;
int h = src->h;
int n = src->n;
int d_line_inc = dst->stride - w * dst->n;
int s_line_inc = src->stride - w * src->n;
+ if ((int)w < 0 || h < 0)
+ return;
+
assert(dst->alpha && src->alpha && dst->n == 1);
if (d_line_inc == 0 && s_line_inc == 0)
@@ -1622,7 +1658,7 @@ static void fast_copy_alpha(fz_pixmap *dst, fz_pixmap *src)
s += n-1;
while (h--)
{
- int ww = w;
+ size_t ww = w;
while (ww--)
{
*d++ = *s;
diff --git a/source/fitz/pixmap.c b/source/fitz/pixmap.c
index 0016c003..17a086c6 100644
--- a/source/fitz/pixmap.c
+++ b/source/fitz/pixmap.c
@@ -259,6 +259,9 @@ clear_cmyk_bitmap(unsigned char *samples, int w, int h, int stride, int value, i
uint32_t *s = (uint32_t *)(void *)samples;
uint8_t *t;
+ if (w < 0 || h < 0)
+ return;
+
if (alpha)
{
int c = w;
@@ -335,17 +338,18 @@ clear_cmyk_bitmap(unsigned char *samples, int w, int h, int stride, int value, i
stride -= w*4;
if ((stride & 3) == 0)
{
+ size_t W = w;
if (stride == 0)
{
- w *= h;
+ W *= h;
h = 1;
}
- w *= 4;
+ W *= 4;
if (value == 0)
{
while (h--)
{
- memset(s, 0, w);
+ memset(s, 0, W);
s += (stride>>2);
}
}
@@ -364,8 +368,8 @@ clear_cmyk_bitmap(unsigned char *samples, int w, int h, int stride, int value, i
const uint32_t a0 = d.word;
while (h--)
{
- int ww = w;
- while (ww--)
+ size_t WW = W;
+ while (WW--)
{
*s++ = a0;
}
@@ -431,15 +435,18 @@ fz_clear_pixmap_with_value(fz_context *ctx, fz_pixmap *pix, int value)
int w, h, n, stride, len;
int alpha = pix->alpha;
+ w = pix->w;
+ h = pix->h;
+ if (w < 0 || h < 0)
+ return;
+
/* CMYK needs special handling (and potentially any other subtractive colorspaces) */
if (pix->colorspace && pix->colorspace->n == 4)
{
- clear_cmyk_bitmap(pix->samples, pix->w, pix->h, pix->stride, 255-value, pix->alpha);
+ clear_cmyk_bitmap(pix->samples, w, h, pix->stride, 255-value, pix->alpha);
return;
}
- w = pix->w;
- h = pix->h;
n = pix->n;
stride = pix->stride;
len = w * n;