Add soft limit to pixmap allocation.

All image loading functions call the new fz_new_pixmap_with_limit
allocation function, which will return NULL if the total amount of
pixmap memory would exceed a set limit. Other vital pixmap allocations
which are not as easily recoverable (such as font rendering, and the
various buffers in the draw device) ignore the limit.

7 files changed, 73 insertions, 38 deletions

diff --git a/fitz/filt_jpxd.c b/fitz/filt_jpxd.c
index 4fb8264d..765aad29 100644
--- a/fitz/filt_jpxd.c
+++ b/fitz/filt_jpxd.c
@@ -108,7 +108,12 @@ fz_load_jpx_image(fz_pixmap **imgp, unsigned char *data, int size, fz_colorspace
 		}
 	}
 
-	img = fz_new_pixmap(colorspace, w, h);
+	img = fz_new_pixmap_with_limit(colorspace, w, h);
+	if (!img)
+	{
+		opj_image_destroy(jpx);
+		return fz_throw("out of memory");
+	}
 
 	p = img->samples;
 	for (y = 0; y < h; y++)
diff --git a/fitz/fitz.h b/fitz/fitz.h
index 111df17e..491db28b 100644
--- a/fitz/fitz.h
+++ b/fitz/fitz.h
@@ -680,6 +680,9 @@ struct fz_pixmap_s
 	int free_samples;
 };
 
+/* will return NULL if soft limit is exceeded */
+fz_pixmap *fz_new_pixmap_with_limit(fz_colorspace *colorspace, int w, int h);
+
 fz_pixmap *fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *samples);
 fz_pixmap *fz_new_pixmap_with_rect(fz_colorspace *, fz_bbox bbox);
 fz_pixmap *fz_new_pixmap(fz_colorspace *, int w, int h);
diff --git a/fitz/res_pixmap.c b/fitz/res_pixmap.c
index c6410a80..09d6e1a4 100644
--- a/fitz/res_pixmap.c
+++ b/fitz/res_pixmap.c
@@ -1,5 +1,8 @@
 #include "fitz.h"
 
+static int fz_memory_limit = 256 << 20;
+static int fz_memory_used = 0;
+
 fz_pixmap *
 fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *samples)
 {
@@ -31,6 +34,7 @@ fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *
 	}
 	else
 	{
+		fz_memory_used += pix->w * pix->h * pix->n;
 		pix->samples = fz_calloc(pix->h, pix->w * pix->n);
 		pix->free_samples = 1;
 	}
@@ -39,6 +43,20 @@ fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *
 }
 
 fz_pixmap *
+fz_new_pixmap_with_limit(fz_colorspace *colorspace, int w, int h)
+{
+	int n = colorspace ? colorspace->n + 1 : 1;
+	int size = w * h * n;
+	if (fz_memory_used + size > fz_memory_limit)
+	{
+		fz_warn("pixmap memory exceeds soft limit %dM + %dM > %dM",
+			fz_memory_used/(1<<20), size/(1<<20), fz_memory_limit/(1<<20));
+		return NULL;
+	}
+	return fz_new_pixmap_with_data(colorspace, w, h, NULL);
+}
+
+fz_pixmap *
 fz_new_pixmap(fz_colorspace *colorspace, int w, int h)
 {
 	return fz_new_pixmap_with_data(colorspace, w, h, NULL);
@@ -66,6 +84,7 @@ fz_drop_pixmap(fz_pixmap *pix)
 {
 	if (pix && --pix->refs == 0)
 	{
+		fz_memory_used -= pix->w * pix->h * pix->n;
 		if (pix->mask)
 			fz_drop_pixmap(pix->mask);
 		if (pix->colorspace)
diff --git a/pdf/pdf_image.c b/pdf/pdf_image.c
index 14d711cd..6d9fb881 100644
--- a/pdf/pdf_image.c
+++ b/pdf/pdf_image.c
@@ -43,7 +43,6 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 	int colorkey[FZ_MAX_COLORS * 2];
 	float decode[FZ_MAX_COLORS * 2];
 
-	int scale;
 	int stride;
 	unsigned char *samples;
 	int i, len;
@@ -156,6 +155,23 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 			colorkey[i] = fz_to_int(fz_array_get(obj, i));
 	}
 
+	/* Allocate now, to fail early if we run out of memory */
+	tile = fz_new_pixmap_with_limit(colorspace, w, h);
+	if (!tile)
+	{
+		if (colorspace)
+			fz_drop_colorspace(colorspace);
+		if (mask)
+			fz_drop_pixmap(mask);
+		return fz_throw("out of memory");
+	}
+
+	if (colorspace)
+		fz_drop_colorspace(colorspace);
+
+	tile->mask = mask;
+	tile->interpolate = interpolate;
+
 	stride = (w * n * bpc + 7) / 8;
 
 	if (cstm)
@@ -167,10 +183,7 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 		error = pdf_open_stream(&stm, xref, fz_to_num(dict), fz_to_gen(dict));
 		if (error)
 		{
-			if (colorspace)
-				fz_drop_colorspace(colorspace);
-			if (mask)
-				fz_drop_pixmap(mask);
+			fz_drop_pixmap(tile);
 			return fz_rethrow(error, "cannot open image data stream (%d 0 R)", fz_to_num(dict));
 		}
 	}
@@ -181,11 +194,8 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 	if (len < 0)
 	{
 		fz_close(stm);
-		if (colorspace)
-			fz_drop_colorspace(colorspace);
-		if (mask)
-			fz_drop_pixmap(mask);
 		fz_free(samples);
+		fz_drop_pixmap(tile);
 		return fz_rethrow(len, "cannot read image data");
 	}
 
@@ -219,22 +229,9 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 			p[i] = ~p[i];
 	}
 
-	/* Unpack samples into pixmap */
-
-	tile = fz_new_pixmap(colorspace, w, h);
+	fz_unpack_tile(tile, samples, n, bpc, stride, indexed);
 
-	scale = 1;
-	if (!indexed)
-	{
-		switch (bpc)
-		{
-		case 1: scale = 255; break;
-		case 2: scale = 85; break;
-		case 4: scale = 17; break;
-		}
-	}
-
-	fz_unpack_tile(tile, samples, n, bpc, stride, scale);
+	fz_free(samples);
 
 	if (usecolorkey)
 		pdf_mask_color_key(tile, n, colorkey);
@@ -242,9 +239,7 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 	if (indexed)
 	{
 		fz_pixmap *conv;
-
 		fz_decode_indexed_tile(tile, decode, (1 << bpc) - 1);
-
 		conv = pdf_expand_indexed_pixmap(tile);
 		fz_drop_pixmap(tile);
 		tile = conv;
@@ -254,14 +249,6 @@ pdf_load_image_imp(fz_pixmap **imgp, pdf_xref *xref, fz_obj *rdb, fz_obj *dict,
 		fz_decode_tile(tile, decode);
 	}
 
-	if (colorspace)
-		fz_drop_colorspace(colorspace);
-
-	tile->mask = mask;
-	tile->interpolate = interpolate;
-
-	fz_free(samples);
-
 	*imgp = tile;
 	return fz_okay;
 }
diff --git a/xps/xps_jpeg.c b/xps/xps_jpeg.c
index 9a7462ef..68bb5eb9 100644
--- a/xps/xps_jpeg.c
+++ b/xps/xps_jpeg.c
@@ -94,7 +94,13 @@ xps_decode_jpeg(fz_pixmap **imagep, byte *rbuf, int rlen)
 	else
 		return fz_throw("bad number of components in jpeg: %d", cinfo.output_components);
 
-	image = fz_new_pixmap(colorspace, cinfo.output_width, cinfo.output_height);
+	image = fz_new_pixmap_with_limit(colorspace, cinfo.output_width, cinfo.output_height);
+	if (!image)
+	{
+		jpeg_finish_decompress(&cinfo);
+		jpeg_destroy_decompress(&cinfo);
+		return fz_throw("out of memory");
+	}
 
 	if (cinfo.density_unit == 1)
 	{
diff --git a/xps/xps_png.c b/xps/xps_png.c
index c1ed9683..245dd285 100644
--- a/xps/xps_png.c
+++ b/xps/xps_png.c
@@ -551,7 +551,13 @@ xps_decode_png(fz_pixmap **imagep, byte *p, int total)
 
 	stride = (png.width * png.n * png.depth + 7) / 8;
 
-	image = fz_new_pixmap(colorspace, png.width, png.height);
+	image = fz_new_pixmap_with_limit(colorspace, png.width, png.height);
+	if (!image)
+	{
+		fz_free(png.samples);
+		return fz_throw("out of memory");
+	}
+
 	image->xres = png.xres;
 	image->yres = png.yres;
 
diff --git a/xps/xps_tiff.c b/xps/xps_tiff.c
index 15406a05..8c58b4d1 100644
--- a/xps/xps_tiff.c
+++ b/xps/xps_tiff.c
@@ -822,7 +822,16 @@ xps_decode_tiff(fz_pixmap **imagep, byte *buf, int len)
 
 	/* Expand into fz_pixmap struct */
 
-	image = fz_new_pixmap(tiff.colorspace, tiff.imagewidth, tiff.imagelength);
+	image = fz_new_pixmap_with_limit(tiff.colorspace, tiff.imagewidth, tiff.imagelength);
+	if (!image)
+	{
+		if (tiff.colormap) fz_free(tiff.colormap);
+		if (tiff.stripoffsets) fz_free(tiff.stripoffsets);
+		if (tiff.stripbytecounts) fz_free(tiff.stripbytecounts);
+		if (tiff.samples) fz_free(tiff.samples);
+		return fz_throw("out of memory");
+	}
+
 	image->xres = tiff.xresolution;
 	image->yres = tiff.yresolution;