Add soft limit to pixmap allocation.

All image loading functions call the new fz_new_pixmap_with_limit
allocation function, which will return NULL if the total amount of
pixmap memory would exceed a set limit. Other vital pixmap allocations
which are not as easily recoverable (such as font rendering, and the
various buffers in the draw device) ignore the limit.

3 files changed, 24 insertions, 3 deletions

diff --git a/xps/xps_jpeg.c b/xps/xps_jpeg.c
index 9a7462ef..68bb5eb9 100644
--- a/xps/xps_jpeg.c
+++ b/xps/xps_jpeg.c
@@ -94,7 +94,13 @@ xps_decode_jpeg(fz_pixmap **imagep, byte *rbuf, int rlen)
 	else
 		return fz_throw("bad number of components in jpeg: %d", cinfo.output_components);
 
-	image = fz_new_pixmap(colorspace, cinfo.output_width, cinfo.output_height);
+	image = fz_new_pixmap_with_limit(colorspace, cinfo.output_width, cinfo.output_height);
+	if (!image)
+	{
+		jpeg_finish_decompress(&cinfo);
+		jpeg_destroy_decompress(&cinfo);
+		return fz_throw("out of memory");
+	}
 
 	if (cinfo.density_unit == 1)
 	{
diff --git a/xps/xps_png.c b/xps/xps_png.c
index c1ed9683..245dd285 100644
--- a/xps/xps_png.c
+++ b/xps/xps_png.c
@@ -551,7 +551,13 @@ xps_decode_png(fz_pixmap **imagep, byte *p, int total)
 
 	stride = (png.width * png.n * png.depth + 7) / 8;
 
-	image = fz_new_pixmap(colorspace, png.width, png.height);
+	image = fz_new_pixmap_with_limit(colorspace, png.width, png.height);
+	if (!image)
+	{
+		fz_free(png.samples);
+		return fz_throw("out of memory");
+	}
+
 	image->xres = png.xres;
 	image->yres = png.yres;
 
diff --git a/xps/xps_tiff.c b/xps/xps_tiff.c
index 15406a05..8c58b4d1 100644
--- a/xps/xps_tiff.c
+++ b/xps/xps_tiff.c
@@ -822,7 +822,16 @@ xps_decode_tiff(fz_pixmap **imagep, byte *buf, int len)
 
 	/* Expand into fz_pixmap struct */
 
-	image = fz_new_pixmap(tiff.colorspace, tiff.imagewidth, tiff.imagelength);
+	image = fz_new_pixmap_with_limit(tiff.colorspace, tiff.imagewidth, tiff.imagelength);
+	if (!image)
+	{
+		if (tiff.colormap) fz_free(tiff.colormap);
+		if (tiff.stripoffsets) fz_free(tiff.stripoffsets);
+		if (tiff.stripbytecounts) fz_free(tiff.stripbytecounts);
+		if (tiff.samples) fz_free(tiff.samples);
+		return fz_throw("out of memory");
+	}
+
 	image->xres = tiff.xresolution;
 	image->yres = tiff.yresolution;