Add soft limit to pixmap allocation.

All image loading functions call the new fz_new_pixmap_with_limit
allocation function, which will return NULL if the total amount of
pixmap memory would exceed a set limit. Other vital pixmap allocations
which are not as easily recoverable (such as font rendering, and the
various buffers in the draw device) ignore the limit.

3 files changed, 28 insertions, 1 deletions

diff --git a/fitz/filt_jpxd.c b/fitz/filt_jpxd.c
index 4fb8264d..765aad29 100644
--- a/fitz/filt_jpxd.c
+++ b/fitz/filt_jpxd.c
@@ -108,7 +108,12 @@ fz_load_jpx_image(fz_pixmap **imgp, unsigned char *data, int size, fz_colorspace
 		}
 	}
 
-	img = fz_new_pixmap(colorspace, w, h);
+	img = fz_new_pixmap_with_limit(colorspace, w, h);
+	if (!img)
+	{
+		opj_image_destroy(jpx);
+		return fz_throw("out of memory");
+	}
 
 	p = img->samples;
 	for (y = 0; y < h; y++)
diff --git a/fitz/fitz.h b/fitz/fitz.h
index 111df17e..491db28b 100644
--- a/fitz/fitz.h
+++ b/fitz/fitz.h
@@ -680,6 +680,9 @@ struct fz_pixmap_s
 	int free_samples;
 };
 
+/* will return NULL if soft limit is exceeded */
+fz_pixmap *fz_new_pixmap_with_limit(fz_colorspace *colorspace, int w, int h);
+
 fz_pixmap *fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *samples);
 fz_pixmap *fz_new_pixmap_with_rect(fz_colorspace *, fz_bbox bbox);
 fz_pixmap *fz_new_pixmap(fz_colorspace *, int w, int h);
diff --git a/fitz/res_pixmap.c b/fitz/res_pixmap.c
index c6410a80..09d6e1a4 100644
--- a/fitz/res_pixmap.c
+++ b/fitz/res_pixmap.c
@@ -1,5 +1,8 @@
 #include "fitz.h"
 
+static int fz_memory_limit = 256 << 20;
+static int fz_memory_used = 0;
+
 fz_pixmap *
 fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *samples)
 {
@@ -31,6 +34,7 @@ fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *
 	}
 	else
 	{
+		fz_memory_used += pix->w * pix->h * pix->n;
 		pix->samples = fz_calloc(pix->h, pix->w * pix->n);
 		pix->free_samples = 1;
 	}
@@ -39,6 +43,20 @@ fz_new_pixmap_with_data(fz_colorspace *colorspace, int w, int h, unsigned char *
 }
 
 fz_pixmap *
+fz_new_pixmap_with_limit(fz_colorspace *colorspace, int w, int h)
+{
+	int n = colorspace ? colorspace->n + 1 : 1;
+	int size = w * h * n;
+	if (fz_memory_used + size > fz_memory_limit)
+	{
+		fz_warn("pixmap memory exceeds soft limit %dM + %dM > %dM",
+			fz_memory_used/(1<<20), size/(1<<20), fz_memory_limit/(1<<20));
+		return NULL;
+	}
+	return fz_new_pixmap_with_data(colorspace, w, h, NULL);
+}
+
+fz_pixmap *
 fz_new_pixmap(fz_colorspace *colorspace, int w, int h)
 {
 	return fz_new_pixmap_with_data(colorspace, w, h, NULL);
@@ -66,6 +84,7 @@ fz_drop_pixmap(fz_pixmap *pix)
 {
 	if (pix && --pix->refs == 0)
 	{
+		fz_memory_used -= pix->w * pix->h * pix->n;
 		if (pix->mask)
 			fz_drop_pixmap(pix->mask);
 		if (pix->colorspace)