Rework image handling for on demand decode

Introduce a new 'fz_image' type; this type contains rudimentary
information about images (such as native, size, colorspace etc)
and a function to call to get a pixmap of that image (with a
size hint).

Instead of passing pixmaps through the device interface (and
holding pixmaps in the display list) we now pass images instead.

The rendering routines therefore call fz_image_to_pixmap to get
pixmaps to render, and fz_pixmap_drop those afterwards.

The file format handling routines therefore need to produce
images rather than pixmaps; xps and cbz currently just wrap
pixmaps as images. PDF is more involved.

The stream handling routines in PDF have been altered so that
they can recognise when the last stream entry in a filter
dictionary is an image decoding filter. Rather than applying
this filter, they read and store the parameters into a
pdf_image_params structure, and stop decoding at that point.

This allows us to read the compressed data for an image into
memory as a block. We can then restart the image decode process
later.

pdf_images therefore consist of the compressed image data for
images. When a pixmap is requested for such an image, the code
checks to see if we have one (of an appropriate size), and if
not, decodes it.

The size hint is used to determine whether it is possible to
subsample the image; currently this is only supported for
JPEGs, but we could add generic subsampling code later.

In order to handle caching the produced images, various changes
have been made to the store and the underlying hash table.

Previously the store was indexed purely by fz_obj keys; we don't
have an fz_obj key any more, so have extended the store by adding
a concept of a key 'type'. A key type is a pointer to a set of
functions that keep/drop/compare and make a hashable key from
a key pointer.

We make a pdf_store.c file that contains functions to offer the
existing fz_obj based functions, and add a new 'type' for keys
(based on the fz_image handle, and the subsample factor) in the
pdf_image.c file.

While working on this, a problem became apparent in the existing
store codel; fz_obj objects had no protection on their reference
counts, hence an interpreter thread could try to alter a ref count
at the same time as a malloc caused an eviction from the store.
This has been solved by using the alloc lock as protection. This in
turn requires some tweaks to the code to make sure we don't try
and keep/drop fz_obj's from the store code while the alloc lock is
held.

A side effect of this work is that when a hash table is created, we
inform it what lock should be used to protect its innards (if any).
If the alloc lock is used, the insert method knows to drop/retake it
to allow it to safely expand the hash table. Callers to the hash
functions have the responsibility of taking/dropping the appropriate
lock, and ensuring that they cope with the possibility that insert
might drop the alloc lock, causing race conditions.

11 files changed, 240 insertions, 127 deletions

diff --git a/fitz/base_hash.c b/fitz/base_hash.c
index 630f6b6a..0fda86cd 100644
--- a/fitz/base_hash.c
+++ b/fitz/base_hash.c
@@ -22,6 +22,7 @@ struct fz_hash_table_s
 	int keylen;
 	int size;
 	int load;
+	int lock; /* -1 or the lock used to protect this hash table */
 	fz_hash_entry *ents;
 };
 
@@ -42,7 +43,7 @@ static unsigned hash(unsigned char *s, int len)
 }
 
 fz_hash_table *
-fz_new_hash_table(fz_context *ctx, int initialsize, int keylen)
+fz_new_hash_table(fz_context *ctx, int initialsize, int keylen, int lock)
 {
 	fz_hash_table *table;
 
@@ -52,6 +53,7 @@ fz_new_hash_table(fz_context *ctx, int initialsize, int keylen)
 	table->keylen = keylen;
 	table->size = initialsize;
 	table->load = 0;
+	table->lock = lock;
 	fz_try(ctx)
 	{
 		table->ents = fz_malloc_array(ctx, table->size, sizeof(fz_hash_entry));
@@ -98,10 +100,45 @@ fz_free_hash(fz_context *ctx, fz_hash_table *table)
 	fz_free(ctx, table);
 }
 
+static void *
+do_hash_insert(fz_context *ctx, fz_hash_table *table, void *key, void *val)
+{
+	fz_hash_entry *ents;
+	unsigned size;
+	unsigned pos;
+
+	ents = table->ents;
+	size = table->size;
+	pos = hash(key, table->keylen) % size;
+
+	if (table->lock >= 0)
+		fz_assert_lock_held(ctx, table->lock);
+
+	while (1)
+	{
+		if (!ents[pos].val)
+		{
+			memcpy(ents[pos].key, key, table->keylen);
+			ents[pos].val = val;
+			table->load ++;
+			return NULL;
+		}
+
+		if (memcmp(key, ents[pos].key, table->keylen) == 0)
+		{
+			fz_warn(ctx, "assert: overwrite hash slot");
+			return ents[pos].val = val;
+		}
+
+		pos = (pos + 1) % size;
+	}
+}
+
 static void
 fz_resize_hash(fz_context *ctx, fz_hash_table *table, int newsize)
 {
 	fz_hash_entry *oldents = table->ents;
+	fz_hash_entry *newents = table->ents;
 	int oldsize = table->size;
 	int oldload = table->load;
 	int i;
@@ -112,7 +149,22 @@ fz_resize_hash(fz_context *ctx, fz_hash_table *table, int newsize)
 		return;
 	}
 
-	table->ents = fz_malloc_array(ctx, newsize, sizeof(fz_hash_entry));
+	if (table->lock == FZ_LOCK_ALLOC)
+		fz_unlock(ctx, FZ_LOCK_ALLOC);
+	newents = fz_malloc_array(ctx, newsize, sizeof(fz_hash_entry));
+	if (table->lock == FZ_LOCK_ALLOC)
+		fz_lock(ctx, FZ_LOCK_ALLOC);
+	if (table->lock >= 0)
+	{
+		if (table->size >= newsize)
+		{
+			/* Someone else fixed it before we could lock! */
+			fz_unlock(ctx, table->lock);
+			fz_free(ctx, newents);
+			return;
+		}
+	}
+	table->ents = newents;
 	memset(table->ents, 0, sizeof(fz_hash_entry) * newsize);
 	table->size = newsize;
 	table->load = 0;
@@ -121,11 +173,15 @@ fz_resize_hash(fz_context *ctx, fz_hash_table *table, int newsize)
 	{
 		if (oldents[i].val)
 		{
-			fz_hash_insert(ctx, table, oldents[i].key, oldents[i].val);
+			do_hash_insert(ctx, table, oldents[i].key, oldents[i].val);
 		}
 	}
 
+	if (table->lock == FZ_LOCK_ALLOC)
+		fz_unlock(ctx, FZ_LOCK_ALLOC);
 	fz_free(ctx, oldents);
+	if (table->lock == FZ_LOCK_ALLOC)
+		fz_lock(ctx, FZ_LOCK_ALLOC);
 }
 
 void *
@@ -135,6 +191,9 @@ fz_hash_find(fz_context *ctx, fz_hash_table *table, void *key)
 	unsigned size = table->size;
 	unsigned pos = hash(key, table->keylen) % size;
 
+	if (table->lock >= 0)
+		fz_assert_lock_held(ctx, table->lock);
+
 	while (1)
 	{
 		if (!ents[pos].val)
@@ -150,37 +209,12 @@ fz_hash_find(fz_context *ctx, fz_hash_table *table, void *key)
 void *
 fz_hash_insert(fz_context *ctx, fz_hash_table *table, void *key, void *val)
 {
-	fz_hash_entry *ents;
-	unsigned size;
-	unsigned pos;
-
 	if (table->load > table->size * 8 / 10)
 	{
 		fz_resize_hash(ctx, table, table->size * 2);
 	}
 
-	ents = table->ents;
-	size = table->size;
-	pos = hash(key, table->keylen) % size;
-
-	while (1)
-	{
-		if (!ents[pos].val)
-		{
-			memcpy(ents[pos].key, key, table->keylen);
-			ents[pos].val = val;
-			table->load ++;
-			return NULL;
-		}
-
-		if (memcmp(key, ents[pos].key, table->keylen) == 0)
-		{
-			fz_warn(ctx, "assert: overwrite hash slot");
-			return ents[pos].val;
-		}
-
-		pos = (pos + 1) % size;
-	}
+	return do_hash_insert(ctx, table, key, val);
 }
 
 void
@@ -191,11 +225,14 @@ fz_hash_remove(fz_context *ctx, fz_hash_table *table, void *key)
 	unsigned pos = hash(key, table->keylen) % size;
 	unsigned hole, look, code;
 
+	if (table->lock >= 0)
+		fz_assert_lock_held(ctx, table->lock);
+
 	while (1)
 	{
 		if (!ents[pos].val)
 		{
-			fz_warn(ctx, "assert: remove inexistent hash entry");
+			fz_warn(ctx, "assert: remove non-existent hash entry");
 			return;
 		}
 
diff --git a/fitz/dev_bbox.c b/fitz/dev_bbox.c
index 636ceb94..b015c585 100644
--- a/fitz/dev_bbox.c
+++ b/fitz/dev_bbox.c
@@ -47,7 +47,7 @@ fz_bbox_fill_shade(fz_device *dev, fz_shade *shade, fz_matrix ctm, float alpha)
 }
 
 static void
-fz_bbox_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha)
+fz_bbox_fill_image(fz_device *dev, fz_image *image, fz_matrix ctm, float alpha)
 {
 	fz_bbox *result = dev->user;
 	fz_bbox bbox = fz_round_rect(fz_transform_rect(ctm, fz_unit_rect));
@@ -55,7 +55,7 @@ fz_bbox_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha)
 }
 
 static void
-fz_bbox_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm,
+fz_bbox_fill_image_mask(fz_device *dev, fz_image *image, fz_matrix ctm,
 	fz_colorspace *colorspace, float *color, float alpha)
 {
 	fz_bbox_fill_image(dev, image, ctm, alpha);
diff --git a/fitz/dev_list.c b/fitz/dev_list.c
index adc691d0..1d82421e 100644
--- a/fitz/dev_list.c
+++ b/fitz/dev_list.c
@@ -37,7 +37,7 @@ struct fz_display_node_s
 		fz_path *path;
 		fz_text *text;
 		fz_shade *shade;
-		fz_pixmap *image;
+		fz_image *image;
 		int blendmode;
 	} item;
 	fz_stroke_state *stroke;
@@ -207,7 +207,7 @@ fz_free_display_node(fz_context *ctx, fz_display_node *node)
 	case FZ_CMD_FILL_IMAGE:
 	case FZ_CMD_FILL_IMAGE_MASK:
 	case FZ_CMD_CLIP_IMAGE_MASK:
-		fz_drop_pixmap(ctx, node->item.image);
+		fz_drop_image(ctx, node->item.image);
 		break;
 	case FZ_CMD_POP_CLIP:
 	case FZ_CMD_BEGIN_MASK:
@@ -435,35 +435,35 @@ fz_list_fill_shade(fz_device *dev, fz_shade *shade, fz_matrix ctm, float alpha)
 }
 
 static void
-fz_list_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha)
+fz_list_fill_image(fz_device *dev, fz_image *image, fz_matrix ctm, float alpha)
 {
 	fz_display_node *node;
 	node = fz_new_display_node(dev->ctx, FZ_CMD_FILL_IMAGE, ctm, NULL, NULL, alpha);
 	node->rect = fz_transform_rect(ctm, fz_unit_rect);
-	node->item.image = fz_keep_pixmap(dev->ctx, image);
+	node->item.image = fz_keep_image(dev->ctx, image);
 	fz_append_display_node(dev->user, node);
 }
 
 static void
-fz_list_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm,
+fz_list_fill_image_mask(fz_device *dev, fz_image *image, fz_matrix ctm,
 	fz_colorspace *colorspace, float *color, float alpha)
 {
 	fz_display_node *node;
 	node = fz_new_display_node(dev->ctx, FZ_CMD_FILL_IMAGE_MASK, ctm, colorspace, color, alpha);
 	node->rect = fz_transform_rect(ctm, fz_unit_rect);
-	node->item.image = fz_keep_pixmap(dev->ctx, image);
+	node->item.image = fz_keep_image(dev->ctx, image);
 	fz_append_display_node(dev->user, node);
 }
 
 static void
-fz_list_clip_image_mask(fz_device *dev, fz_pixmap *image, fz_rect *rect, fz_matrix ctm)
+fz_list_clip_image_mask(fz_device *dev, fz_image *image, fz_rect *rect, fz_matrix ctm)
 {
 	fz_display_node *node;
 	node = fz_new_display_node(dev->ctx, FZ_CMD_CLIP_IMAGE_MASK, ctm, NULL, NULL, 0);
 	node->rect = fz_transform_rect(ctm, fz_unit_rect);
 	if (rect)
 		node->rect = fz_intersect_rect(node->rect, *rect);
-	node->item.image = fz_keep_pixmap(dev->ctx, image);
+	node->item.image = fz_keep_image(dev->ctx, image);
 	fz_append_display_node(dev->user, node);
 }
 
diff --git a/fitz/dev_null.c b/fitz/dev_null.c
index b4ba5cbe..d817932d 100644
--- a/fitz/dev_null.c
+++ b/fitz/dev_null.c
@@ -103,14 +103,14 @@ fz_fill_shade(fz_device *dev, fz_shade *shade, fz_matrix ctm, float alpha)
 }
 
 void
-fz_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha)
+fz_fill_image(fz_device *dev, fz_image *image, fz_matrix ctm, float alpha)
 {
 	if (dev->fill_image)
 		dev->fill_image(dev, image, ctm, alpha);
 }
 
 void
-fz_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm,
+fz_fill_image_mask(fz_device *dev, fz_image *image, fz_matrix ctm,
 	fz_colorspace *colorspace, float *color, float alpha)
 {
 	if (dev->fill_image_mask)
@@ -118,7 +118,7 @@ fz_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm,
 }
 
 void
-fz_clip_image_mask(fz_device *dev, fz_pixmap *image, fz_rect *rect, fz_matrix ctm)
+fz_clip_image_mask(fz_device *dev, fz_image *image, fz_rect *rect, fz_matrix ctm)
 {
 	if (dev->clip_image_mask)
 		dev->clip_image_mask(dev, image, rect, ctm);
diff --git a/fitz/dev_trace.c b/fitz/dev_trace.c
index 1c2e1ed1..9faffc35 100644
--- a/fitz/dev_trace.c
+++ b/fitz/dev_trace.c
@@ -197,7 +197,7 @@ fz_trace_ignore_text(fz_device *dev, fz_text *text, fz_matrix ctm)
 }
 
 static void
-fz_trace_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha)
+fz_trace_fill_image(fz_device *dev, fz_image *image, fz_matrix ctm, float alpha)
 {
 	printf("<fill_image alpha=\"%g\" ", alpha);
 	fz_trace_matrix(ctm);
@@ -213,7 +213,7 @@ fz_trace_fill_shade(fz_device *dev, fz_shade *shade, fz_matrix ctm, float alpha)
 }
 
 static void
-fz_trace_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm,
+fz_trace_fill_image_mask(fz_device *dev, fz_image *image, fz_matrix ctm,
 fz_colorspace *colorspace, float *color, float alpha)
 {
 	printf("<fill_image_mask ");
@@ -223,7 +223,7 @@ fz_colorspace *colorspace, float *color, float alpha)
 }
 
 static void
-fz_trace_clip_image_mask(fz_device *dev, fz_pixmap *image, fz_rect *rect, fz_matrix ctm)
+fz_trace_clip_image_mask(fz_device *dev, fz_image *image, fz_rect *rect, fz_matrix ctm)
 {
 	printf("<clip_image_mask ");
 	fz_trace_matrix(ctm);
diff --git a/fitz/filt_dctd.c b/fitz/filt_dctd.c
index 4357f3d7..3abe0468 100644
--- a/fitz/filt_dctd.c
+++ b/fitz/filt_dctd.c
@@ -12,6 +12,7 @@ struct fz_dctd_s
 	int color_transform;
 	int init;
 	int stride;
+	int factor;
 	unsigned char *scanline;
 	unsigned char *rp, *wp;
 	struct jpeg_decompress_struct cinfo;
@@ -150,6 +151,9 @@ read_dctd(fz_stream *stm, unsigned char *buf, int len)
 			break;
 		}
 
+		cinfo->scale_num = 8/state->factor;
+		cinfo->scale_denom = 8;
+
 		jpeg_start_decompress(cinfo);
 
 		state->stride = cinfo->output_width * cinfo->output_components;
@@ -216,6 +220,12 @@ skip:
 fz_stream *
 fz_open_dctd(fz_stream *chain, int color_transform)
 {
+	return fz_open_resized_dctd(chain, color_transform, 1);
+}
+
+fz_stream *
+fz_open_resized_dctd(fz_stream *chain, int color_transform, int factor)
+{
 	fz_context *ctx = chain->ctx;
 	fz_dctd *state = NULL;
 
@@ -228,6 +238,7 @@ fz_open_dctd(fz_stream *chain, int color_transform)
 		state->chain = chain;
 		state->color_transform = color_transform;
 		state->init = 0;
+		state->factor = factor;
 	}
 	fz_catch(ctx)
 	{
diff --git a/fitz/fitz.h b/fitz/fitz.h
index d2672031..856df7e5 100644
--- a/fitz/fitz.h
+++ b/fitz/fitz.h
@@ -504,7 +504,7 @@ extern char *fz_optarg;
 
 typedef struct fz_hash_table_s fz_hash_table;
 
-fz_hash_table *fz_new_hash_table(fz_context *ctx, int initialsize, int keylen);
+fz_hash_table *fz_new_hash_table(fz_context *ctx, int initialsize, int keylen, int lock);
 void fz_debug_hash(fz_context *ctx, fz_hash_table *table);
 void fz_empty_hash(fz_context *ctx, fz_hash_table *table);
 void fz_free_hash(fz_context *ctx, fz_hash_table *table);
@@ -810,6 +810,36 @@ enum {
 	FZ_STORE_DEFAULT = 256 << 20,
 };
 
+typedef struct fz_store_hash_s fz_store_hash;
+
+struct fz_store_hash_s
+{
+	fz_store_free_fn *free;
+	union
+	{
+		struct
+		{
+			int i0;
+			int i1;
+		} i;
+		struct
+		{
+			void *ptr;
+			int i;
+		} pi;
+	} u;
+};
+
+typedef struct fz_store_type_s fz_store_type;
+
+struct fz_store_type_s
+{
+	int (*make_hash_key)(fz_store_hash *, void *);
+	void *(*keep_key)(fz_context *,void *);
+	void (*drop_key)(fz_context *,void *);
+	int (*cmp_key)(void *, void *);
+};
+
 void fz_new_store_context(fz_context *ctx, unsigned int max);
 void fz_drop_store_context(fz_context *ctx);
 fz_store *fz_store_keep(fz_context *ctx);
@@ -818,9 +848,9 @@ void fz_debug_store(fz_context *ctx);
 void *fz_keep_storable(fz_context *, fz_storable *);
 void fz_drop_storable(fz_context *, fz_storable *);
 
-void fz_store_item(fz_context *ctx, fz_obj *key, void *val, unsigned int itemsize);
-void *fz_find_item(fz_context *ctx, fz_store_free_fn *freefn, fz_obj *key);
-void fz_remove_item(fz_context *ctx, fz_store_free_fn *freefn, fz_obj *key);
+void *fz_store_item(fz_context *ctx, void *key, void *val, unsigned int itemsize, fz_store_type *type);
+void *fz_find_item(fz_context *ctx, fz_store_free_fn *free, void *key, fz_store_type *type);
+void fz_remove_item(fz_context *ctx, fz_store_free_fn *free, void *key, fz_store_type *type);
 void fz_empty_store(fz_context *ctx);
 int fz_store_scavenge(fz_context *ctx, unsigned int size, int *phase);
 
@@ -959,6 +989,7 @@ fz_stream *fz_open_a85d(fz_stream *chain);
 fz_stream *fz_open_ahxd(fz_stream *chain);
 fz_stream *fz_open_rld(fz_stream *chain);
 fz_stream *fz_open_dctd(fz_stream *chain, int color_transform);
+fz_stream *fz_open_resized_dctd(fz_stream *chain, int color_transform, int factor);
 fz_stream *fz_open_faxd(fz_stream *chain,
 	int k, int end_of_line, int encoded_byte_align,
 	int columns, int rows, int end_of_block, int black_is_1);
@@ -1023,6 +1054,27 @@ void fz_write_pnm(fz_context *ctx, fz_pixmap *pixmap, char *filename);
 void fz_write_pam(fz_context *ctx, fz_pixmap *pixmap, char *filename, int savealpha);
 void fz_write_png(fz_context *ctx, fz_pixmap *pixmap, char *filename, int savealpha);
 
+/*
+ * Images are either a reference to a pixmap, or details of how to make
+ * a pixmap. To know how to make a pixmap we need a block of compressed
+ * data (typically a stream decoded all the way until the image filter)
+ * and then the details of the params for the filter to do the final step.
+ */
+typedef struct fz_image_s fz_image;
+
+struct fz_image_s
+{
+	fz_storable storable;
+	int w, h;
+	fz_image *mask;
+	fz_colorspace *colorspace;
+	fz_pixmap *(*get_pixmap)(fz_context *, fz_image *, int w, int h);
+};
+
+fz_pixmap *fz_image_to_pixmap(fz_context *, fz_image *, int w, int h);
+void fz_drop_image(fz_context *ctx, fz_image *image);
+fz_image *fz_keep_image(fz_context *ctx, fz_image *image);
+
 fz_pixmap *fz_load_jpx(fz_context *ctx, unsigned char *data, int size, fz_colorspace *cs);
 fz_pixmap *fz_load_jpeg(fz_context *doc, unsigned char *data, int size);
 fz_pixmap *fz_load_png(fz_context *doc, unsigned char *data, int size);
@@ -1413,9 +1465,9 @@ struct fz_device_s
 	void (*ignore_text)(fz_device *, fz_text *, fz_matrix);
 
 	void (*fill_shade)(fz_device *, fz_shade *shd, fz_matrix ctm, float alpha);
-	void (*fill_image)(fz_device *, fz_pixmap *img, fz_matrix ctm, float alpha);
-	void (*fill_image_mask)(fz_device *, fz_pixmap *img, fz_matrix ctm, fz_colorspace *, float *color, float alpha);
-	void (*clip_image_mask)(fz_device *, fz_pixmap *img, fz_rect *rect, fz_matrix ctm);
+	void (*fill_image)(fz_device *, fz_image *img, fz_matrix ctm, float alpha);
+	void (*fill_image_mask)(fz_device *, fz_image *img, fz_matrix ctm, fz_colorspace *, float *color, float alpha);
+	void (*clip_image_mask)(fz_device *, fz_image *img, fz_rect *rect, fz_matrix ctm);
 
 	void (*pop_clip)(fz_device *);
 
@@ -1439,9 +1491,9 @@ void fz_clip_stroke_text(fz_device *dev, fz_text *text, fz_stroke_state *stroke,
 void fz_ignore_text(fz_device *dev, fz_text *text, fz_matrix ctm);
 void fz_pop_clip(fz_device *dev);
 void fz_fill_shade(fz_device *dev, fz_shade *shade, fz_matrix ctm, float alpha);
-void fz_fill_image(fz_device *dev, fz_pixmap *image, fz_matrix ctm, float alpha);
-void fz_fill_image_mask(fz_device *dev, fz_pixmap *image, fz_matrix ctm, fz_colorspace *colorspace, float *color, float alpha);
-void fz_clip_image_mask(fz_device *dev, fz_pixmap *image, fz_rect *rect, fz_matrix ctm);
+void fz_fill_image(fz_device *dev, fz_image *image, fz_matrix ctm, float alpha);
+void fz_fill_image_mask(fz_device *dev, fz_image *image, fz_matrix ctm, fz_colorspace *colorspace, float *color, float alpha);
+void fz_clip_image_mask(fz_device *dev, fz_image *image, fz_rect *rect, fz_matrix ctm);
 void fz_begin_mask(fz_device *dev, fz_rect area, int luminosity, fz_colorspace *colorspace, float *bc);
 void fz_end_mask(fz_device *dev);
 void fz_begin_group(fz_device *dev, fz_rect area, int isolated, int knockout, int blendmode, float alpha);
diff --git a/fitz/res_colorspace.c b/fitz/res_colorspace.c
index 413da3ae..c25fc99f 100644
--- a/fitz/res_colorspace.c
+++ b/fitz/res_colorspace.c
@@ -465,7 +465,7 @@ fz_std_conv_pixmap(fz_context *ctx, fz_pixmap *src, fz_pixmap *dst)
 		fz_hash_table *lookup;
 		unsigned char *color;
 
-		lookup = fz_new_hash_table(ctx, 509, srcn);
+		lookup = fz_new_hash_table(ctx, 509, srcn, -1);
 
 		for (y = 0; y < src->h; y++)
 		{
diff --git a/fitz/res_pixmap.c b/fitz/res_pixmap.c
index 500eb4be..328a8c2d 100644
--- a/fitz/res_pixmap.c
+++ b/fitz/res_pixmap.c
@@ -551,3 +551,23 @@ fz_pixmap_size(fz_context *ctx, fz_pixmap * pix)
 		return 0;
 	return sizeof(*pix) + pix->n * pix->w * pix->h;
 }
+
+fz_pixmap *
+fz_image_to_pixmap(fz_context *ctx, fz_image *image, int w, int h)
+{
+	if (image == NULL)
+		return NULL;
+	return image->get_pixmap(ctx, image, w, h);
+}
+
+fz_image *
+fz_keep_image(fz_context *ctx, fz_image *image)
+{
+	return (fz_image *)fz_keep_storable(ctx, &image->storable);
+}
+
+void
+fz_drop_image(fz_context *ctx, fz_image *image)
+{
+	fz_drop_storable(ctx, &image->storable);
+}
diff --git a/fitz/res_store.c b/fitz/res_store.c
index 5eec1f0d..461ea7c8 100644
--- a/fitz/res_store.c
+++ b/fitz/res_store.c
@@ -1,21 +1,14 @@
 #include "fitz.h"
-#include "mupdf.h"
 
 struct fz_item_s
 {
-	fz_obj *key;
+	void *key;
 	fz_storable *val;
 	unsigned int size;
 	fz_item *next;
 	fz_item *prev;
 	fz_store *store;
-};
-
-struct refkey
-{
-	fz_store_free_fn *free;
-	int num;
-	int gen;
+	fz_store_type *type;
 };
 
 struct fz_store_s
@@ -43,7 +36,7 @@ fz_new_store_context(fz_context *ctx, unsigned int max)
 	store = fz_malloc_struct(ctx, fz_store);
 	fz_try(ctx)
 	{
-		store->hash = fz_new_hash_table(ctx, 4096, sizeof(struct refkey));
+		store->hash = fz_new_hash_table(ctx, 4096, sizeof(fz_store_hash), FZ_LOCK_ALLOC);
 	}
 	fz_catch(ctx)
 	{
@@ -114,20 +107,19 @@ evict(fz_context *ctx, fz_item *item)
 		store->head = item->next;
 	/* Drop a reference to the value (freeing if required) */
 	drop = (item->val->refs > 0 && --item->val->refs == 0);
-	fz_unlock(ctx, FZ_LOCK_ALLOC);
 	/* Remove from the hash table */
-	if (fz_is_indirect(item->key))
+	if (item->type->make_hash_key)
 	{
-		struct refkey refkey;
-		refkey.free = item->val->free;
-		refkey.num = fz_to_num(item->key);
-		refkey.gen = fz_to_gen(item->key);
-		fz_hash_remove(ctx, store->hash, &refkey);
+		fz_store_hash hash = { NULL };
+		hash.free = item->val->free;
+		if (item->type->make_hash_key(&hash, item->key))
+			fz_hash_remove(ctx, store->hash, &hash);
 	}
+	fz_unlock(ctx, FZ_LOCK_ALLOC);
 	if (drop)
 		item->val->free(ctx, item->val);
 	/* Always drops the key and free the item */
-	fz_drop_obj(item->key);
+	item->type->drop_key(ctx, item->key);
 	fz_free(ctx, item);
 	fz_lock(ctx, FZ_LOCK_ALLOC);
 }
@@ -194,30 +186,21 @@ ensure_space(fz_context *ctx, unsigned int tofree)
 	return count;
 }
 
-void
-fz_store_item(fz_context *ctx, fz_obj *key, void *val_, unsigned int itemsize)
+void *
+fz_store_item(fz_context *ctx, void *key, void *val_, unsigned int itemsize, fz_store_type *type)
 {
 	fz_item *item = NULL;
 	unsigned int size;
 	fz_storable *val = (fz_storable *)val_;
 	fz_store *store = ctx->store;
-	int indirect;
-	struct refkey refkey;
+	fz_store_hash hash = { NULL };
+	int use_hash = 0;
 
 	if (!store)
-		return;
+		return NULL;
 
 	fz_var(item);
 
-	/* Form the key before we take the lock */ 
-	indirect = fz_is_indirect(key);
-	if (indirect)
-	{
-		refkey.free = val->free;
-		refkey.num = fz_to_num(key);
-		refkey.gen = fz_to_gen(key);
-	}
-
 	/* If we fail for any reason, we swallow the exception and continue.
 	 * All that the above program will see is that we failed to store
 	 * the item. */
@@ -227,9 +210,16 @@ fz_store_item(fz_context *ctx, fz_obj *key, void *val_, unsigned int itemsize)
 	}
 	fz_catch(ctx)
 	{
-		return;
+		return NULL;
 	}
 
+	if (type->make_hash_key)
+	{
+		hash.free = val->free;
+		use_hash = type->make_hash_key(&hash, key);
+	}
+
+	type->keep_key(ctx, key);
 	fz_lock(ctx, FZ_LOCK_ALLOC);
 	if (store->max != FZ_STORE_UNLIMITED)
 	{
@@ -242,34 +232,41 @@ fz_store_item(fz_context *ctx, fz_obj *key, void *val_, unsigned int itemsize)
 				/* Failed to free any space */
 				fz_unlock(ctx, FZ_LOCK_ALLOC);
 				fz_free(ctx, item);
-				return;
+				type->drop_key(ctx, key);
+				return NULL;
 			}
 		}
 	}
 	store->size += itemsize;
 
-	item->key = fz_keep_obj(key);
+	item->key = key;
 	item->val = val;
 	item->size = itemsize;
 	item->next = NULL;
+	item->type = type;
 
 	/* If we can index it fast, put it into the hash table */
-	if (indirect)
+	if (use_hash)
 	{
-		fz_unlock(ctx, FZ_LOCK_ALLOC);
+		fz_pixmap *existing;
+
 		fz_try(ctx)
 		{
-			fz_hash_insert(ctx, store->hash, &refkey, item);
+			/* May drop and retake the lock */
+			existing = fz_hash_insert(ctx, store->hash, &hash, item);
 		}
 		fz_catch(ctx)
 		{
-			fz_free(ctx, item);
-			fz_lock(ctx, FZ_LOCK_ALLOC);
 			store->size -= itemsize;
 			fz_unlock(ctx, FZ_LOCK_ALLOC);
-			return;
+			fz_free(ctx, item);
+			return NULL;
+		}
+		if (existing)
+		{
+			fz_unlock(ctx, FZ_LOCK_ALLOC);
+			return existing;
 		}
-		fz_lock(ctx, FZ_LOCK_ALLOC);
 	}
 	/* Now we can never fail, bump the ref */
 	if (val->refs > 0)
@@ -283,15 +280,17 @@ fz_store_item(fz_context *ctx, fz_obj *key, void *val_, unsigned int itemsize)
 	store->head = item;
 	item->prev = NULL;
 	fz_unlock(ctx, FZ_LOCK_ALLOC);
+
+	return NULL;
 }
 
 void *
-fz_find_item(fz_context *ctx, fz_store_free_fn *free, fz_obj *key)
+fz_find_item(fz_context *ctx, fz_store_free_fn *free, void *key, fz_store_type *type)
 {
-	struct refkey refkey;
 	fz_item *item;
 	fz_store *store = ctx->store;
-	int indirect;
+	fz_store_hash hash = { NULL };
+	int use_hash = 0;
 
 	if (!store)
 		return NULL;
@@ -299,27 +298,24 @@ fz_find_item(fz_context *ctx, fz_store_free_fn *free, fz_obj *key)
 	if (!key)
 		return NULL;
 
-	/* Form the key before we take the lock */
-	indirect = fz_is_indirect(key);
-	if (indirect)
+	if (type->make_hash_key)
 	{
-		refkey.free = free;
-		refkey.num = fz_to_num(key);
-		refkey.gen = fz_to_gen(key);
+		hash.free = free;
+		use_hash = type->make_hash_key(&hash, key);
 	}
 
 	fz_lock(ctx, FZ_LOCK_ALLOC);
-	if (indirect)
+	if (use_hash)
 	{
 		/* We can find objects keyed on indirected objects quickly */
-		item = fz_hash_find(ctx, store->hash, &refkey);
+		item = fz_hash_find(ctx, store->hash, &hash);
 	}
 	else
 	{
 		/* Others we have to hunt for slowly */
 		for (item = store->head; item; item = item->next)
 		{
-			if (item->val->free == free && !fz_objcmp(item->key, key))
+			if (item->val->free == free && !type->cmp_key(item->key, key))
 				break;
 		}
 	}
@@ -355,35 +351,33 @@ fz_find_item(fz_context *ctx, fz_store_free_fn *free, fz_obj *key)
 }
 
 void
-fz_remove_item(fz_context *ctx, fz_store_free_fn *free, fz_obj *key)
+fz_remove_item(fz_context *ctx, fz_store_free_fn *free, void *key, fz_store_type *type)
 {
-	struct refkey refkey;
 	fz_item *item;
 	fz_store *store = ctx->store;
-	int drop, indirect;
+	int drop;
+	fz_store_hash hash;
+	int use_hash = 0;
 
-	/* Form the key before we take the lock */
-	indirect = fz_is_indirect(key);
-	if (indirect)
+	if (type->make_hash_key)
 	{
-		refkey.free = free;
-		refkey.num = fz_to_num(key);
-		refkey.gen = fz_to_gen(key);
+		hash.free = free;
+		use_hash = type->make_hash_key(&hash, key);
 	}
 
 	fz_lock(ctx, FZ_LOCK_ALLOC);
-	if (indirect)
+	if (use_hash)
 	{
 		/* We can find objects keyed on indirect objects quickly */
-		item = fz_hash_find(ctx, store->hash, &refkey);
+		item = fz_hash_find(ctx, store->hash, &hash);
 		if (item)
-			fz_hash_remove(ctx, store->hash, &refkey);
+			fz_hash_remove(ctx, store->hash, &hash);
 	}
 	else
 	{
 		/* Others we have to hunt for slowly */
 		for (item = store->head; item; item = item->next)
-			if (item->val->free == free && !fz_objcmp(item->key, key))
+			if (item->val->free == free && !type->cmp_key(item->key, key))
 				break;
 	}
 	if (item)
@@ -400,7 +394,7 @@ fz_remove_item(fz_context *ctx, fz_store_free_fn *free, fz_obj *key)
 		fz_unlock(ctx, FZ_LOCK_ALLOC);
 		if (drop)
 			item->val->free(ctx, item->val);
-		fz_drop_obj(item->key);
+		type->drop_key(ctx, item->key);
 		fz_free(ctx, item);
 	}
 	else
diff --git a/fitz/stm_read.c b/fitz/stm_read.c
index 2066b14c..81411c54 100644
--- a/fitz/stm_read.c
+++ b/fitz/stm_read.c
@@ -101,7 +101,7 @@ fz_read_all(fz_stream *stm, int initial)
 		if (initial < 1024)
 			initial = 1024;
 
-		buf = fz_new_buffer(ctx, initial);
+		buf = fz_new_buffer(ctx, initial+1);
 
 		while (1)
 		{
@@ -110,7 +110,6 @@ fz_read_all(fz_stream *stm, int initial)
 
 			if (buf->len / 200 > initial)
 			{
-				fz_drop_buffer(ctx, buf);
 				fz_throw(ctx, "compression bomb detected");
 			}