Switch to reading content streams on the fly during interpretation.

Previously, before interpreting a pages content stream we would
load it entirely into a buffer. Then we would interpret that
buffer. This has a cost in memory use.

Here, we update the code to read from a stream on the fly.

This has required changes in various different parts of the code.

Firstly, we have removed all use of the FILE lock - as stream
reads can now safely be interrupted by resource (or object) reads
from elsewhere in the file, the file lock becomes a very hard
thing to maintain, and doesn't actually benefit us at all. The
choices were to either use a recursive lock, or to remove it
entirely; I opted for the latter.

The file lock enum value remains as a placeholder for future use in
extendable data streams.

Secondly, we add a new 'concat' filter that concatenates a series of
streams together into one, optionally putting whitespace between each
stream (as the pdf parser requires this).

Finally, we change page/xobject/pattern content streams to work
on the fly, but we leave type3 glyphs using buffers (as presumably
these will be run repeatedly).

4 files changed, 103 insertions, 21 deletions

diff --git a/fitz/filt_basic.c b/fitz/filt_basic.c
index ac6a5903..7d504f29 100644
--- a/fitz/filt_basic.c
+++ b/fitz/filt_basic.c
@@ -62,6 +62,104 @@ fz_open_null(fz_stream *chain, int len, int offset)
 	return fz_new_stream(ctx, state, read_null, close_null);
 }
 
+/* Concat filter concatenates several streams into one */
+
+struct concat_filter
+{
+	int max;
+	int count;
+	int current;
+	int pad; /* 1 if we should add whitespace padding between streams */
+	int ws; /* 1 if we should send a whitespace padding byte next */
+	fz_stream *chain[1];
+};
+
+static int
+read_concat(fz_stream *stm, unsigned char *buf, int len)
+{
+	struct concat_filter *state = (struct concat_filter *)stm->state;
+	int n;
+	int read = 0;
+
+	if (len <= 0)
+		return 0;
+
+	while (state->current != state->count && len > 0)
+	{
+		/* If we need to send a whitespace char, do that */
+		if (state->ws)
+		{
+			*buf++ = 32;
+			read++;
+			len--;
+			state->ws = 0;
+			continue;
+		}
+		/* Otherwise, read as much data as will fit in the buffer */
+		n = fz_read(state->chain[state->current], buf, len);
+		read += n;
+		buf += n;
+		len -= n;
+		/* If we didn't read any, then we must have hit the end of
+		 * our buffer space. Move to the next stream, and remember to
+		 * pad. */
+		if (n == 0)
+		{
+			fz_close(state->chain[state->current]);
+			state->current++;
+			state->ws = state->pad;
+		}
+	}
+
+	return read;
+}
+
+static void
+close_concat(fz_context *ctx, void *state_)
+{
+	struct concat_filter *state = (struct concat_filter *)state_;
+	int i;
+
+	for (i = state->current; i < state->count; i++)
+	{
+		fz_close(state->chain[i]);
+	}
+	fz_free(ctx, state);
+}
+
+fz_stream *
+fz_open_concat(fz_context *ctx, int len, int pad)
+{
+	struct concat_filter *state;
+
+	fz_try(ctx)
+	{
+		state = fz_calloc(ctx, 1, sizeof(struct concat_filter) + (len-1)*sizeof(fz_stream *));
+		state->max = len;
+		state->count = 0;
+		state->current = 0;
+		state->pad = pad;
+		state->ws = 0; /* We never send padding byte at the start */
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
+
+	return fz_new_stream(ctx, state, read_concat, close_concat);
+}
+
+void
+fz_concat_push(fz_stream *concat, fz_stream *chain)
+{
+	struct concat_filter *state = (struct concat_filter *)concat->state;
+
+	if (state->count == state->max)
+		fz_throw(concat->ctx, "Concat filter size exceeded");
+
+	state->chain[state->count++] = chain;
+}
+
 /* ASCII Hex Decode */
 
 typedef struct fz_ahxd_s fz_ahxd;
diff --git a/fitz/fitz-internal.h b/fitz/fitz-internal.h
index 197d0279..b3f7a73c 100644
--- a/fitz/fitz-internal.h
+++ b/fitz/fitz-internal.h
@@ -419,7 +419,6 @@ struct fz_stream_s
 	int pos;
 	int avail;
 	int bits;
-	int locked;
 	unsigned char *bp, *rp, *wp, *ep;
 	void *state;
 	int (*read)(fz_stream *stm, unsigned char *buf, int len);
@@ -428,8 +427,6 @@ struct fz_stream_s
 	unsigned char buf[4096];
 };
 
-void fz_lock_stream(fz_stream *stm);
-
 fz_stream *fz_new_stream(fz_context *ctx, void*, int(*)(fz_stream*, unsigned char*, int), void(*)(fz_context *, void *));
 fz_stream *fz_keep_stream(fz_stream *stm);
 void fz_fill_buffer(fz_stream *stm);
@@ -521,6 +518,8 @@ static inline int fz_is_eof_bits(fz_stream *stm)
 
 fz_stream *fz_open_copy(fz_stream *chain);
 fz_stream *fz_open_null(fz_stream *chain, int len, int offset);
+fz_stream *fz_open_concat(fz_context *ctx, int max, int pad);
+void fz_concat_push(fz_stream *concat, fz_stream *chain); /* Ownership of chain is passed in */
 fz_stream *fz_open_arc4(fz_stream *chain, unsigned char *key, unsigned keylen);
 fz_stream *fz_open_aesd(fz_stream *chain, unsigned char *key, unsigned keylen);
 fz_stream *fz_open_a85d(fz_stream *chain);
diff --git a/fitz/res_font.c b/fitz/res_font.c
index c279c75b..966cbc6e 100644
--- a/fitz/res_font.c
+++ b/fitz/res_font.c
@@ -692,7 +692,7 @@ static fz_rect
 fz_bound_t3_glyph(fz_context *ctx, fz_font *font, int gid, fz_matrix trm)
 {
 	fz_matrix ctm;
-	fz_buffer *contents;
+	void *contents;
 	fz_rect bounds;
 	fz_bbox bbox;
 	fz_device *dev;
@@ -726,7 +726,7 @@ fz_pixmap *
 fz_render_t3_glyph(fz_context *ctx, fz_font *font, int gid, fz_matrix trm, fz_colorspace *model)
 {
 	fz_matrix ctm;
-	fz_buffer *contents;
+	void *contents;
 	fz_bbox bbox;
 	fz_device *dev;
 	fz_pixmap *glyph;
@@ -786,7 +786,7 @@ void
 fz_render_t3_glyph_direct(fz_context *ctx, fz_device *dev, fz_font *font, int gid, fz_matrix trm, void *gstate)
 {
 	fz_matrix ctm;
-	fz_buffer *contents;
+	void *contents;
 
 	if (gid < 0 || gid > 255)
 		return;
diff --git a/fitz/stm_open.c b/fitz/stm_open.c
index ced32d80..be069fb9 100644
--- a/fitz/stm_open.c
+++ b/fitz/stm_open.c
@@ -24,7 +24,6 @@ fz_new_stream(fz_context *ctx, void *state,
 
 	stm->bits = 0;
 	stm->avail = 0;
-	stm->locked = 0;
 
 	stm->bp = stm->buf;
 	stm->rp = stm->bp;
@@ -40,16 +39,6 @@ fz_new_stream(fz_context *ctx, void *state,
 	return stm;
 }
 
-void
-fz_lock_stream(fz_stream *stm)
-{
-	if (stm)
-	{
-		fz_lock(stm->ctx, FZ_LOCK_FILE);
-		stm->locked = 1;
-	}
-}
-
 fz_stream *
 fz_keep_stream(fz_stream *stm)
 {
@@ -67,8 +56,6 @@ fz_close(fz_stream *stm)
 	{
 		if (stm->close)
 			stm->close(stm->ctx, stm->state);
-		if (stm->locked)
-			fz_unlock(stm->ctx, FZ_LOCK_FILE);
 		fz_free(stm->ctx, stm);
 	}
 }
@@ -78,7 +65,6 @@ fz_close(fz_stream *stm)
 static int read_file(fz_stream *stm, unsigned char *buf, int len)
 {
 	int n = read(*(int*)stm->state, buf, len);
-	fz_assert_lock_held(stm->ctx, FZ_LOCK_FILE);
 	if (n < 0)
 		fz_throw(stm->ctx, "read error: %s", strerror(errno));
 	return n;
@@ -87,7 +73,6 @@ static int read_file(fz_stream *stm, unsigned char *buf, int len)
 static void seek_file(fz_stream *stm, int offset, int whence)
 {
 	int n = lseek(*(int*)stm->state, offset, whence);
-	fz_assert_lock_held(stm->ctx, FZ_LOCK_FILE);
 	if (n < 0)
 		fz_throw(stm->ctx, "cannot lseek: %s", strerror(errno));
 	stm->pos = n;