Initial work on progressive loading

We are testing this using a new -p flag to mupdf that sets a bitrate at
which data will appear to arrive progressively as time goes on. For
example:

  mupdf -p 102400 pdf_reference17.pdf

Details of the scheme used here are presented in docs/progressive.txt

7 files changed, 96 insertions, 0 deletions

diff --git a/include/mupdf/fitz/context.h b/include/mupdf/fitz/context.h
index d7365227..737f163e 100644
--- a/include/mupdf/fitz/context.h
+++ b/include/mupdf/fitz/context.h
@@ -71,11 +71,13 @@ FZ_NORETURN void fz_rethrow_message(fz_context *, const char *, ...)  __printfli
 void fz_warn(fz_context *ctx, const char *fmt, ...) __printflike(2, 3);
 const char *fz_caught_message(fz_context *ctx);
 int fz_caught(fz_context *ctx);
+void fz_rethrow_if(fz_context *ctx, int errcode);
 
 enum
 {
 	FZ_ERROR_NONE = 0,
 	FZ_ERROR_GENERIC = 1,
+	FZ_ERROR_TRYLATER = 2,
 	FZ_ERROR_COUNT
 };
 
diff --git a/include/mupdf/fitz/device.h b/include/mupdf/fitz/device.h
index 7c64c8b8..b39ff4ee 100644
--- a/include/mupdf/fitz/device.h
+++ b/include/mupdf/fitz/device.h
@@ -218,6 +218,16 @@ typedef struct fz_cookie_s fz_cookie;
 	value of progress to that of progress_max.
 
 	errors: count of errors during current rendering.
+
+	incomplete_ok: If this is set to 1 by the caller, then TRYLATER
+	errors are swallowed as they occur, setting the 'incomplete' flag.
+	Rendering continues as much as possible ignoring errors. The caller
+	is expected to check the 'incomplete' flag at the end to see if the
+	rendering may be considered final or not.
+
+	incomplete: Initially should be set to 0. Will be set to non-zero
+	if a TRYLATER error is thrown during rendering and the incomplete_ok
+	flag is set.
 */
 struct fz_cookie_s
 {
@@ -225,6 +235,8 @@ struct fz_cookie_s
 	int progress;
 	int progress_max; /* -1 for unknown */
 	int errors;
+	int incomplete_ok;
+	int incomplete;
 };
 
 /*
diff --git a/include/mupdf/fitz/stream.h b/include/mupdf/fitz/stream.h
index 8fba3379..1f616eef 100644
--- a/include/mupdf/fitz/stream.h
+++ b/include/mupdf/fitz/stream.h
@@ -28,6 +28,9 @@ typedef struct fz_stream_s fz_stream;
 */
 fz_stream *fz_open_file(fz_context *ctx, const char *filename);
 
+fz_stream *fz_open_fd_progressive(fz_context *ctx, int fd, int bps);
+fz_stream *fz_open_file_progressive(fz_context *ctx, const char *filename, int bps);
+
 /*
 	fz_open_file_w: Open the named file and wrap it in a stream.
 
@@ -125,6 +128,14 @@ int fz_read(fz_stream *stm, unsigned char *data, int len);
 */
 fz_buffer *fz_read_all(fz_stream *stm, int initial);
 
+enum
+{
+	FZ_STREAM_META_PROGRESSIVE = 1,
+	FZ_STREAM_META_LENGTH = 2
+};
+
+int fz_stream_meta(fz_stream *stm, int key, int size, void *ptr);
+
 struct fz_stream_s
 {
 	fz_context *ctx;
@@ -139,6 +150,7 @@ struct fz_stream_s
 	int (*read)(fz_stream *stm, unsigned char *buf, int len);
 	void (*close)(fz_context *ctx, void *state);
 	void (*seek)(fz_stream *stm, int offset, int whence);
+	int (*meta)(fz_stream *stm, int key, int size, void *ptr);
 	unsigned char buf[4096];
 };
 
diff --git a/include/mupdf/pdf/document.h b/include/mupdf/pdf/document.h
index 9a23fd7e..3afabfdb 100644
--- a/include/mupdf/pdf/document.h
+++ b/include/mupdf/pdf/document.h
@@ -168,6 +168,18 @@ void pdf_update_page(pdf_document *doc, pdf_page *page);
 */
 int pdf_has_unsaved_changes(pdf_document *doc);
 
+typedef struct pdf_obj_read_state_s pdf_obj_read_state;
+
+struct
+{
+	int offset;
+	int num;
+	int numofs;
+	int gen;
+	int genofs;
+}
+pdf_obj_read_state_s;
+
 struct pdf_document_s
 {
 	fz_document super;
@@ -188,6 +200,52 @@ struct pdf_document_s
 	int freeze_updates;
 
 	int page_count;
+
+	/* State indicating which file parsing method we are using */
+	int file_reading_linearly;
+	int file_length;
+
+	pdf_obj *linear_obj; /* Linearized object (if used) */
+	pdf_obj **linear_page_refs; /* Page objects for linear loading */
+	int linear_page1_obj_num;
+
+	/* The state for the pdf_progressive_advance parser */
+	int linear_pos;
+	int linear_page_num;
+
+	int hint_object_offset;
+	int hint_object_length;
+	int hints_loaded; /* Set to 1 after the hints loading has completed,
+			   * whether successful or not! */
+	/* Page n references shared object references:
+	 *   hint_shared_ref[i]
+	 * where
+	 *      i = s to e-1
+	 *	s = hint_page[n]->index
+	 *	e = hint_page[n+1]->index
+	 * Shared object reference r accesses objects:
+	 *   rs to re-1
+	 * where
+	 *   rs = hint_shared[r]->number
+	 *   re = hint_shared[r]->count + rs
+	 * These are guaranteed to lie within the region starting at
+	 * hint_shared[r]->offset of length hint_shared[r]->length
+	 */
+	struct
+	{
+		int number; /* Page object number */
+		int offset; /* Offset of page object */
+		int index; /* Index into shared hint_shared_ref */
+	} *hint_page;
+	int *hint_shared_ref;
+	struct
+	{
+		int number; /* Object number of first object */
+		int offset; /* Offset of first object */
+	} *hint_shared;
+	int hint_obj_offsets_max;
+	int *hint_obj_offsets;
+
 	int resources_localised;
 
 	pdf_lexbuf_large lexbuf;
diff --git a/include/mupdf/pdf/font.h b/include/mupdf/pdf/font.h
index 90b0fe01..ce140051 100644
--- a/include/mupdf/pdf/font.h
+++ b/include/mupdf/pdf/font.h
@@ -112,6 +112,7 @@ unsigned char *pdf_lookup_substitute_cjk_font(int ros, int serif, unsigned int *
 pdf_font_desc *pdf_load_type3_font(pdf_document *doc, pdf_obj *rdb, pdf_obj *obj);
 void pdf_load_type3_glyphs(pdf_document *doc, pdf_font_desc *fontdesc, int nestedDepth);
 pdf_font_desc *pdf_load_font(pdf_document *doc, pdf_obj *rdb, pdf_obj *obj, int nestedDepth);
+pdf_font_desc *pdf_load_hail_mary_font(pdf_document *doc);
 
 pdf_font_desc *pdf_new_font_desc(fz_context *ctx);
 pdf_font_desc *pdf_keep_font(fz_context *ctx, pdf_font_desc *fontdesc);
diff --git a/include/mupdf/pdf/page.h b/include/mupdf/pdf/page.h
index c1fffa57..f0a8de0d 100644
--- a/include/mupdf/pdf/page.h
+++ b/include/mupdf/pdf/page.h
@@ -91,6 +91,13 @@ struct pdf_page_s
 	float duration;
 	int transition_present;
 	fz_transition transition;
+	int incomplete;
+};
+
+enum
+{
+	PDF_PAGE_INCOMPLETE_CONTENTS = 1,
+	PDF_PAGE_INCOMPLETE_ANNOTS = 2
 };
 
 #endif
diff --git a/include/mupdf/pdf/xref.h b/include/mupdf/pdf/xref.h
index 5d4ecde9..de53da7c 100644
--- a/include/mupdf/pdf/xref.h
+++ b/include/mupdf/pdf/xref.h
@@ -82,6 +82,10 @@ void pdf_repair_xref(pdf_document *doc, pdf_lexbuf *buf);
 void pdf_repair_obj_stms(pdf_document *doc);
 pdf_obj *pdf_new_ref(pdf_document *doc, pdf_obj *obj);
 
+int pdf_repair_obj(pdf_document *doc, pdf_lexbuf *buf, int *stmofsp, int *stmlenp, pdf_obj **encrypt, pdf_obj **id, pdf_obj **page, int *tmpofs);
+
+pdf_obj *pdf_progressive_advance(pdf_document *doc, int pagenum);
+
 void pdf_print_xref(pdf_document *);
 
 #endif