Bug 691691: Add way of clearing cached objects out of the xref.

We add various facilities here, intended to allow us to efficiently
minimise the memory we use for holding cached pdf objects.

Firstly, we add the ability to 'mark' all the currently loaded objects.

Next we add the ability to 'clear the xref' - to drop all the currently
loaded objects that have no other references except the ones held by the
xref table itself.

Finally, we add the ability to 'clear the xref to the last mark' - to
drop all the currently loaded objects that have been created since the
last 'mark' operation and have no other references except the ones held
by the xref table.

We expose this to the user by adding a new device hint 'FZ_NO_CACHE'.
If set on the device, then the PDF interpreter will pdf_mark_xref before
starting and pdf_clear_xref_to_mark afterwards. Thus no additional
objects will be retained in memory after a given page is run, unless
someone else picks them up and takes a reference to them as part of
the run.

We amend our simple example app to set this device hint when loading
pages as part of a search.

7 files changed, 174 insertions, 35 deletions

diff --git a/include/mupdf/fitz/device.h b/include/mupdf/fitz/device.h
index a95a5ab1..25959424 100644
--- a/include/mupdf/fitz/device.h
+++ b/include/mupdf/fitz/device.h
@@ -206,6 +206,7 @@ enum
 	FZ_IGNORE_SHADE = 2,
 	FZ_DONT_INTERPOLATE_IMAGES = 4,
 	FZ_MAINTAIN_CONTAINER_STACK = 8,
+	FZ_NO_CACHE = 16,
 };
 
 /*
diff --git a/include/mupdf/pdf/object.h b/include/mupdf/pdf/object.h
index 0f0a48d2..2c9600a2 100644
--- a/include/mupdf/pdf/object.h
+++ b/include/mupdf/pdf/object.h
@@ -106,6 +106,8 @@ void pdf_sort_dict(pdf_obj *dict);
 */
 void pdf_set_obj_parent(pdf_obj *obj, int num);
 
+int pdf_obj_refs(pdf_obj *ref);
+
 int pdf_obj_parent_num(pdf_obj *obj);
 
 int pdf_sprint_obj(char *s, int n, pdf_obj *obj, int tight);
diff --git a/include/mupdf/pdf/xref.h b/include/mupdf/pdf/xref.h
index f9e1657f..5f1a49d1 100644
--- a/include/mupdf/pdf/xref.h
+++ b/include/mupdf/pdf/xref.h
@@ -35,13 +35,19 @@ typedef struct pdf_xref_entry_s pdf_xref_entry;
 struct pdf_xref_entry_s
 {
 	char type;	/* 0=unset (f)ree i(n)use (o)bjstm */
+	unsigned char flags; /* bit 0 = marked */
+	unsigned short gen;	/* generation / objstm index */
 	int ofs;	/* file offset / objstm object number */
-	int gen;	/* generation / objstm index */
 	int stm_ofs;	/* on-disk stream */
 	fz_buffer *stm_buf; /* in-memory stream (for updated objects) */
 	pdf_obj *obj;	/* stored/cached object */
 };
 
+enum
+{
+	PDF_OBJ_FLAG_MARK = 1,
+};
+
 struct pdf_xref_s
 {
 	int len;
@@ -83,6 +89,10 @@ void pdf_repair_xref(pdf_document *doc, pdf_lexbuf *buf);
 void pdf_repair_obj_stms(pdf_document *doc);
 pdf_obj *pdf_new_ref(pdf_document *doc, pdf_obj *obj);
 
+void pdf_mark_xref(pdf_document *doc);
+void pdf_clear_xref(pdf_document *doc);
+void pdf_clear_xref_to_mark(pdf_document *doc);
+
 int pdf_repair_obj(pdf_document *doc, pdf_lexbuf *buf, int *stmofsp, int *stmlenp, pdf_obj **encrypt, pdf_obj **id, pdf_obj **page, int *tmpofs);
 
 pdf_obj *pdf_progressive_advance(pdf_document *doc, int pagenum);
diff --git a/platform/x11/pdfapp.c b/platform/x11/pdfapp.c
index cade7509..bb619614 100644
--- a/platform/x11/pdfapp.c
+++ b/platform/x11/pdfapp.c
@@ -26,7 +26,7 @@ enum
 	PDFAPP_OUTLINE_LOAD_NOW = 2
 };
 
-static void pdfapp_showpage(pdfapp_t *app, int loadpage, int drawpage, int repaint, int transition);
+static void pdfapp_showpage(pdfapp_t *app, int loadpage, int drawpage, int repaint, int transition, int searching);
 static void pdfapp_updatepage(pdfapp_t *app);
 
 static void pdfapp_warn(pdfapp_t *app, const char *fmt, ...)
@@ -330,7 +330,7 @@ void pdfapp_open_progressive(pdfapp_t *app, char *filename, int reload, int bps)
 		app->pany = 0;
 	}
 
-	pdfapp_showpage(app, 1, 1, 1, 0);
+	pdfapp_showpage(app, 1, 1, 1, 0, 0);
 }
 
 void pdfapp_close(pdfapp_t *app)
@@ -512,7 +512,7 @@ static void pdfapp_panview(pdfapp_t *app, int newx, int newy)
 	app->pany = newy;
 }
 
-static void pdfapp_loadpage(pdfapp_t *app)
+static void pdfapp_loadpage(pdfapp_t *app, int no_cache)
 {
 	fz_device *mdev = NULL;
 	int errored = 0;
@@ -561,6 +561,8 @@ static void pdfapp_loadpage(pdfapp_t *app)
 		/* Create display lists */
 		app->page_list = fz_new_display_list(app->ctx);
 		mdev = fz_new_list_device(app->ctx, app->page_list);
+		if (no_cache)
+			fz_enable_device_hints(mdev, FZ_NO_CACHE);
 		cookie.incomplete_ok = 1;
 		fz_run_page_contents(app->doc, app->page, mdev, &fz_identity, &cookie);
 		fz_free_device(mdev);
@@ -688,7 +690,7 @@ static void pdfapp_updatepage(pdfapp_t *app)
 		fz_free_device(idev);
 	}
 
-	pdfapp_showpage(app, 0, 0, 1, 0);
+	pdfapp_showpage(app, 0, 0, 1, 0, 0);
 }
 
 void pdfapp_reloadpage(pdfapp_t *app)
@@ -705,10 +707,10 @@ void pdfapp_reloadpage(pdfapp_t *app)
 		}
 		app->outline_deferred = 0;
 	}
-	pdfapp_showpage(app, 1, 1, 1, 0);
+	pdfapp_showpage(app, 1, 1, 1, 0, 0);
 }
 
-static void pdfapp_showpage(pdfapp_t *app, int loadpage, int drawpage, int repaint, int transition)
+static void pdfapp_showpage(pdfapp_t *app, int loadpage, int drawpage, int repaint, int transition, int searching)
 {
 	char buf[MAX_TITLE];
 	fz_device *idev;
@@ -733,7 +735,7 @@ static void pdfapp_showpage(pdfapp_t *app, int loadpage, int drawpage, int repai
 
 	if (loadpage)
 	{
-		pdfapp_loadpage(app);
+		pdfapp_loadpage(app, searching);
 
 		/* Zero search hit position */
 		app->hit_count = 0;
@@ -878,7 +880,7 @@ void pdfapp_gotopage(pdfapp_t *app, int number)
 	}
 	app->hist[app->histlen++] = app->pageno;
 	app->pageno = number + 1;
-	pdfapp_showpage(app, 1, 1, 1, 0);
+	pdfapp_showpage(app, 1, 1, 1, 0, 0);
 }
 
 void pdfapp_inverthit(pdfapp_t *app)
@@ -916,7 +918,7 @@ static void pdfapp_search_in_direction(pdfapp_t *app, enum panning *panto, int d
 		if (page != app->pageno)
 		{
 			app->pageno = page;
-			pdfapp_showpage(app, 1, 0, 0, 0);
+			pdfapp_showpage(app, 1, 0, 0, 0, 1);
 		}
 
 		app->hit_count = fz_search_text_page(app->ctx, app->page_text, app->search, app->hit_bbox, nelem(app->hit_bbox));
@@ -937,7 +939,7 @@ static void pdfapp_search_in_direction(pdfapp_t *app, enum panning *panto, int d
 	pdfapp_warn(app, "String '%s' not found.", app->search);
 
 	app->pageno = firstpage;
-	pdfapp_showpage(app, 1, 0, 0, 0);
+	pdfapp_showpage(app, 1, 0, 0, 0, 0);
 	wincursor(app, HAND);
 	winrepaint(app);
 }
@@ -982,7 +984,7 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 							app->pageno = app->pagecount;
 						else
 							app->pageno--;
-						pdfapp_showpage(app, 1, 1, 0, 0);
+						pdfapp_showpage(app, 1, 1, 0, 0, 1);
 					}
 
 					pdfapp_onkey(app, 'n');
@@ -1034,13 +1036,13 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 		app->resolution *= ZOOMSTEP;
 		if (app->resolution > MAXRES)
 			app->resolution = MAXRES;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 	case '-':
 		app->resolution /= ZOOMSTEP;
 		if (app->resolution < MINRES)
 			app->resolution = MINRES;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 
 	case 'W':
@@ -1049,7 +1051,7 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 			app->resolution = MAXRES;
 		else if (app->resolution < MINRES)
 			app->resolution = MINRES;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 	case 'H':
 		app->resolution *= (double) app->winh / (double) fz_pixmap_height(app->ctx, app->image);
@@ -1057,36 +1059,36 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 			app->resolution = MAXRES;
 		else if (app->resolution < MINRES)
 			app->resolution = MINRES;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 
 	case 'L':
 		app->rotate -= 90;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 	case 'R':
 		app->rotate += 90;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 
 	case 'c':
 		app->grayscale ^= 1;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 
 	case 'i':
 		app->invert ^= 1;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 
 #ifndef NDEBUG
 	case 'a':
 		app->rotate -= 15;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 	case 's':
 		app->rotate += 15;
-		pdfapp_showpage(app, 0, 1, 1, 0);
+		pdfapp_showpage(app, 0, 1, 1, 0, 0);
 		break;
 #endif
 
@@ -1108,27 +1110,27 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 		}
 		app->shrinkwrap = 1;
 		app->panx = app->pany = 0;
-		pdfapp_showpage(app, 0, 0, 1, 0);
+		pdfapp_showpage(app, 0, 0, 1, 0, 0);
 		break;
 
 	case 'h':
 		app->panx += fz_pixmap_width(app->ctx, app->image) / 10;
-		pdfapp_showpage(app, 0, 0, 1, 0);
+		pdfapp_showpage(app, 0, 0, 1, 0, 0);
 		break;
 
 	case 'j':
 		app->pany -= fz_pixmap_height(app->ctx, app->image) / 10;
-		pdfapp_showpage(app, 0, 0, 1, 0);
+		pdfapp_showpage(app, 0, 0, 1, 0, 0);
 		break;
 
 	case 'k':
 		app->pany += fz_pixmap_height(app->ctx, app->image) / 10;
-		pdfapp_showpage(app, 0, 0, 1, 0);
+		pdfapp_showpage(app, 0, 0, 1, 0, 0);
 		break;
 
 	case 'l':
 		app->panx -= fz_pixmap_width(app->ctx, app->image) / 10;
-		pdfapp_showpage(app, 0, 0, 1, 0);
+		pdfapp_showpage(app, 0, 0, 1, 0, 0);
 		break;
 
 	/*
@@ -1308,7 +1310,7 @@ void pdfapp_onkey(pdfapp_t *app, int c)
 		case DONT_PAN:
 			break;
 		}
-		pdfapp_showpage(app, loadpage, 1, 1, 1);
+		pdfapp_showpage(app, loadpage, 1, 1, 1, 0);
 	}
 }
 
@@ -1508,7 +1510,7 @@ void pdfapp_onmouse(pdfapp_t *app, int x, int y, int btn, int modifiers, int sta
 					app->resolution = MAXRES;
 				if (app->resolution < MINRES)
 					app->resolution = MINRES;
-				pdfapp_showpage(app, 0, 1, 1, 0);
+				pdfapp_showpage(app, 0, 1, 1, 0, 0);
 			}
 			else
 			{
@@ -1574,7 +1576,7 @@ void pdfapp_onmouse(pdfapp_t *app, int x, int y, int btn, int modifiers, int sta
 					if( app->pageno > 1 )
 					{
 						app->pageno--;
-						pdfapp_showpage(app, 1, 1, 1, 0);
+						pdfapp_showpage(app, 1, 1, 1, 0, 0);
 						newy = -fz_pixmap_height(app->ctx, app->image);
 					}
 					app->beyondy = 0;
@@ -1584,7 +1586,7 @@ void pdfapp_onmouse(pdfapp_t *app, int x, int y, int btn, int modifiers, int sta
 					if( app->pageno < app->pagecount )
 					{
 						app->pageno++;
-						pdfapp_showpage(app, 1, 1, 1, 0);
+						pdfapp_showpage(app, 1, 1, 1, 0, 0);
 						newy = 0;
 					}
 					app->beyondy = 0;
diff --git a/source/pdf/pdf-object.c b/source/pdf/pdf-object.c
index a77f5a62..1576525e 100644
--- a/source/pdf/pdf-object.c
+++ b/source/pdf/pdf-object.c
@@ -1778,3 +1778,8 @@ pdf_print_ref(pdf_obj *ref)
 	pdf_print_obj(pdf_resolve_indirect(ref));
 }
 #endif
+
+int pdf_obj_refs(pdf_obj *ref)
+{
+	return (ref ? ref->refs : 0);
+}
diff --git a/source/pdf/pdf-run.c b/source/pdf/pdf-run.c
index 4ac92bb0..a77dd50e 100644
--- a/source/pdf/pdf-run.c
+++ b/source/pdf/pdf-run.c
@@ -36,7 +36,24 @@ static void pdf_run_page_contents_with_usage(pdf_document *doc, pdf_page *page,
 
 void pdf_run_page_contents(pdf_document *doc, pdf_page *page, fz_device *dev, const fz_matrix *ctm, fz_cookie *cookie)
 {
-	pdf_run_page_contents_with_usage(doc, page, dev, ctm, "View", cookie);
+	fz_context *ctx = doc->ctx;
+	int nocache = !!(dev->hints & FZ_NO_CACHE);
+
+	if (nocache)
+		pdf_mark_xref(doc);
+	fz_try(ctx)
+	{
+		pdf_run_page_contents_with_usage(doc, page, dev, ctm, "View", cookie);
+	}
+	fz_always(ctx)
+	{
+		if (nocache)
+			pdf_clear_xref_to_mark(doc);
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
 	if (page->incomplete & PDF_PAGE_INCOMPLETE_CONTENTS)
 		fz_throw(doc->ctx, FZ_ERROR_TRYLATER, "incomplete rendering");
 }
@@ -44,7 +61,24 @@ void pdf_run_page_contents(pdf_document *doc, pdf_page *page, fz_device *dev, co
 
 void pdf_run_annot(pdf_document *doc, pdf_page *page, pdf_annot *annot, fz_device *dev, const fz_matrix *ctm, fz_cookie *cookie)
 {
-	pdf_run_annot_with_usage(doc, page, annot, dev, ctm, "View", cookie);
+	fz_context *ctx = doc->ctx;
+	int nocache = !!(dev->hints & FZ_NO_CACHE);
+
+	if (nocache)
+		pdf_mark_xref(doc);
+	fz_try(ctx)
+	{
+		pdf_run_annot_with_usage(doc, page, annot, dev, ctm, "View", cookie);
+	}
+	fz_always(ctx)
+	{
+		if (nocache)
+			pdf_clear_xref_to_mark(doc);
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
 	if (page->incomplete & PDF_PAGE_INCOMPLETE_ANNOTS)
 		fz_throw(doc->ctx, FZ_ERROR_TRYLATER, "incomplete rendering");
 }
@@ -78,8 +112,25 @@ static void pdf_run_page_annots_with_usage(pdf_document *doc, pdf_page *page, fz
 void
 pdf_run_page_with_usage(pdf_document *doc, pdf_page *page, fz_device *dev, const fz_matrix *ctm, char *event, fz_cookie *cookie)
 {
-	pdf_run_page_contents_with_usage(doc, page, dev, ctm, event, cookie);
-	pdf_run_page_annots_with_usage(doc, page, dev, ctm, event, cookie);
+	fz_context *ctx = doc->ctx;
+	int nocache = !!(dev->hints & FZ_NO_CACHE);
+
+	if (nocache)
+		pdf_mark_xref(doc);
+	fz_try(ctx)
+	{
+		pdf_run_page_contents_with_usage(doc, page, dev, ctm, event, cookie);
+		pdf_run_page_annots_with_usage(doc, page, dev, ctm, event, cookie);
+	}
+	fz_always(ctx)
+	{
+		if (nocache)
+			pdf_clear_xref_to_mark(doc);
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
 	if (page->incomplete)
 		fz_throw(doc->ctx, FZ_ERROR_TRYLATER, "incomplete rendering");
 }
diff --git a/source/pdf/pdf-xref.c b/source/pdf/pdf-xref.c
index 1cf89de9..f9b72a4d 100644
--- a/source/pdf/pdf-xref.c
+++ b/source/pdf/pdf-xref.c
@@ -2458,3 +2458,71 @@ fz_document_handler pdf_no_run_document_handler =
 	(fz_document_open_fn *)&pdf_open_document_no_run,
 	(fz_document_open_with_stream_fn *)&pdf_open_document_no_run_with_stream
 };
+
+void pdf_mark_xref(pdf_document *doc)
+{
+	int x, e;
+
+	for (x = 0; x < doc->num_xref_sections; x++)
+	{
+		pdf_xref *xref = &doc->xref_sections[x];
+
+		for (e = 0; e < xref->len; e++)
+		{
+			pdf_xref_entry *entry = &xref->table[e];
+
+			if (entry->obj)
+			{
+				entry->flags |= PDF_OBJ_FLAG_MARK;
+			}
+		}
+	}
+}
+
+void pdf_clear_xref(pdf_document *doc)
+{
+	int x, e;
+
+	for (x = 0; x < doc->num_xref_sections; x++)
+	{
+		pdf_xref *xref = &doc->xref_sections[x];
+
+		for (e = 0; e < xref->len; e++)
+		{
+			pdf_xref_entry *entry = &xref->table[e];
+
+			if (entry->obj)
+			{
+				if (pdf_obj_refs(entry->obj) == 1)
+				{
+					pdf_drop_obj(entry->obj);
+					entry->obj = NULL;
+				}
+			}
+		}
+	}
+}
+
+void pdf_clear_xref_to_mark(pdf_document *doc)
+{
+	int x, e;
+
+	for (x = 0; x < doc->num_xref_sections; x++)
+	{
+		pdf_xref *xref = &doc->xref_sections[x];
+
+		for (e = 0; e < xref->len; e++)
+		{
+			pdf_xref_entry *entry = &xref->table[e];
+
+			if (entry->obj)
+			{
+				if ((entry->flags & PDF_OBJ_FLAG_MARK) == 0 && pdf_obj_refs(entry->obj) == 1)
+				{
+					pdf_drop_obj(entry->obj);
+					entry->obj = NULL;
+				}
+			}
+		}
+	}
+}