prevent deadlock under memory pressure

In multiple places, between acquiring and releasing the FREETYPE lock,
exceptions may be thrown which aren't caught in order to properly
release the lock. This patch introduces the necessary fz_try/fz_always/
fz_catch invocations to prevent a potential deadlock in these situations.

RJW: Also fix another problem pointed out by zeniko. Thanks!

3 files changed, 37 insertions, 6 deletions

diff --git a/fitz/base_hash.c b/fitz/base_hash.c
index 1da8cac9..22b64ec6 100644
--- a/fitz/base_hash.c
+++ b/fitz/base_hash.c
@@ -160,7 +160,7 @@ fz_resize_hash(fz_context *ctx, fz_hash_table *table, int newsize)
 
 	if (table->lock == FZ_LOCK_ALLOC)
 		fz_unlock(ctx, FZ_LOCK_ALLOC);
-	newents = fz_malloc_array(ctx, newsize, sizeof(fz_hash_entry));
+	newents = fz_malloc_array_no_throw(ctx, newsize, sizeof(fz_hash_entry));
 	if (table->lock == FZ_LOCK_ALLOC)
 		fz_lock(ctx, FZ_LOCK_ALLOC);
 	if (table->lock >= 0)
@@ -176,6 +176,8 @@ fz_resize_hash(fz_context *ctx, fz_hash_table *table, int newsize)
 			return;
 		}
 	}
+	if (newents == NULL)
+		fz_throw(ctx, "hash table resize failed; out of memory (%d entries)", newsize);
 	table->ents = newents;
 	memset(table->ents, 0, sizeof(fz_hash_entry) * newsize);
 	table->size = newsize;
diff --git a/fitz/res_font.c b/fitz/res_font.c
index 8fb51377..3263f88c 100644
--- a/fitz/res_font.c
+++ b/fitz/res_font.c
@@ -515,8 +515,19 @@ retry_unhinted:
 		return NULL;
 	}
 
-	result = fz_copy_ft_bitmap(ctx, face->glyph->bitmap_left, face->glyph->bitmap_top, &face->glyph->bitmap);
-	fz_unlock(ctx, FZ_LOCK_FREETYPE);
+	fz_try(ctx)
+	{
+		result = fz_copy_ft_bitmap(ctx, face->glyph->bitmap_left, face->glyph->bitmap_top, &face->glyph->bitmap);
+	}
+	fz_always(ctx)
+	{
+		fz_unlock(ctx, FZ_LOCK_FREETYPE);
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
+
 	return result;
 }
 
@@ -623,9 +634,19 @@ fz_render_ft_stroked_glyph(fz_context *ctx, fz_font *font, int gid, const fz_mat
 	}
 
 	bitmap = (FT_BitmapGlyph)glyph;
-	pixmap = fz_copy_ft_bitmap(ctx, bitmap->left, bitmap->top, &bitmap->bitmap);
-	FT_Done_Glyph(glyph);
-	fz_unlock(ctx, FZ_LOCK_FREETYPE);
+	fz_try(ctx)
+	{
+		pixmap = fz_copy_ft_bitmap(ctx, bitmap->left, bitmap->top, &bitmap->bitmap);
+	}
+	fz_always(ctx)
+	{
+		FT_Done_Glyph(glyph);
+		fz_unlock(ctx, FZ_LOCK_FREETYPE);
+	}
+	fz_catch(ctx)
+	{
+		fz_rethrow(ctx);
+	}
 
 	return pixmap;
 }
diff --git a/pdf/pdf_font.c b/pdf/pdf_font.c
index de43796f..742602e3 100644
--- a/pdf/pdf_font.c
+++ b/pdf/pdf_font.c
@@ -416,10 +416,12 @@ pdf_load_simple_font(pdf_document *xref, pdf_obj *dict)
 	char ebuffer[256][32];
 	int i, k, n;
 	int fterr;
+	int has_lock = 0;
 	fz_context *ctx = xref->ctx;
 
 	fz_var(fontdesc);
 	fz_var(etable);
+	fz_var(has_lock);
 
 	basefont = pdf_to_name(pdf_dict_gets(dict, "BaseFont"));
 
@@ -556,6 +558,7 @@ pdf_load_simple_font(pdf_document *xref, pdf_obj *dict)
 			etable[i] = ft_char_index(face, i);
 
 		fz_lock(ctx, FZ_LOCK_FREETYPE);
+		has_lock = 1;
 
 		/* built-in and substitute fonts may be a different type than what the document expects */
 		subtype = pdf_to_name(pdf_dict_gets(dict, "Subtype"));
@@ -673,6 +676,7 @@ pdf_load_simple_font(pdf_document *xref, pdf_obj *dict)
 		}
 
 		fz_unlock(ctx, FZ_LOCK_FREETYPE);
+		has_lock = 0;
 
 		fontdesc->encoding = pdf_new_identity_cmap(ctx, 0, 1);
 		fontdesc->size += pdf_cmap_size(ctx, fontdesc->encoding);
@@ -714,6 +718,7 @@ pdf_load_simple_font(pdf_document *xref, pdf_obj *dict)
 		else
 		{
 			fz_lock(ctx, FZ_LOCK_FREETYPE);
+			has_lock = 1;
 			fterr = FT_Set_Char_Size(face, 1000, 1000, 72, 72);
 			if (fterr)
 				fz_warn(ctx, "freetype set character size: %s", ft_error_string(fterr));
@@ -722,12 +727,15 @@ pdf_load_simple_font(pdf_document *xref, pdf_obj *dict)
 				pdf_add_hmtx(ctx, fontdesc, i, i, ft_width(ctx, fontdesc, i));
 			}
 			fz_unlock(ctx, FZ_LOCK_FREETYPE);
+			has_lock = 0;
 		}
 
 		pdf_end_hmtx(ctx, fontdesc);
 	}
 	fz_catch(ctx)
 	{
+		if (has_lock)
+			fz_unlock(ctx, FZ_LOCK_FREETYPE);
 		if (fontdesc && etable != fontdesc->cid_to_gid)
 			fz_free(ctx, etable);
 		pdf_drop_font(ctx, fontdesc);