From e78eafeabaf35e29a8cba12ac4dd05284d0b1cb2 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Fri, 28 Aug 2015 22:47:43 -0700 Subject: Merge to XFA: Check array bounds for opj_dwt_decode_1() and friends. Based on an earlier patch by jun_fang@foxitsoftware.com. BUG=450844 R=jun_fang@foxitsoftware.com Review URL: https://codereview.chromium.org/1320443003 . (cherry picked from commit 463b77b4f1e4257cd89f3460b5a6fdb102f44265) Review URL: https://codereview.chromium.org/1319303003 . --- third_party/libopenjpeg20/dwt.c | 64 +++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/third_party/libopenjpeg20/dwt.c b/third_party/libopenjpeg20/dwt.c index bea4574257..9b40c746db 100644 --- a/third_party/libopenjpeg20/dwt.c +++ b/third_party/libopenjpeg20/dwt.c @@ -46,14 +46,12 @@ /** @defgroup DWT DWT - Implementation of a discrete wavelet transform */ /*@{*/ -#define OPJ_WS(i) v->mem[(i)*2] -#define OPJ_WD(i) v->mem[(1+(i)*2)] - /** @name Local data structures */ /*@{*/ typedef struct dwt_local { OPJ_INT32* mem; + OPJ_SIZE_T mem_count; OPJ_INT32 dn; OPJ_INT32 sn; OPJ_INT32 cas; @@ -107,16 +105,16 @@ static void opj_dwt_interleave_v(opj_dwt_t* v, OPJ_INT32 *a, OPJ_INT32 x); /** Forward 5-3 wavelet transform in 1-D */ -static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); +static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); /** Inverse 5-3 wavelet transform in 1-D */ static void opj_dwt_decode_1(opj_dwt_t *v); -static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); +static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); /** Forward 9-7 wavelet transform in 1-D */ -static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); +static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas); /** Explicit calculation of the Quantization Stepsizes */ @@ -124,10 +122,10 @@ static void opj_dwt_encode_stepsize(OPJ_INT32 stepsize, OPJ_INT32 numbps, opj_st /** Inverse wavelet transform in 2-D. */ -static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 i, DWT1DFN fn); +static OPJ_BOOL opj_dwt_decode_tile(const opj_tcd_tilecomp_t* tilec, OPJ_UINT32 i, DWT1DFN fn); -static OPJ_BOOL opj_dwt_encode_procedure( opj_tcd_tilecomp_t * tilec, - void (*p_function)(OPJ_INT32 *, OPJ_INT32,OPJ_INT32,OPJ_INT32) ); +static OPJ_BOOL opj_dwt_encode_procedure(const opj_tcd_tilecomp_t * tilec, + void (*p_function)(OPJ_INT32 *, OPJ_SIZE_T,OPJ_INT32,OPJ_INT32,OPJ_INT32) ); static OPJ_UINT32 opj_dwt_max_resolution(opj_tcd_resolution_t* restrict r, OPJ_UINT32 i); @@ -156,13 +154,20 @@ static void opj_v4dwt_decode_step2(opj_v4_t* l, opj_v4_t* w, OPJ_INT32 k, OPJ_IN /*@}*/ -#define OPJ_S(i) a[(i)*2] -#define OPJ_D(i) a[(1+(i)*2)] -#define OPJ_S_(i) ((i)<0?OPJ_S(0):((i)>=sn?OPJ_S(sn-1):OPJ_S(i))) -#define OPJ_D_(i) ((i)<0?OPJ_D(0):((i)>=dn?OPJ_D(dn-1):OPJ_D(i))) +#define IDX_S(i) (i)*2 +#define IDX_D(i) 1+(i)*2 +#define UNDERFLOW_SN(i) ((i)>=sn&&sn>0) +#define UNDERFLOW_DN(i) ((i)>=dn&&dn>0) +#define OVERFLOW_S(i) (IDX_S(i)>=a_count) +#define OVERFLOW_D(i) (IDX_D(i)>=a_count) + +#define OPJ_S(i) a[IDX_S(i)] +#define OPJ_D(i) a[IDX_D(i)] +#define OPJ_S_(i) ((i)<0?OPJ_S(0):(UNDERFLOW_SN(i)?OPJ_S(sn-1):OVERFLOW_S(i)?OPJ_S(i-1):OPJ_S(i))) +#define OPJ_D_(i) ((i)<0?OPJ_D(0):(UNDERFLOW_DN(i)?OPJ_D(dn-1):OVERFLOW_D(i)?OPJ_D(i-1):OPJ_D(i))) /* new */ -#define OPJ_SS_(i) ((i)<0?OPJ_S(0):((i)>=dn?OPJ_S(dn-1):OPJ_S(i))) -#define OPJ_DD_(i) ((i)<0?OPJ_D(0):((i)>=sn?OPJ_D(sn-1):OPJ_D(i))) +#define OPJ_SS_(i) ((i)<0?OPJ_S(0):(UNDERFLOW_DN(i)?OPJ_S(dn-1):OVERFLOW_S(i)?OPJ_S(i-1):OPJ_S(i))) +#define OPJ_DD_(i) ((i)<0?OPJ_D(0):(UNDERFLOW_SN(i)?OPJ_D(sn-1):OVERFLOW_D(i)?OPJ_D(i-1):OPJ_D(i))) /* */ /* This table contains the norms of the 5-3 wavelets for different bands. */ @@ -283,7 +288,7 @@ void opj_dwt_interleave_v(opj_dwt_t* v, OPJ_INT32 *a, OPJ_INT32 x) { /* */ /* Forward 5-3 wavelet transform in 1-D. */ /* */ -void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { +void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { OPJ_INT32 i; if (!cas) { @@ -304,7 +309,7 @@ void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { /* */ /* Inverse 5-3 wavelet transform in 1-D. */ /* */ -void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { +void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { OPJ_INT32 i; if (!cas) { @@ -326,13 +331,13 @@ void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) /* Inverse 5-3 wavelet transform in 1-D. */ /* */ void opj_dwt_decode_1(opj_dwt_t *v) { - opj_dwt_decode_1_(v->mem, v->dn, v->sn, v->cas); + opj_dwt_decode_1_(v->mem, v->mem_count, v->dn, v->sn, v->cas); } /* */ /* Forward 9-7 wavelet transform in 1-D. */ /* */ -void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { +void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) { OPJ_INT32 i; if (!cas) { if ((dn > 0) || (sn > 1)) { /* NEW : CASE ONE ELEMENT */ @@ -385,7 +390,7 @@ void opj_dwt_encode_stepsize(OPJ_INT32 stepsize, OPJ_INT32 numbps, opj_stepsize_ /* */ /* Forward 5-3 wavelet transform in 2-D. */ /* */ -INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_function)(OPJ_INT32 *, OPJ_INT32,OPJ_INT32,OPJ_INT32) ) +INLINE OPJ_BOOL opj_dwt_encode_procedure(const opj_tcd_tilecomp_t * tilec,void (*p_function)(OPJ_INT32 *, OPJ_SIZE_T,OPJ_INT32,OPJ_INT32,OPJ_INT32) ) { OPJ_INT32 i, j, k; OPJ_INT32 *a = 00; @@ -395,7 +400,8 @@ INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_fun OPJ_INT32 rw; /* width of the resolution level computed */ OPJ_INT32 rh; /* height of the resolution level computed */ - OPJ_UINT32 l_data_size; + OPJ_SIZE_T l_data_count; + OPJ_SIZE_T l_data_size; opj_tcd_resolution_t * l_cur_res = 0; opj_tcd_resolution_t * l_last_res = 0; @@ -407,8 +413,9 @@ INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_fun l_cur_res = tilec->resolutions + l; l_last_res = l_cur_res - 1; - l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32); - bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size); + l_data_count = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions); + l_data_size = l_data_count * (OPJ_UINT32)sizeof(OPJ_INT32); + bj = (OPJ_INT32*)opj_malloc(l_data_size); if (! bj) { return OPJ_FALSE; } @@ -437,7 +444,7 @@ INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_fun bj[k] = aj[k*w]; } - (*p_function) (bj, dn, sn, cas_col); + (*p_function) (bj, l_data_count, dn, sn, cas_col); opj_dwt_deinterleave_v(bj, aj, dn, sn, w, cas_col); } @@ -448,7 +455,7 @@ INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_fun for (j = 0; j < rh; j++) { aj = a + j * w; for (k = 0; k < rw; k++) bj[k] = aj[k]; - (*p_function) (bj, dn, sn, cas_row); + (*p_function) (bj, l_data_count, dn, sn, cas_row); opj_dwt_deinterleave_h(bj, aj, dn, sn, cas_row); } @@ -557,7 +564,7 @@ OPJ_UINT32 opj_dwt_max_resolution(opj_tcd_resolution_t* restrict r, OPJ_UINT32 i /* */ /* Inverse wavelet transform in 2-D. */ /* */ -OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres, DWT1DFN dwt_1D) { +OPJ_BOOL opj_dwt_decode_tile(const opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres, DWT1DFN dwt_1D) { opj_dwt_t h; opj_dwt_t v; @@ -568,13 +575,14 @@ OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres, DWT1D OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); - h.mem = (OPJ_INT32*) - opj_aligned_malloc(opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32)); + h.mem_count = opj_dwt_max_resolution(tr, numres); + h.mem = (OPJ_INT32*)opj_aligned_malloc(h.mem_count * sizeof(OPJ_INT32)); if (! h.mem){ /* FIXME event manager error callback */ return OPJ_FALSE; } + v.mem_count = h.mem_count; v.mem = h.mem; while( --numres) { -- cgit v1.2.3