Skip to content
Permalink
Browse files Browse the repository at this point in the history
vc2enc_dwt: pad the temporary buffer by the slice size
Since non-Haar wavelets need to look into pixels outside the frame, we
need to pad the buffer. The old factor of two seemed to be a workaround
that fact and only padded to the left and bottom. This correctly pads
by the slice size and as such reduces memory usage and potential
exploits.
Reported by Liu Bingchang.

Ideally, there should be no temporary buffer but the encoder is designed
to deinterleave the coefficients into the classical wavelet structure
with the lower frequency values in the top left corner.

Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>
(cherry picked from commit 3228ac7)
  • Loading branch information
Rostislav Pehlivanov committed Nov 9, 2017
1 parent 479e65b commit 94e538a
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 6 deletions.
5 changes: 3 additions & 2 deletions libavcodec/vc2enc.c
Expand Up @@ -1106,8 +1106,9 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)

/* DWT init */
if (ff_vc2enc_init_transforms(&s->transform_args[i].t,
s->plane[0].coef_stride,
s->plane[0].dwt_height))
s->plane[i].coef_stride,
s->plane[i].dwt_height,
s->slice_width, s->slice_height))
goto alloc_fail;
}

Expand Down
12 changes: 9 additions & 3 deletions libavcodec/vc2enc_dwt.c
Expand Up @@ -211,19 +211,25 @@ static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data,
deinterleave(data, stride, width, height, synth);
}

av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_width, int p_height)
av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride,
int p_height, int slice_w, int slice_h)
{
s->vc2_subband_dwt[VC2_TRANSFORM_9_7] = vc2_subband_dwt_97;
s->vc2_subband_dwt[VC2_TRANSFORM_5_3] = vc2_subband_dwt_53;

s->buffer = av_malloc(2*p_width*p_height*sizeof(dwtcoef));
/* Pad by the slice size, only matters for non-Haar wavelets */
s->buffer = av_calloc((p_stride + slice_w)*(p_height + slice_h), sizeof(dwtcoef));
if (!s->buffer)
return 1;

s->padding = (slice_h >> 1)*p_stride + (slice_w >> 1);
s->buffer += s->padding;

return 0;
}

av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s)
{
av_freep(&s->buffer);
av_free(s->buffer - s->padding);
s->buffer = NULL;
}
4 changes: 3 additions & 1 deletion libavcodec/vc2enc_dwt.h
Expand Up @@ -44,12 +44,14 @@ enum VC2TransformType {

typedef struct VC2TransformContext {
dwtcoef *buffer;
int padding;
void (*vc2_subband_dwt[VC2_TRANSFORMS_NB])(struct VC2TransformContext *t,
dwtcoef *data, ptrdiff_t stride,
int width, int height);
} VC2TransformContext;

int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_width, int p_height);
int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_stride, int p_height,
int slice_w, int slice_h);
void ff_vc2enc_free_transforms(VC2TransformContext *t);

#endif /* AVCODEC_VC2ENC_DWT_H */

0 comments on commit 94e538a

Please sign in to comment.