Skip to content

Commit 94e538a

Browse files
author
Rostislav Pehlivanov
committed
vc2enc_dwt: pad the temporary buffer by the slice size
Since non-Haar wavelets need to look into pixels outside the frame, we need to pad the buffer. The old factor of two seemed to be a workaround that fact and only padded to the left and bottom. This correctly pads by the slice size and as such reduces memory usage and potential exploits. Reported by Liu Bingchang. Ideally, there should be no temporary buffer but the encoder is designed to deinterleave the coefficients into the classical wavelet structure with the lower frequency values in the top left corner. Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com> (cherry picked from commit 3228ac7)
1 parent 479e65b commit 94e538a

File tree

3 files changed

+15
-6
lines changed

3 files changed

+15
-6
lines changed

Diff for: libavcodec/vc2enc.c

+3-2
Original file line numberDiff line numberDiff line change
@@ -1106,8 +1106,9 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx)
11061106

11071107
/* DWT init */
11081108
if (ff_vc2enc_init_transforms(&s->transform_args[i].t,
1109-
s->plane[0].coef_stride,
1110-
s->plane[0].dwt_height))
1109+
s->plane[i].coef_stride,
1110+
s->plane[i].dwt_height,
1111+
s->slice_width, s->slice_height))
11111112
goto alloc_fail;
11121113
}
11131114

Diff for: libavcodec/vc2enc_dwt.c

+9-3
Original file line numberDiff line numberDiff line change
@@ -211,19 +211,25 @@ static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data,
211211
deinterleave(data, stride, width, height, synth);
212212
}
213213

214-
av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_width, int p_height)
214+
av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride,
215+
int p_height, int slice_w, int slice_h)
215216
{
216217
s->vc2_subband_dwt[VC2_TRANSFORM_9_7] = vc2_subband_dwt_97;
217218
s->vc2_subband_dwt[VC2_TRANSFORM_5_3] = vc2_subband_dwt_53;
218219

219-
s->buffer = av_malloc(2*p_width*p_height*sizeof(dwtcoef));
220+
/* Pad by the slice size, only matters for non-Haar wavelets */
221+
s->buffer = av_calloc((p_stride + slice_w)*(p_height + slice_h), sizeof(dwtcoef));
220222
if (!s->buffer)
221223
return 1;
222224

225+
s->padding = (slice_h >> 1)*p_stride + (slice_w >> 1);
226+
s->buffer += s->padding;
227+
223228
return 0;
224229
}
225230

226231
av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s)
227232
{
228-
av_freep(&s->buffer);
233+
av_free(s->buffer - s->padding);
234+
s->buffer = NULL;
229235
}

Diff for: libavcodec/vc2enc_dwt.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,14 @@ enum VC2TransformType {
4444

4545
typedef struct VC2TransformContext {
4646
dwtcoef *buffer;
47+
int padding;
4748
void (*vc2_subband_dwt[VC2_TRANSFORMS_NB])(struct VC2TransformContext *t,
4849
dwtcoef *data, ptrdiff_t stride,
4950
int width, int height);
5051
} VC2TransformContext;
5152

52-
int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_width, int p_height);
53+
int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_stride, int p_height,
54+
int slice_w, int slice_h);
5355
void ff_vc2enc_free_transforms(VC2TransformContext *t);
5456

5557
#endif /* AVCODEC_VC2ENC_DWT_H */

0 commit comments

Comments
 (0)