Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix overflows in outline processing #432

Merged
merged 1 commit into from Oct 8, 2020
Merged

Fix overflows in outline processing #432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
62 changes: 53 additions & 9 deletions libass/ass_outline.c
View file
Open in desktop
Expand Up @@ -48,6 +48,11 @@ static void outline_clear(ASS_Outline *outline)
outline->n_segments = outline->max_segments = 0;
}

static bool valid_point(const FT_Vector *pt)
{
return abs(pt->x) <= OUTLINE_MAX && abs(pt->y) <= OUTLINE_MAX;
}

bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
{
if (!source || !source->n_points) {
Expand Down Expand Up @@ -77,12 +82,16 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
continue;
}

if (!valid_point(source->points + j))
goto fail;
switch (FT_CURVE_TAG(source->tags[j])) {
case FT_CURVE_TAG_ON:
st = S_ON;
break;

case FT_CURVE_TAG_CONIC:
if (!valid_point(source->points + last))
goto fail;
pt.x = source->points[last].x;
pt.y = -source->points[last].y;
switch (FT_CURVE_TAG(source->tags[last])) {
Expand Down Expand Up @@ -110,6 +119,8 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
outline->points[outline->n_points++] = pt;

for (j++; j <= last; j++) {
if (!valid_point(source->points + j))
goto fail;
switch (FT_CURVE_TAG(source->tags[j])) {
case FT_CURVE_TAG_ON:
switch (st) {
Expand Down Expand Up @@ -209,13 +220,34 @@ bool outline_scale_pow2(ASS_Outline *outline, const ASS_Outline *source,
return true;
}

int32_t lim_x = OUTLINE_MAX;
if (scale_ord_x > 0)
lim_x = scale_ord_x < 32 ? lim_x >> scale_ord_x : 0;
else
scale_ord_x = FFMAX(scale_ord_x, -32);

int32_t lim_y = OUTLINE_MAX;
if (scale_ord_y > 0)
lim_y = scale_ord_y < 32 ? lim_y >> scale_ord_y : 0;
else
scale_ord_y = FFMAX(scale_ord_y, -32);
astiob marked this conversation as resolved.
Show resolved Hide resolved

if (!lim_x || !lim_y) {
outline_clear(outline);
return false;
}

if (!outline_alloc(outline, source->n_points, source->n_segments))
return false;

int sx = scale_ord_x + 32;
int sy = scale_ord_y + 32;
const ASS_Vector *pt = source->points;
for (size_t i = 0; i < source->n_points; i++) {
if (abs(pt[i].x) > lim_x || abs(pt[i].y) > lim_y) {
outline_free(outline);
return false;
}
// that's equivalent to pt[i].x << scale_ord_x,
// but works even for negative coordinate and/or shift amount
outline->points[i].x = pt[i].x * ((int64_t) 1 << sx) >> 32;
Expand Down Expand Up @@ -244,6 +276,10 @@ bool outline_transform_2d(ASS_Outline *outline, const ASS_Outline *source,
for (int k = 0; k < 2; k++)
v[k] = m[k][0] * pt[i].x + m[k][1] * pt[i].y + m[k][2];

if (!(fabs(v[0]) < OUTLINE_MAX && fabs(v[1]) < OUTLINE_MAX)) {
outline_free(outline);
return false;
}
outline->points[i].x = lrint(v[0]);
outline->points[i].y = lrint(v[1]);
}
Expand Down Expand Up @@ -271,8 +307,15 @@ bool outline_transform_3d(ASS_Outline *outline, const ASS_Outline *source,
v[k] = m[k][0] * pt[i].x + m[k][1] * pt[i].y + m[k][2];

double w = 1 / FFMAX(v[2], 0.1);
outline->points[i].x = lrint(v[0] * w);
outline->points[i].y = lrint(v[1] * w);
v[0] *= w;
v[1] *= w;

if (!(fabs(v[0]) < OUTLINE_MAX && fabs(v[1]) < OUTLINE_MAX)) {
outline_free(outline);
return false;
}
outline->points[i].x = lrint(v[0]);
outline->points[i].y = lrint(v[1]);
}
memcpy(outline->segments, source->segments, source->n_segments);
outline->n_points = source->n_points;
Expand All @@ -299,6 +342,9 @@ void outline_free(ASS_Outline *outline)
*/
bool outline_add_point(ASS_Outline *outline, ASS_Vector pt, char segment)
{
if(abs(pt.x) > OUTLINE_MAX || abs(pt.y) > OUTLINE_MAX)
return false;

if (outline->n_points >= outline->max_points) {
size_t new_size = 2 * outline->max_points;
if (!ASS_REALLOC_ARRAY(outline->points, new_size))
Expand Down Expand Up @@ -1351,7 +1397,7 @@ bool outline_stroke(ASS_Outline *result, ASS_Outline *result1,

const int dir = 3;
int rad = FFMAX(xbord, ybord);
assert(rad >= eps);
assert(rad >= eps && rad <= OUTLINE_MAX);

StrokerState str;
str.result[0] = result;
Expand All @@ -1374,12 +1420,10 @@ bool outline_stroke(ASS_Outline *result, ASS_Outline *result1,
str.err_c = 390 * rel_err * rel_err;
str.err_a = e;

for (size_t i = 0; i < path->n_points; i++) {
if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX)
return false;
if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX)
return false;
}
#ifndef NDEBUG
for (size_t i = 0; i < path->n_points; i++)
assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX);
#endif

ASS_Vector *start = path->points, *cur = start;
for (size_t i = 0; i < path->n_segments; i++) {
Expand Down
3 changes: 2 additions & 1 deletion libass/ass_outline.h
View file
Open in desktop
Expand Up @@ -83,8 +83,9 @@ typedef struct {
char *segments;
} ASS_Outline;

#define OUTLINE_MIN (-((int32_t) 1 << 28))
// ouline point coordinates should always be in [-OUTLINE_MAX, +OUTLINE_MAX] range
#define OUTLINE_MAX (((int32_t) 1 << 28) - 1)
// cubic spline splitting requires 8 * OUTLINE_MAX + 4 <= INT32_MAX

bool outline_alloc(ASS_Outline *outline, size_t n_points, size_t n_segments);
bool outline_convert(ASS_Outline *outline, const FT_Outline *source);
Expand Down
10 changes: 4 additions & 6 deletions libass/ass_rasterizer.c
View file
Open in desktop
Expand Up @@ -269,12 +269,10 @@ bool rasterizer_set_outline(RasterizerData *rst,
}
rst->size[0] = rst->n_first;

for (size_t i = 0; i < path->n_points; i++) {
if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX)
return false;
if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX)
return false;
}
#ifndef NDEBUG
for (size_t i = 0; i < path->n_points; i++)
assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX);
#endif

ASS_Vector *start = path->points, *cur = start;
for (size_t i = 0; i < path->n_segments; i++) {
Expand Down
8 changes: 6 additions & 2 deletions libass/ass_render.c
View file
Open in desktop
Expand Up @@ -1438,8 +1438,12 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info,
w *= STROKER_PRECISION / POSITION_PRECISION;
frexp(w * (FFMAX(mxx, myx) + mzx * rz), &k->scale_ord_x);
frexp(w * (FFMAX(mxy, myy) + mzy * rz), &k->scale_ord_y);
k->border.x = lrint(ldexp(bord_x, k->scale_ord_x) / STROKER_PRECISION);
k->border.y = lrint(ldexp(bord_y, k->scale_ord_y) / STROKER_PRECISION);
bord_x = ldexp(bord_x, k->scale_ord_x);
bord_y = ldexp(bord_y, k->scale_ord_y);
if (!(bord_x < OUTLINE_MAX && bord_y < OUTLINE_MAX))
return;
k->border.x = lrint(bord_x / STROKER_PRECISION);
k->border.y = lrint(bord_y / STROKER_PRECISION);
if (!k->border.x && !k->border.y) {
ass_cache_inc_ref(info->bm);
info->bm_o = info->bm;
Expand Down