Skip to content

Commit

Permalink
Fix integer overflow in agg
Browse files Browse the repository at this point in the history
  • Loading branch information
szekerest committed Aug 14, 2014
1 parent 87054b0 commit 5b9f95d
Showing 1 changed file with 10 additions and 6 deletions.
16 changes: 10 additions & 6 deletions renderers/agg/include/agg_rasterizer_cells_aa.h
Original file line number Diff line number Diff line change
Expand Up @@ -316,17 +316,21 @@ namespace mapserver
void rasterizer_cells_aa<Cell>::line(int x1, int y1, int x2, int y2)
{
enum dx_limit_e { dx_limit = 16384 << poly_subpixel_shift };

int dx = x2 - x1;

if(dx >= dx_limit || dx <= -dx_limit)

// Checking the limit and avoid integer overflows
if(((x1 < x2)? ((x1 < 0)? (x1 + dx_limit <= x2) : (x1 <= x2 - dx_limit)) :
((x2 < 0)? (x2 + dx_limit <= x1) : (x2 <= x1 - dx_limit))) ||
((y1 < y2)? ((y1 < 0)? (y1 + dx_limit <= y2) : (y1 <= y2 - dx_limit)) :
((y2 < 0)? (y2 + dx_limit <= y1) : (y2 <= y1 - dx_limit))))
{
int cx = (x1 + x2) >> 1;
int cy = (y1 + y2) >> 1;
int cx = (x2&x1) + ((x2^x1)>>1);
int cy = (y2&y1) + ((y2^y1)>>1);

line(x1, y1, cx, cy);
line(cx, cy, x2, y2);
}

int dx = x2 - x1;
int dy = y2 - y1;
int ex1 = x1 >> poly_subpixel_shift;
int ex2 = x2 >> poly_subpixel_shift;
Expand Down

0 comments on commit 5b9f95d

Please sign in to comment.