Permalink
Browse files

Fix overflow in integer calculations (#4970)

  • Loading branch information...
szekerest committed Aug 14, 2014
1 parent 310d993 commit fcd7cf3a15d256dc96e4f0dd914dd2ae08a66596
Showing with 9 additions and 5 deletions.
  1. +9 −5 renderers/agg/include/agg_rasterizer_cells_aa.h
@@ -317,16 +317,20 @@ namespace mapserver
{
enum dx_limit_e { dx_limit = 16384 << poly_subpixel_shift };
- int dx = x2 - x1;
-
- if(dx >= dx_limit || dx <= -dx_limit)
+ // Checking the limit and avoid integer overflows
+ if(((x1 < x2)? ((x1 < 0)? (x1 + dx_limit <= x2) : (x1 <= x2 - dx_limit)) :
+ ((x2 < 0)? (x2 + dx_limit <= x1) : (x2 <= x1 - dx_limit))) ||
+ ((y1 < y2)? ((y1 < 0)? (y1 + dx_limit <= y2) : (y1 <= y2 - dx_limit)) :
+ ((y2 < 0)? (y2 + dx_limit <= y1) : (y2 <= y1 - dx_limit))))
{
- int cx = (x1 + x2) >> 1;
- int cy = (y1 + y2) >> 1;
+ int cx = (x2&x1) + ((x2^x1)>>1);
+ int cy = (y2&y1) + ((y2^y1)>>1);
+
line(x1, y1, cx, cy);
line(cx, cy, x2, y2);
}
+ int dx = x2 - x1;
int dy = y2 - y1;
int ex1 = x1 >> poly_subpixel_shift;
int ex2 = x2 >> poly_subpixel_shift;

0 comments on commit fcd7cf3

Please sign in to comment.