Skip to content

Commit

Permalink
fix_gcd: Handle smallest 64-bit Fixnums correctly.
Browse files Browse the repository at this point in the history
  • Loading branch information
kstephens committed Jan 21, 2010
1 parent 2f5d9ce commit 4863695
Showing 1 changed file with 4 additions and 6 deletions.
10 changes: 4 additions & 6 deletions numeric.c
Original file line number Diff line number Diff line change
Expand Up @@ -3028,17 +3028,15 @@ fix_gcd(int argc, VALUE *argv, VALUE self) {
rb_raise(rb_eArgError, "wrong number of arguments (%d for %d)", argc, 1);
} else {
/* Handle Fixnum#gcd(Fixnum) here.
* Note: Cannot handle FIXNUM_MIN here due to overflow during negation.
* Note: Cannot handle values <= FIXNUM_MIN here due to overflow during negation.
*/
long a = FIX2LONG(self);
long b = FIX2LONG(argv[0]);
long a, b;

if ( a != FIXNUM_MIN && FIXNUM_P(argv[0]) && b != FIXNUM_MIN ) {
/* fprintf(stderr, "Using Fixnum#gcd(Fixnum)\n"); */
if ( FIXNUM_P(argv[0]) && (a = FIX2LONG(self)) > FIXNUM_MIN && (b = FIX2LONG(argv[0])) > FIXNUM_MIN ) {
long min = a < 0 ? - a : a;
long max = b < 0 ? - b : b;
while ( min > 0 ) {
int tmp = min;
long tmp = min;
min = max % min;
max = tmp;
}
Expand Down

0 comments on commit 4863695

Please sign in to comment.