PR#6321: guarantee that "hypot infinity nan = infinity"

Changes

@@ -162,6 +162,8 @@ Standard library:
   (Xavier Leroy)
 - PR#6316: Scanf.scanf failure on %u formats when reading big integers
   (Xavier Leroy, Benoît Vaugon)
+- PR#6321: guarantee that "hypot infinity nan = infinity"
+  (for conformance with ISO C99)  (Xavier Leroy)
 - PR#6390, GPR#36: expose Sys.{int_size,max_wosize} for improved js_of_ocaml
   portability (Hugo Heuzard)
 * PR#6494: Add equal function in modules

byterun/floats.c

@@ -17,6 +17,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <float.h>
 #include <limits.h>
 
 #include "caml/alloc.h"
@@ -468,9 +469,11 @@ CAMLexport double caml_hypot(double x, double y)
   return hypot(x, y);
 #else
   double tmp, ratio;
-  if (x != x) return x;  /* NaN */
-  if (y != y) return y;  /* NaN */
   x = fabs(x); y = fabs(y);
+  if (x != x) /* x is NaN */
+    return y > DBL_MAX ? y : x;  /* PR#6321 */
+  if (y != y) /* y is NaN */
+    return x > DBL_MAX ? x : y;  /* PR#6321 */
   if (x < y) { tmp = x; x = y; y = tmp; }
   if (x == 0.0) return 0.0;
   ratio = y / x;

stdlib/pervasives.mli

@@ -401,7 +401,8 @@ external hypot : float -> float -> float = "caml_hypot_float" "caml_hypot"
 (** [hypot x y] returns [sqrt(x *. x + y *. y)], that is, the length
   of the hypotenuse of a right-angled triangle with sides of length
   [x] and [y], or, equivalently, the distance of the point [(x,y)]
-  to origin.
+  to origin.  If one of [x] or [y] is infinite, returns [infinity]
+  even if the other is [nan].
   @since 4.00.0  *)
 
 external cosh : float -> float = "caml_cosh_float" "cosh"