[libc] add exception to atof differential fuzz

The differential fuzzer for atof found a bug in glibc's handling of
hexadecimal rounding. Since we can't easily update glibc and we want to
avoid false positives when running the fuzzer, I've added an exception
to skip all hexadecimal subnormal cases.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D149359

libc/fuzzing/stdlib/CMakeLists.txt

@@ -14,6 +14,8 @@ add_libc_fuzzer(
     StringParserOutputDiff.h
   DEPENDS
     libc.src.stdlib.atof
+  COMPILE_OPTIONS
+    -DLLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
 )
 
 add_libc_fuzzer(

libc/fuzzing/stdlib/atof_differential_fuzz.cpp

@@ -16,6 +16,40 @@
 
 #include "fuzzing/stdlib/StringParserOutputDiff.h"
 
+// TODO: Remove this once glibc fixes hex subnormal rounding. See
+// https://sourceware.org/bugzilla/show_bug.cgi?id=30220
+#ifdef LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+#include <ctype.h>
+constexpr double MIN_NORMAL = 0x1p-1022;
+
+bool has_hex_prefix(const uint8_t *str) {
+  size_t index = 0;
+
+  // Skip over leading whitespace
+  while (isspace(str[index])) {
+    ++index;
+  }
+  // Skip over sign
+  if (str[index] == '-' || str[index] == '+') {
+    ++index;
+  }
+  return str[index] == '0' && (tolower(str[index + 1])) == 'x';
+}
+
+bool should_be_skipped(const uint8_t *str) {
+  double init_result = ::atof(reinterpret_cast<const char *>(str));
+  if (init_result < 0) {
+    init_result = -init_result;
+  }
+  if (init_result < MIN_NORMAL && init_result != 0) {
+    return has_hex_prefix(str);
+  }
+  return false;
+}
+#else
+bool should_be_skipped(const uint8_t *) { return false; }
+#endif // LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   uint8_t *container = new uint8_t[size + 1];
   if (!container)
@@ -26,7 +60,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     container[i] = data[i];
   container[size] = '\0'; // Add null terminator to container.
 
-  StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container, size);
+  if (!should_be_skipped(container)) {
+    StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container,
+                                   size);
+  }
   delete[] container;
   return 0;
 }