151 changes: 115 additions & 36 deletions libc/src/math/generic/expm1f.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//===-- Implementation of expm1f function ---------------------------------===//
//===-- Single-precision e^x - 1 function ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand All @@ -7,52 +7,131 @@
//===----------------------------------------------------------------------===//

#include "src/math/expm1f.h"
#include "common_constants.h" // Lookup tables EXP_M1 and EXP_M2.
#include "src/__support/FPUtil/BasicOperations.h"
#include "src/__support/FPUtil/FEnvImpl.h"
#include "src/__support/FPUtil/FMA.h"
#include "src/__support/FPUtil/FPBits.h"
#include "src/__support/FPUtil/PolyEval.h"
#include "src/__support/common.h"
#include "src/math/expf.h"

#include <errno.h>

namespace __llvm_libc {

// When |x| > Ln2, catastrophic cancellation does not occur with the
// subtraction expf(x) - 1.0f, so we use it to compute expm1f(x).
//
// We divide [-Ln2; Ln2] into 3 subintervals [-Ln2; -1/8], [-1/8; 1/8],
// [1/8; Ln2]. And we use a degree-6 polynomial to approximate exp(x) - 1 in
// each interval. The coefficients were generated by Sollya's fpminmax.
//
// See libc/utils/mathtools/expm1f.sollya for more detail.
INLINE_FMA
LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
const float ln2 =
0.69314718055994530941723212145817656807550013436025f; // For C++17:
// 0x1.62e'42ffp-1
float abs_x = __llvm_libc::fputil::abs(x);

if (abs_x <= ln2) {
if (abs_x <= 0.125f) {
return x * __llvm_libc::fputil::polyeval(
x, 1.0f, 0.5f, 0.16666664183139801025390625f,
4.1666664183139801025390625e-2f,
8.3379410207271575927734375e-3f,
1.3894210569560527801513671875e-3f);
using FPBits = typename fputil::FPBits<float>;
FPBits xbits(x);

// When x < log(2^-25) or nan
if (unlikely(xbits.uintval() >= 0xc18a'a123U)) {
// exp(-Inf) = 0
if (xbits.is_inf())
return -1.0f;
// exp(nan) = nan
if (xbits.is_nan())
return x;
int round_mode = fputil::get_round();
if (round_mode == FE_UPWARD || round_mode == FE_TOWARDZERO)
return -0x1.ffff'fep-1f; // -1.0f + 0x1.0p-24f
return -1.0f;
}
// x >= 89 or nan
if (unlikely(!xbits.get_sign() && (xbits.uintval() >= 0x42b2'0000))) {
if (xbits.uintval() < 0x7f80'0000U) {
int rounding = fputil::get_round();
if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO)
return static_cast<float>(FPBits(FPBits::MAX_NORMAL));

errno = ERANGE;
}
if (x > 0.125f) {
return __llvm_libc::fputil::polyeval(
x, 1.23142086749794543720781803131103515625e-7f,
0.9999969005584716796875f, 0.500031292438507080078125f,
0.16650259494781494140625f, 4.21491153538227081298828125e-2f,
7.53940828144550323486328125e-3f,
2.05591344274580478668212890625e-3f);
return x + static_cast<float>(FPBits::inf());
}

int unbiased_exponent = static_cast<int>(xbits.get_unbiased_exponent());
// |x| < 2^-4
if (unbiased_exponent < 123) {
// |x| < 2^-25
if (unbiased_exponent < 102) {
// x = -0.0f
if (unlikely(xbits.uintval() == 0x8000'0000U))
return x;
// When |x| < 2^-25, the relative error:
// |(e^x - 1) - x| / |x| < |x^2| / |x| = |x| < 2^-25 < epsilon(1)/2.
// So the correctly rounded values of expm1(x) are:
// = x + eps(x) if rounding mode = FE_UPWARD,
// or (rounding mode = FE_TOWARDZERO and x is negative),
// = x otherwise.
// To simplify the rounding decision and make it more efficient, we use
// fma(x, x, x) ~ x + x^2 instead.
return fputil::fma(x, x, x);
}
return __llvm_libc::fputil::polyeval(
x, -6.899231408397099585272371768951416015625e-8f,
0.999998271465301513671875f, 0.4999825656414031982421875f,
0.16657467186450958251953125f, 4.1390590369701385498046875e-2f,
7.856394164264202117919921875e-3f,
9.380675037391483783721923828125e-4f);
// 2^-25 <= |x| < 2^-4
double xd = static_cast<double>(x);
double xsq = xd * xd;
// Degree-8 minimax polynomial generated by Sollya with:
// > display = hexadecimal;
// > P = fpminimax(expm1(x)/x, 7, [|D...|], [-2^-4, 2^-4]);
double r =
fputil::polyeval(xd, 0x1p-1, 0x1.55555555559abp-3, 0x1.55555555551a7p-5,
0x1.111110f70f2a4p-7, 0x1.6c16c17639e82p-10,
0x1.a02526febbea6p-13, 0x1.a01dc40888fcdp-16);
return static_cast<float>(fputil::fma(r, xsq, xd));
}

// For -18 < x < 89, to compute exp(x), we perform the following range
// reduction: find hi, mid, lo such that:
// x = hi + mid + lo, in which
// hi is an integer,
// mid * 2^7 is an integer
// -2^(-8) <= lo < 2^-8.
// In particular,
// hi + mid = round(x * 2^7) * 2^(-7).
// Then,
// exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo).
// We store exp(hi) and exp(mid) in the lookup tables EXP_M1 and EXP_M2
// respectively. exp(lo) is computed using a degree-7 minimax polynomial
// generated by Sollya.

// Exceptional value
if (xbits.uintval() == 0xbdc1'c6cbU) {
// x = -0x1.838d96p-4f
int round_mode = fputil::get_round();
if (round_mode == FE_TONEAREST || round_mode == FE_DOWNWARD)
return -0x1.71c884p-4f;
return -0x1.71c882p-4f;
}

// x_hi = hi + mid.
int x_hi = static_cast<int>(x * 0x1.0p7f);
// Subtract (hi + mid) from x to get lo.
x -= static_cast<float>(x_hi) * 0x1.0p-7f;
double xd = static_cast<double>(x);
// Make sure that -2^(-8) <= lo < 2^-8.
if (x >= 0x1.0p-8f) {
++x_hi;
xd -= 0x1.0p-7;
}
if (x < -0x1.0p-8f) {
--x_hi;
xd += 0x1.0p-7;
}
return expf(x) - 1.0f;
x_hi += 104 << 7;
// hi = x_hi >> 7
double exp_hi = EXP_M1[x_hi >> 7];
// lo = x_hi & 0x0000'007fU;
double exp_mid = EXP_M2[x_hi & 0x7f];
double exp_hi_mid = exp_hi * exp_mid;
// Degree-7 minimax polynomial generated by Sollya with the following
// commands:
// > display = hexadecimal;
// > Q = fpminimax(expm1(x)/x, 6, [|D...|], [-2^-8, 2^-8]);
// > Q;
double exp_lo = fputil::polyeval(
xd, 0x1p0, 0x1p0, 0x1p-1, 0x1.5555555555555p-3, 0x1.55555555553ap-5,
0x1.1111111204dfcp-7, 0x1.6c16cb2da593ap-10, 0x1.9ff1648996d2ep-13);
return static_cast<float>(fputil::fma(exp_hi_mid, exp_lo, -1.0));
}

} // namespace __llvm_libc
7 changes: 6 additions & 1 deletion libc/test/src/math/exhaustive/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,20 @@ add_fp_unittest(

add_fp_unittest(
expm1f_test
NO_RUN_POSTBUILD
NEED_MPFR
SUITE
libc_math_exhaustive_tests
SRCS
expm1f_test.cpp
expm1f_test.cpp
DEPENDS
.exhaustive_test
libc.include.math
libc.src.math.expf
libc.src.math.expm1f
libc.src.__support.FPUtil.fputil
LINK_OPTIONS
-lpthread
)

add_fp_unittest(
Expand Down
6 changes: 4 additions & 2 deletions libc/test/src/math/exhaustive/exhaustive_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ void LlvmLibcExhaustiveTest<T>::test_full_range(T start, T stop, int nthreads,
std::cout << msg.str();
msg.str("");

check(begin, end, rounding);
bool result;
check(begin, end, rounding, result);

msg << "** Finished testing from " << std::dec << begin << " to " << end
<< " [0x" << std::hex << begin << ", 0x" << end << ")" << std::endl;
<< " [0x" << std::hex << begin << ", 0x" << end
<< ") : " << (result ? "PASSED" : "FAILED") << std::endl;
std::cout << msg.str();
});
begin += increment;
Expand Down
3 changes: 2 additions & 1 deletion libc/test/src/math/exhaustive/exhaustive_test.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@ struct LlvmLibcExhaustiveTest : public __llvm_libc::testing::Test {
void test_full_range(T start, T stop, int nthreads,
mpfr::RoundingMode rounding);

virtual void check(T start, T stop, mpfr::RoundingMode rounding) = 0;
virtual void check(T start, T stop, mpfr::RoundingMode rounding,
bool &result) = 0;
};
76 changes: 65 additions & 11 deletions libc/test/src/math/exhaustive/expm1f_test.cpp
Original file line number Diff line number Diff line change
@@ -1,27 +1,81 @@
//===-- Exhaustive test for expm1f-----------------------------------------===//
//===-- Exhaustive test for expm1f ----------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "exhaustive_test.h"
#include "src/__support/FPUtil/FPBits.h"
#include "src/math/expm1f.h"
#include "utils/MPFRWrapper/MPFRUtils.h"
#include <math.h>
#include "utils/UnitTest/FPMatcher.h"

using FPBits = __llvm_libc::fputil::FPBits<float>;

namespace mpfr = __llvm_libc::testing::mpfr;

TEST(LlvmLibcExpm1fExhaustiveTest, AllValues) {
uint32_t bits = 0;
do {
FPBits x(bits);
if (!x.is_inf_or_nan() && float(x) < 88.70f) {
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, float(x),
__llvm_libc::expm1f(float(x)), 1.5);
}
} while (bits++ < 0xffff'ffffU);
struct LlvmLibcExpfExhaustiveTest : public LlvmLibcExhaustiveTest<uint32_t> {
void check(uint32_t start, uint32_t stop, mpfr::RoundingMode rounding,
bool &result) override {
mpfr::ForceRoundingMode r(rounding);
uint32_t bits = start;
result = false;
do {
FPBits xbits(bits);
float x = float(xbits);
EXPECT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 0.5,
rounding);
} while (bits++ < stop);
result = true;
}
};

static constexpr int NUM_THREADS = 16;

// Range: [0, 89];
static constexpr uint32_t POS_START = 0x0000'0000U;
static constexpr uint32_t POS_STOP = 0x42b2'0000U;

TEST_F(LlvmLibcExpfExhaustiveTest, PostiveRangeRoundNearestTieToEven) {
test_full_range(POS_START, POS_STOP, NUM_THREADS,
mpfr::RoundingMode::Nearest);
}

TEST_F(LlvmLibcExpfExhaustiveTest, PostiveRangeRoundUp) {
test_full_range(POS_START, POS_STOP, NUM_THREADS, mpfr::RoundingMode::Upward);
}

TEST_F(LlvmLibcExpfExhaustiveTest, PostiveRangeRoundDown) {
test_full_range(POS_START, POS_STOP, NUM_THREADS,
mpfr::RoundingMode::Downward);
}

TEST_F(LlvmLibcExpfExhaustiveTest, PostiveRangeRoundTowardZero) {
test_full_range(POS_START, POS_STOP, NUM_THREADS,
mpfr::RoundingMode::TowardZero);
}

// Range: [-104, 0];
static constexpr uint32_t NEG_START = 0x8000'0000U;
static constexpr uint32_t NEG_STOP = 0xc2d0'0000U;

TEST_F(LlvmLibcExpfExhaustiveTest, NegativeRangeRoundNearestTieToEven) {
test_full_range(NEG_START, NEG_STOP, NUM_THREADS,
mpfr::RoundingMode::Nearest);
}

TEST_F(LlvmLibcExpfExhaustiveTest, NegativeRangeRoundUp) {
test_full_range(NEG_START, NEG_STOP, NUM_THREADS, mpfr::RoundingMode::Upward);
}

TEST_F(LlvmLibcExpfExhaustiveTest, NegativeRangeRoundDown) {
test_full_range(NEG_START, NEG_STOP, NUM_THREADS,
mpfr::RoundingMode::Downward);
}

TEST_F(LlvmLibcExpfExhaustiveTest, NegativeRangeRoundTowardZero) {
test_full_range(NEG_START, NEG_STOP, NUM_THREADS,
mpfr::RoundingMode::TowardZero);
}
23 changes: 15 additions & 8 deletions libc/test/src/math/expm1f_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,12 @@ TEST(LlvmLibcExpm1fTest, Overflow) {
TEST(LlvmLibcExpm1fTest, Underflow) {
errno = 0;
EXPECT_FP_EQ(-1.0f, __llvm_libc::expm1f(float(FPBits(0xff7fffffU))));
EXPECT_MATH_ERRNO(ERANGE);

float x = float(FPBits(0xc2cffff8U));
EXPECT_FP_EQ(-1.0f, __llvm_libc::expm1f(x));
EXPECT_MATH_ERRNO(ERANGE);

x = float(FPBits(0xc2d00008U));
EXPECT_FP_EQ(-1.0f, __llvm_libc::expm1f(x));
EXPECT_MATH_ERRNO(ERANGE);
}

// Test with inputs which are the borders of underflow/overflow but still
Expand All @@ -72,19 +69,28 @@ TEST(LlvmLibcExpm1fTest, Borderline) {

errno = 0;
x = float(FPBits(0x42affff8U));
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 1.0);
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
EXPECT_MATH_ERRNO(0);

x = float(FPBits(0x42b00008U));
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 1.0);
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
EXPECT_MATH_ERRNO(0);

x = float(FPBits(0xc2affff8U));
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 1.0);
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
EXPECT_MATH_ERRNO(0);

x = float(FPBits(0xc2b00008U));
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 1.0);
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
EXPECT_MATH_ERRNO(0);

x = float(FPBits(0x3dc252ddU));
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
EXPECT_MATH_ERRNO(0);
}

Expand All @@ -104,6 +110,7 @@ TEST(LlvmLibcExpm1fTest, InFloatRange) {
// wider precision.
if (isnan(result) || isinf(result) || errno != 0)
continue;
ASSERT_MPFR_MATCH(mpfr::Operation::Expm1, x, __llvm_libc::expm1f(x), 2.2);
ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Expm1, x,
__llvm_libc::expm1f(x), 0.5);
}
}