348 changes: 235 additions & 113 deletions libc/src/math/generic/range_reduction_double_common.h
View file

Large diffs are not rendered by default.

254 changes: 52 additions & 202 deletions libc/src/math/generic/range_reduction_double_fma.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,174 +15,62 @@
#include "src/__support/FPUtil/nearest_integer.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h"
#include "src/math/generic/range_reduction_double_common.h"

namespace LIBC_NAMESPACE_DECL {

namespace fma {

using LIBC_NAMESPACE::fputil::DoubleDouble;

LIBC_INLINE constexpr int FAST_PASS_EXPONENT = 32;
LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
using FPBits = typename fputil::FPBits<double>;
FPBits xbits(x);

// Digits of 2^(16*i) / pi, generated by Sollya with:
// For [2..62]:
// > for i from 3 to 63 do {
// pi_inv = 2^(16*(i - 3)) / pi;
// pn = nearestint(pi_inv);
// pi_frac = pi_inv - pn;
// a = round(pi_frac, D, RN);
// b = round(pi_frac - a, D, RN);
// c = round(pi_frac - a - b, D, RN);
// d = round(pi_frac - a - b - c, D, RN);
// print("{", 2^7 * a, ",", 2^7 * b, ",", 2^7 * c, ",", 2^7 * d, "},");
// };
// For [0..1]:
// The leading bit of 2^(16*(i - 3)) / pi is very small, so we add 0.25 so that
// the conditions for the algorithms are still satisfied, and one of those
// conditions guarantees that ulp(0.25 * x_reduced) >= 2, and will safely be
// discarded.
// for i from 0 to 2 do {
// pi_frac = 0.25 + 2^(16*(i - 3)) / pi;
// a = round(pi_frac, D, RN);
// b = round(pi_frac - a, D, RN);
// c = round(pi_frac - a - b, D, RN);
// d = round(pi_frac - a - b - c, D, RN);
// print("{", 2^7 * a, ",", 2^7 * b, ",", 2^7 * c, ",", 2^7 * d, "},");
// };
// For The fast pass using double-double, we only need 3 parts (a, b, c), but
// for the accurate pass using Float128, instead of using another table of
// Float128s, we simply add the fourth path (a, b, c, d), which simplify the
// implementation a bit and saving some memory.
LIBC_INLINE constexpr double ONE_TWENTY_EIGHT_OVER_PI[64][4] = {
{0x1.0000000000014p5, 0x1.7cc1b727220a9p-49, 0x1.3f84eafa3ea6ap-103,
-0x1.11f924eb53362p-157},
{0x1.0000000145f3p5, 0x1.b727220a94fe1p-49, 0x1.d5f47d4d37703p-104,
0x1.b6295993c439p-158},
{0x1.000145f306dcap5, -0x1.bbead603d8a83p-50, 0x1.f534ddc0db629p-106,
0x1.664f10e4107f9p-160},
{0x1.45f306dc9c883p5, -0x1.6b01ec5417056p-49, -0x1.6447e493ad4cep-103,
0x1.e21c820ff28b2p-157},
{-0x1.f246c6efab581p4, 0x1.3abe8fa9a6eep-53, 0x1.b6c52b3278872p-107,
0x1.07f9458eaf7afp-164},
{0x1.391054a7f09d6p4, -0x1.70565911f924fp-53, 0x1.2b3278872084p-107,
-0x1.ae9c5421443aap-162},
{0x1.529fc2757d1f5p2, 0x1.a6ee06db14acdp-53, -0x1.8778df7c035d4p-107,
0x1.d5ef5de2b0db9p-161},
{-0x1.ec54170565912p-1, 0x1.b6c52b3278872p-59, 0x1.07f9458eaf7afp-116,
-0x1.d4f246dc8e2dfp-173},
{-0x1.505c1596447e5p5, 0x1.b14acc9e21c82p-49, 0x1.fe5163abdebbcp-106,
0x1.586dc91b8e909p-160},
{-0x1.596447e493ad5p1, 0x1.93c439041fe51p-54, 0x1.8eaf7aef1586ep-108,
-0x1.b7238b7b645a4p-163},
{0x1.bb81b6c52b328p5, -0x1.de37df00d74e3p-49, 0x1.7bd778ac36e49p-103,
-0x1.1c5bdb22d1ffap-158},
{0x1.b6c52b3278872p5, 0x1.07f9458eaf7afp-52, -0x1.d4f246dc8e2dfp-109,
0x1.374b801924bbbp-164},
{0x1.2b3278872084p5, -0x1.ae9c5421443aap-50, 0x1.b7246e3a424ddp-106,
0x1.700324977504fp-161},
{-0x1.8778df7c035d4p5, 0x1.d5ef5de2b0db9p-49, 0x1.1b8e909374b8p-104,
0x1.924bba8274648p-160},
{-0x1.bef806ba71508p4, -0x1.443a9e48db91cp-50, -0x1.6f6c8b47fe6dbp-104,
-0x1.115f62e6de302p-158},
{-0x1.ae9c5421443aap-2, 0x1.b7246e3a424ddp-58, 0x1.700324977504fp-113,
-0x1.cdbc603c429c7p-167},
{-0x1.38a84288753c9p5, -0x1.b7238b7b645a4p-51, 0x1.924bba8274648p-112,
0x1.cfe1deb1cb12ap-166},
{-0x1.0a21d4f246dc9p3, 0x1.d2126e9700325p-53, -0x1.a22bec5cdbc6p-107,
-0x1.e214e34ed658cp-162},
{-0x1.d4f246dc8e2dfp3, 0x1.374b801924bbbp-52, -0x1.f62e6de301e21p-106,
-0x1.38d3b5963045ep-160},
{-0x1.236e4716f6c8bp4, -0x1.1ff9b6d115f63p-50, 0x1.921cfe1deb1cbp-106,
0x1.29a73ee88235fp-162},
{0x1.b8e909374b802p4, -0x1.b6d115f62e6dep-50, -0x1.80f10a71a76b3p-105,
0x1.cfba208d7d4bbp-160},
{0x1.09374b801924cp4, -0x1.15f62e6de301ep-50, -0x1.0a71a76b2c609p-105,
0x1.1046bea5d7689p-159},
{-0x1.68ffcdb688afbp3, -0x1.736f180f10a72p-53, 0x1.62534e7dd1047p-107,
-0x1.0568a25dbd8b3p-161},
{0x1.924bba8274648p0, 0x1.cfe1deb1cb12ap-54, -0x1.63045df7282b4p-108,
-0x1.44bb7b16638fep-162},
{-0x1.a22bec5cdbc6p5, -0x1.e214e34ed658cp-50, -0x1.177dca0ad144cp-106,
0x1.213a671c09ad1p-160},
{0x1.3a32439fc3bd6p1, 0x1.cb129a73ee882p-54, 0x1.afa975da24275p-109,
-0x1.8e3f652e8207p-164},
{-0x1.b78c0788538d4p4, 0x1.29a73ee88235fp-50, 0x1.4baed1213a672p-104,
-0x1.fb29741037d8dp-159},
{0x1.fc3bd63962535p5, -0x1.822efb9415a29p-51, 0x1.a24274ce38136p-105,
-0x1.741037d8cdc54p-159},
{-0x1.4e34ed658c117p2, -0x1.f7282b4512edfp-52, 0x1.d338e04d68bfp-107,
-0x1.bec66e29c67cbp-162},
{0x1.62534e7dd1047p5, -0x1.0568a25dbd8b3p-49, -0x1.c7eca5d040df6p-105,
-0x1.9b8a719f2b318p-160},
{-0x1.63045df7282b4p4, -0x1.44bb7b16638fep-50, 0x1.ad17df904e647p-104,
0x1.639835339f49dp-158},
{0x1.d1046bea5d769p5, -0x1.bd8b31c7eca5dp-49, -0x1.037d8cdc538dp-107,
0x1.a99cfa4e422fcp-161},
{0x1.afa975da24275p3, -0x1.8e3f652e8207p-52, 0x1.3991d63983534p-106,
-0x1.82d8dee81d108p-160},
{-0x1.a28976f62cc72p5, 0x1.35a2fbf209cc9p-53, -0x1.4e33e566305b2p-109,
0x1.08bf177bf2507p-163},
{-0x1.76f62cc71fb29p5, -0x1.d040df633714ep-49, -0x1.9f2b3182d8defp-104,
0x1.f8bbdf9283b2p-158},
{0x1.d338e04d68bfp5, -0x1.bec66e29c67cbp-50, 0x1.9cfa4e422fc5ep-105,
-0x1.036be27003b4p-161},
{0x1.c09ad17df904ep4, 0x1.91d639835339fp-50, 0x1.272117e2ef7e5p-104,
-0x1.7c4e007680022p-158},
{0x1.68befc827323bp5, -0x1.c67cacc60b638p-50, 0x1.17e2ef7e4a0ecp-104,
0x1.ff897ffde0598p-158},
{-0x1.037d8cdc538dp5, 0x1.a99cfa4e422fcp-49, 0x1.77bf250763ff1p-103,
0x1.7ffde05980fefp-158},
{-0x1.8cdc538cf9599p5, 0x1.f49c845f8bbep-50, -0x1.b5f13801da001p-104,
0x1.e05980fef2f12p-158},
{-0x1.4e33e566305b2p3, 0x1.08bf177bf2507p-51, 0x1.8ffc4bffef02dp-105,
-0x1.fc04343b9d298p-160},
{-0x1.f2b3182d8dee8p4, -0x1.d1081b5f13802p-52, 0x1.2fffbc0b301fep-107,
-0x1.a1dce94beb25cp-163},
{-0x1.8c16c6f740e88p5, -0x1.036be27003b4p-49, -0x1.0fd33f8086877p-109,
-0x1.d297d64b824b2p-164},
{0x1.3908bf177bf25p5, 0x1.d8ffc4bffef03p-53, -0x1.9fc04343b9d29p-108,
-0x1.f592e092c9813p-162},
{0x1.7e2ef7e4a0ec8p4, -0x1.da00087e99fcp-56, -0x1.0d0ee74a5f593p-110,
0x1.f6d367ecf27cbp-166},
{-0x1.081b5f13801dap4, -0x1.0fd33f8086877p-61, -0x1.d297d64b824b2p-116,
-0x1.8130d834f648bp-170},
{-0x1.af89c00ed0004p5, -0x1.fa67f010d0ee7p-50, -0x1.297d64b824b26p-104,
-0x1.30d834f648b0cp-162},
{-0x1.c00ed00043f4dp5, 0x1.fde5e2316b415p-55, -0x1.2e092c98130d8p-110,
-0x1.a7b24585ce04dp-165},
{0x1.2fffbc0b301fep5, -0x1.a1dce94beb25cp-51, -0x1.25930261b069fp-107,
0x1.b74f463f669e6p-162},
{-0x1.0fd33f8086877p3, -0x1.d297d64b824b2p-52, -0x1.8130d834f648bp-106,
-0x1.738132c3402bap-163},
{-0x1.9fc04343b9d29p4, -0x1.f592e092c9813p-50, -0x1.b069ec9161738p-107,
-0x1.32c3402ba515bp-163},
{-0x1.0d0ee74a5f593p2, 0x1.f6d367ecf27cbp-54, 0x1.36e9e8c7ecd3dp-111,
-0x1.00ae9456c229cp-165},
{-0x1.dce94beb25c12p5, -0x1.64c0986c1a7b2p-49, -0x1.161738132c34p-103,
-0x1.5d28ad8453814p-158},
{-0x1.4beb25c12593p5, -0x1.30d834f648b0cp-50, 0x1.8fd9a797fa8b6p-104,
-0x1.5b08a7028341dp-159},
{0x1.b47db4d9fb3cap4, -0x1.a7b24585ce04dp-53, 0x1.3cbfd45aea4f7p-107,
0x1.63f5f2f8bd9e8p-161},
{-0x1.25930261b069fp5, 0x1.b74f463f669e6p-50, -0x1.5d28ad8453814p-110,
-0x1.a0e84c2f8c608p-166},
{0x1.fb3c9f2c26dd4p4, -0x1.738132c3402bap-51, -0x1.456c229c0a0dp-105,
-0x1.d0985f18c10ebp-159},
{-0x1.b069ec9161738p5, -0x1.32c3402ba515bp-51, -0x1.14e050683a131p-108,
0x1.0739f78a5292fp-162},
{-0x1.ec9161738132cp5, -0x1.a015d28ad8454p-50, 0x1.faf97c5ecf41dp-104,
-0x1.821d6b5b4565p-160},
{-0x1.61738132c3403p5, 0x1.16ba93dd63f5fp-49, 0x1.7c5ecf41ce7dep-104,
0x1.4a525d4d7f6bfp-159},
{0x1.fb34f2ff516bbp3, -0x1.b08a7028341d1p-51, 0x1.9e839cfbc5295p-105,
-0x1.a2b2809409dc1p-159},
{0x1.3cbfd45aea4f7p5, 0x1.63f5f2f8bd9e8p-49, 0x1.ce7de294a4baap-104,
-0x1.404a04ee072a3p-158},
{-0x1.5d28ad8453814p2, -0x1.a0e84c2f8c608p-54, -0x1.d6b5b45650128p-108,
-0x1.3b81ca8bdea7fp-164},
{-0x1.15b08a7028342p5, 0x1.7b3d0739f78a5p-50, 0x1.497535fdafd89p-105,
-0x1.ca8bdea7f33eep-164},
};
int x_e_m62 = xbits.get_biased_exponent() - (FPBits::EXP_BIAS + 62);
idx = static_cast<unsigned>((x_e_m62 >> 4) + 3);
// Scale x down by 2^(-(16 * (idx - 3))
xbits.set_biased_exponent((x_e_m62 & 15) + FPBits::EXP_BIAS + 62);
// 2^62 <= |x_reduced| < 2^(62 + 16) = 2^78
x_reduced = xbits.get_val();
// x * c_hi = ph.hi + ph.lo exactly.
DoubleDouble ph =
fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
// x * c_mid = pm.hi + pm.lo exactly.
DoubleDouble pm =
fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
// x * c_lo = pl.hi + pl.lo exactly.
DoubleDouble pl =
fputil::exact_mult<SPLIT>(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
// Extract integral parts and fractional parts of (ph.lo + pm.hi).
double sum_hi = ph.lo + pm.hi;
double kd = fputil::nearest_integer(sum_hi);

// x * 128/pi mod 1 ~ y_hi + y_mid + y_lo
y_hi = (ph.lo - kd) + pm.hi; // Exact
y_mid = fputil::exact_add(pm.lo, pl.hi);
y_lo = pl.lo;

// y_l = x * c_lo_2 + pl.lo
double y_l =
fputil::multiply_add(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][3], y_lo);
DoubleDouble y = fputil::exact_add(y_hi, y_mid.hi);
y.lo += (y_mid.lo + y_l);

// Digits of pi/128, generated by Sollya with:
// > a = round(pi/128, D, RN);
// > b = round(pi/128 - a, D, RN);
constexpr DoubleDouble PI_OVER_128_DD = {0x1.1a62633145c07p-60,
0x1.921fb54442d18p-6};

// Error bound: with {a} denote the fractional part of a, i.e.:
// {a} = a - round(a)
// Then,
// | {x * 128/pi} - (y_hi + y_lo) | <= ulp(ulp(y_hi)) <= 2^-105
// | {x mod pi/128} - (u.hi + u.lo) | < 2 * 2^-6 * 2^-105 = 2^-110
u = fputil::quick_mult<SPLIT>(y, PI_OVER_128_DD);

return static_cast<unsigned>(static_cast<int64_t>(kd));
}

// Lookup table for sin(k * pi / 128) with k = 0, ..., 255.
// Table is generated with Sollya as follow:
Expand Down Expand Up @@ -258,6 +146,7 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = {
{-0x1.c57bc2e24aa15p-57, 0x1.ff621e3796d7ep-1},
{-0x1.1354d4556e4cbp-55, 0x1.ffd886084cd0dp-1},
{0, 1},
#ifndef LIBC_MATH_HAS_SMALL_TABLES
{-0x1.1354d4556e4cbp-55, 0x1.ffd886084cd0dp-1},
{-0x1.c57bc2e24aa15p-57, 0x1.ff621e3796d7ep-1},
{0x1.521ecd0c67e35p-57, 0x1.fe9cdad01883ap-1},
Expand Down Expand Up @@ -449,48 +338,9 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = {
{0x1.9a088a8bf6b2cp-59, -0x1.2d52092ce19f6p-4},
{0x1.912bd0d569a9p-61, -0x1.91f65f10dd814p-5},
{0x1.b1d63091a013p-64, -0x1.92155f7a3667ep-6},
#endif // !LIBC_MATH_HAS_SMALL_TABLES
};

// For |x| < 2^-32, return k and u such that:
// k = round(x * 128/pi)
// x mod pi/128 = x - k * pi/128 ~ u.hi + u.lo
LIBC_INLINE unsigned range_reduction_small(double x, DoubleDouble &u) {
// Digits of pi/128, generated by Sollya with:
// > a = round(pi/128, D, RN);
// > b = round(pi/128 - a, D, RN);
constexpr DoubleDouble PI_OVER_128_DD = {0x1.1a62633145c07p-60,
0x1.921fb54442d18p-6};

double prod_hi = x * ONE_TWENTY_EIGHT_OVER_PI[3][0];
double kd = fputil::nearest_integer(prod_hi);

// Let y = x - k * (pi/128)
// Then |y| < pi / 256
// With extra rounding errors, we can bound |y| < 2^-6.
double y_hi = fputil::multiply_add(kd, -PI_OVER_128_DD.hi, x); // Exact
// u_hi + u_lo ~ (y_hi + kd*(-PI_OVER_128_DD[1]))
// and |u_lo| < 2* ulp(u_hi)
// The upper bound 2^-6 is over-estimated, we should still have:
// |u_hi + u_lo| < 2^-6.
u.hi = fputil::multiply_add(kd, -PI_OVER_128_DD.lo, y_hi);
u.lo = y_hi - u.hi; // Exact;
u.lo = fputil::multiply_add(kd, -PI_OVER_128_DD.lo, u.lo);
// Error bound:
// For |x| < 2^32:
// |x * high part of 128/pi| < 2^32 * 2^6 = 2^38
// So |k| = |round(x * high part of 128/pi)| < 2^38
// And hence,
// |(x mod pi/128) - (u.hi + u.lo)| <= ulp(2 * kd * PI_OVER_128_DD.lo)
// < 2 * 2^38 * 2^-59 * 2^-52
// = 2^-72
// Note: if we limit the input exponent to the same as in non-FMA version,
// i.e., |x| < 2^-23, then the output errors can be bounded by 2^-81, similar
// to the large range reduction bound.
return static_cast<unsigned>(static_cast<int64_t>(kd));
}

} // namespace fma

} // namespace LIBC_NAMESPACE_DECL

#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_FMA_H
253 changes: 53 additions & 200 deletions libc/src/math/generic/range_reduction_double_nofma.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,174 +15,63 @@
#include "src/__support/FPUtil/nearest_integer.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h"
#include "src/math/generic/range_reduction_double_common.h"

namespace LIBC_NAMESPACE_DECL {

namespace nofma {

using fputil::DoubleDouble;

LIBC_INLINE constexpr int FAST_PASS_EXPONENT = 23;
LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
using FPBits = typename fputil::FPBits<double>;
FPBits xbits(x);

// Digits of 2^(16*i) / pi, generated by Sollya with:
// For [2..62]:
// > for i from 3 to 63 do {
// pi_inv = 2^(16*(i - 3)) / pi;
// pn = nearestint(pi_inv);
// pi_frac = pi_inv - pn;
// a = round(pi_frac, 51, RN);
// b = round(pi_frac - a, 51, RN);
// c = round(pi_frac - a - b, D, RN);
// d = round(pi_frac - a - b - c, D, RN);
// print("{", 2^7 * a, ",", 2^7 * b, ",", 2^7 * c, ",", 2^7 * d, "},");
// };
// For [0..1]:
// The leading bit of 2^(16*(i - 3)) / pi is very small, so we add 0.25 so that
// the conditions for the algorithms are still satisfied, and one of those
// conditions guarantees that ulp(0.25 * x_reduced) >= 2, and will safely be
// discarded.
// for i from 0 to 2 do {
// pi_frac = 0.25 + 2^(16*(i - 3)) / pi;
// a = round(pi_frac, 51, RN);
// b = round(pi_frac - a, 51, RN);
// c = round(pi_frac - a - b, D, RN);
// d = round(pi_frac - a - b - c, D, RN);
// print("{", 2^7 * a, ",", 2^7 * b, ",", 2^7 * c, ",", 2^7 * d, "},");
// };
// For The fast pass using double-double, we only need 3 parts (a, b, c), but
// for the accurate pass using Float128, instead of using another table of
// Float128s, we simply add the fourth path (a, b, c, d), which simplify the
// implementation a bit and saving some memory.
LIBC_INLINE constexpr double ONE_TWENTY_EIGHT_OVER_PI[64][4] = {
{0x1.0000000000014p5, 0x1.7cc1b727220a8p-49, 0x1.4fe13abe8fa9ap-101,
0x1.bb81b6c52b328p-155},
{0x1.0000000145f3p5, 0x1.b727220a94fep-49, 0x1.3abe8fa9a6eep-101,
0x1.b6c52b3278872p-155},
{0x1.000145f306dc8p5, 0x1.c882a53f84ebp-47, -0x1.70565911f924fp-101,
0x1.2b3278872084p-155},
{0x1.45f306dc9c884p5, -0x1.5ac07b1505c14p-47, -0x1.96447e493ad4dp-99,
0x1.3c439041fe516p-154},
{-0x1.f246c6efab58p4, -0x1.ec5417056591p-49, -0x1.f924eb53361dep-101,
-0x1.bef806ba71508p-156},
{0x1.391054a7f09d4p4, 0x1.f47d4d377036cp-48, 0x1.8a5664f10e41p-100,
0x1.fe5163abdebbcp-154},
{0x1.529fc2757d1f4p2, 0x1.34ddc0db62958p-50, 0x1.93c439041fe51p-102,
0x1.8eaf7aef1586ep-156},
{-0x1.ec5417056591p-1, -0x1.f924eb53361ep-53, 0x1.c820ff28b1d5fp-105,
-0x1.443a9e48db91cp-162},
{-0x1.505c1596447e4p5, -0x1.275a99b0ef1cp-48, 0x1.07f9458eaf7afp-100,
-0x1.d4f246dc8e2dfp-157},
{-0x1.596447e493ad4p1, -0x1.9b0ef1bef806cp-52, 0x1.63abdebbc561bp-106,
0x1.c91b8e909374cp-160},
{0x1.bb81b6c52b328p5, -0x1.de37df00d74e4p-49, 0x1.5ef5de2b0db92p-101,
0x1.b8e909374b802p-156},
{0x1.b6c52b3278874p5, -0x1.f7c035d38a844p-47, 0x1.778ac36e48dc7p-99,
0x1.2126e97003249p-153},
{0x1.2b3278872084p5, -0x1.ae9c5421443a8p-50, -0x1.e48db91c5bdb2p-102,
-0x1.68ffcdb688afbp-157},
{-0x1.8778df7c035d4p5, 0x1.d5ef5de2b0db8p-49, 0x1.2371d2126e97p-101,
0x1.924bba8274648p-160},
{-0x1.bef806ba71508p4, -0x1.443a9e48db91cp-50, -0x1.6f6c8b47fe6dbp-104,
-0x1.115f62e6de302p-158},
{-0x1.ae9c5421443a8p-2, -0x1.e48db91c5bdb4p-54, 0x1.d2e006492eea1p-106,
-0x1.8b9b78c078854p-160},
{-0x1.38a84288753c8p5, -0x1.1b7238b7b645cp-47, 0x1.c00c925dd413ap-99,
0x1.921cfe1deb1cbp-154},
{-0x1.0a21d4f246dc8p3, -0x1.c5bdb22d1ff9cp-50, 0x1.25dd413a3243ap-103,
-0x1.e214e34ed658cp-162},
{-0x1.d4f246dc8e2ep3, 0x1.26e9700324978p-49, -0x1.5f62e6de301e2p-102,
-0x1.4e34ed658c117p-158},
{-0x1.236e4716f6c8cp4, 0x1.700324977505p-49, -0x1.736f180f10a72p-101,
0x1.62534e7dd1047p-155},
{0x1.b8e909374b8p4, 0x1.924bba8274648p-48, 0x1.cfe1deb1cb12ap-102,
-0x1.63045df7282b4p-156},
{0x1.09374b801924cp4, -0x1.15f62e6de302p-50, 0x1.deb1cb129a73fp-102,
-0x1.77dca0ad144bbp-158},
{-0x1.68ffcdb688afcp3, 0x1.d1921cfe1debp-50, 0x1.cb129a73ee882p-102,
0x1.afa975da24275p-157},
{0x1.924bba8274648p0, 0x1.cfe1deb1cb128p-54, 0x1.a73ee88235f53p-106,
-0x1.44bb7b16638fep-162},
{-0x1.a22bec5cdbc6p5, -0x1.e214e34ed658cp-50, -0x1.177dca0ad144cp-106,
0x1.213a671c09ad1p-160},
{0x1.3a32439fc3bd8p1, -0x1.c69dacb1822fp-51, 0x1.1afa975da2427p-105,
0x1.338e04d68befdp-159},
{-0x1.b78c0788538d4p4, 0x1.29a73ee88236p-50, -0x1.5a28976f62cc7p-103,
-0x1.fb29741037d8dp-159},
{0x1.fc3bd63962534p5, 0x1.cfba208d7d4bcp-48, -0x1.12edec598e3f6p-100,
-0x1.4ba081bec66e3p-154},
{-0x1.4e34ed658c118p2, 0x1.046bea5d7689p-51, 0x1.3a671c09ad17ep-104,
-0x1.bec66e29c67cbp-162},
{0x1.62534e7dd1048p5, -0x1.415a28976f62cp-47, -0x1.8e3f652e8207p-100,
0x1.3991d63983534p-154},
{-0x1.63045df7282b4p4, -0x1.44bb7b16638fcp-50, -0x1.94ba081bec66ep-102,
-0x1.4e33e566305b2p-157},
{0x1.d1046bea5d768p5, 0x1.213a671c09adp-48, 0x1.7df904e64758ep-100,
0x1.835339f49c846p-154},
{0x1.afa975da24274p3, 0x1.9c7026b45f7e4p-50, 0x1.3991d63983534p-106,
-0x1.82d8dee81d108p-160},
{-0x1.a28976f62cc7p5, -0x1.fb29741037d8cp-47, -0x1.b8a719f2b3183p-100,
0x1.3908bf177bf25p-155},
{-0x1.76f62cc71fb28p5, -0x1.741037d8cdc54p-47, 0x1.cc1a99cfa4e42p-101,
0x1.7e2ef7e4a0ec8p-156},
{0x1.d338e04d68bfp5, -0x1.bec66e29c67ccp-50, 0x1.339f49c845f8cp-102,
-0x1.081b5f13801dap-156},
{0x1.c09ad17df905p4, -0x1.9b8a719f2b318p-48, -0x1.6c6f740e8840ep-103,
0x1.41d8ffc4bffefp-157},
{0x1.68befc827323cp5, -0x1.38cf9598c16c8p-47, 0x1.08bf177bf2507p-99,
0x1.8ffc4bffef02dp-153},
{-0x1.037d8cdc538dp5, 0x1.a99cfa4e422fcp-49, 0x1.77bf250763ff1p-103,
0x1.7ffde05980fefp-158},
{-0x1.8cdc538cf9598p5, -0x1.82d8dee81d108p-48, -0x1.b5f13801da001p-104,
0x1.e05980fef2f12p-158},
{-0x1.4e33e566305bp3, -0x1.bdd03a21036cp-49, 0x1.d8ffc4bffef03p-101,
-0x1.9fc04343b9d29p-156},
{-0x1.f2b3182d8dee8p4, -0x1.d1081b5f138p-52, -0x1.da00087e99fcp-104,
-0x1.0d0ee74a5f593p-158},
{-0x1.8c16c6f740e88p5, -0x1.036be27003b4p-49, -0x1.0fd33f8086877p-109,
-0x1.d297d64b824b2p-164},
{0x1.3908bf177bf24p5, 0x1.0763ff12fffbcp-47, 0x1.6603fbcbc462dp-104,
0x1.a0a6d1f6d367fp-158},
{0x1.7e2ef7e4a0ec8p4, -0x1.da00087e99fcp-56, -0x1.0d0ee74a5f593p-110,
0x1.f6d367ecf27cbp-166},
{-0x1.081b5f13801dcp4, 0x1.fff7816603fbcp-48, 0x1.788c5ad05369p-101,
-0x1.25930261b069fp-155},
{-0x1.af89c00ed0004p5, -0x1.fa67f010d0ee8p-50, 0x1.6b414da3eda6dp-103,
-0x1.30d834f648b0cp-162},
{-0x1.c00ed00043f4cp5, -0x1.fc04343b9d298p-48, 0x1.4da3eda6cfd9ep-103,
0x1.3e584dba7a32p-157},
{0x1.2fffbc0b301fcp5, 0x1.e5e2316b414dcp-47, -0x1.c125930261b07p-99,
0x1.84dba7a31fb35p-153},
{-0x1.0fd33f8086878p3, 0x1.8b5a0a6d1f6d4p-50, -0x1.30261b069ec91p-103,
-0x1.85ce04cb0d00bp-157},
{-0x1.9fc04343b9d28p4, -0x1.7d64b824b2604p-48, -0x1.86c1a7b24585dp-101,
0x1.fb34f2ff516bbp-157},
{-0x1.0d0ee74a5f594p2, 0x1.1f6d367ecf27cp-50, 0x1.6136e9e8c7ecdp-103,
0x1.e5fea2d7527bbp-158},
{-0x1.dce94beb25c14p5, 0x1.a6cfd9e4f9614p-47, -0x1.22c2e70265868p-100,
-0x1.5d28ad8453814p-158},
{-0x1.4beb25c12593p5, -0x1.30d834f648b0cp-50, 0x1.8fd9a797fa8b6p-104,
-0x1.5b08a7028341dp-159},
{0x1.b47db4d9fb3c8p4, 0x1.f2c26dd3d18fcp-48, 0x1.9a797fa8b5d4ap-100,
-0x1.14e050683a131p-156},
{-0x1.25930261b06ap5, 0x1.36e9e8c7ecd3cp-47, 0x1.7fa8b5d49eeb2p-100,
-0x1.41a0e84c2f8c6p-158},
{0x1.fb3c9f2c26dd4p4, -0x1.738132c3402bcp-51, 0x1.aea4f758fd7ccp-103,
-0x1.d0985f18c10ebp-159},
{-0x1.b069ec9161738p5, -0x1.32c3402ba515cp-51, 0x1.eeb1faf97c5edp-104,
-0x1.7c63043ad6b69p-161},
{-0x1.ec9161738132cp5, -0x1.a015d28ad8454p-50, 0x1.faf97c5ecf41dp-104,
-0x1.821d6b5b4565p-160},
{-0x1.61738132c3404p5, 0x1.45aea4f758fd8p-47, -0x1.a0e84c2f8c608p-102,
-0x1.d6b5b45650128p-156},
{0x1.fb34f2ff516bcp3, -0x1.6c229c0a0d074p-49, -0x1.30be31821d6b6p-104,
0x1.2ea6bfb5fb12p-158},
{0x1.3cbfd45aea4f8p5, -0x1.4e050683a130cp-48, 0x1.ce7de294a4baap-104,
-0x1.404a04ee072a3p-158},
{-0x1.5d28ad8453814p2, -0x1.a0e84c2f8c608p-54, -0x1.d6b5b45650128p-108,
-0x1.3b81ca8bdea7fp-164},
{-0x1.15b08a702834p5, -0x1.d0985f18c10ecp-47, 0x1.4a4ba9afed7ecp-100,
0x1.1f8d5d0856033p-154},
};
int x_e_m62 = xbits.get_biased_exponent() - (FPBits::EXP_BIAS + 62);
idx = static_cast<unsigned>((x_e_m62 >> 4) + 3);
// Scale x down by 2^(-(16 * (idx - 3))
xbits.set_biased_exponent((x_e_m62 & 15) + FPBits::EXP_BIAS + 62);
// 2^62 <= |x_reduced| < 2^(62 + 16) = 2^78
x_reduced = xbits.get_val();
// x * c_hi = ph.hi + ph.lo exactly.
DoubleDouble x_split = fputil::split(x_reduced);
DoubleDouble ph = fputil::exact_mult<SPLIT>(x_split, x_reduced,
ONE_TWENTY_EIGHT_OVER_PI[idx][0]);
// x * c_mid = pm.hi + pm.lo exactly.
DoubleDouble pm = fputil::exact_mult<SPLIT>(x_split, x_reduced,
ONE_TWENTY_EIGHT_OVER_PI[idx][1]);
// x * c_lo = pl.hi + pl.lo exactly.
DoubleDouble pl = fputil::exact_mult<SPLIT>(x_split, x_reduced,
ONE_TWENTY_EIGHT_OVER_PI[idx][2]);
// Extract integral parts and fractional parts of (ph.lo + pm.hi).
double sum_hi = ph.lo + pm.hi;
double kd = fputil::nearest_integer(sum_hi);

// x * 128/pi mod 1 ~ y_hi + y_mid + y_lo
y_hi = (ph.lo - kd) + pm.hi; // Exact
y_mid = fputil::exact_add(pm.lo, pl.hi);
y_lo = pl.lo;

// y_l = x * c_lo_2 + pl.lo
double y_l =
fputil::multiply_add(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][3], y_lo);
DoubleDouble y = fputil::exact_add(y_hi, y_mid.hi);
y.lo += (y_mid.lo + y_l);

// Digits of pi/128, generated by Sollya with:
// > a = round(pi/128, D, RN);
// > b = round(pi/128 - a, D, RN);
constexpr DoubleDouble PI_OVER_128_DD = {0x1.1a62633145c07p-60,
0x1.921fb54442d18p-6};

// Error bound: with {a} denote the fractional part of a, i.e.:
// {a} = a - round(a)
// Then,
// | {x * 128/pi} - (y_hi + y_lo) | <= ulp(ulp(y_hi)) <= 2^-105
// | {x mod pi/128} - (u.hi + u.lo) | < 2 * 2^-6 * 2^-105 = 2^-110
u = fputil::quick_mult<SPLIT>(y, PI_OVER_128_DD);

return static_cast<unsigned>(static_cast<int64_t>(kd));
}

// Lookup table for sin(k * pi / 128) with k = 0, ..., 255.
// Table is generated with Sollya as follow:
Expand Down Expand Up @@ -258,6 +147,7 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = {
{0x1.e3a843d1db55fp-53, 0x1.ff621e3796d7cp-1},
{0x1.765595d548d9ap-54, 0x1.ffd886084cd0cp-1},
{0, 1},
#ifndef LIBC_MATH_HAS_SMALL_TABLES
{0x1.765595d548d9ap-54, 0x1.ffd886084cd0cp-1},
{0x1.e3a843d1db55fp-53, 0x1.ff621e3796d7cp-1},
{-0x1.eade132f3981dp-53, 0x1.fe9cdad01883cp-1},
Expand Down Expand Up @@ -449,46 +339,9 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = {
{-0x1.ccbeeeae8129ap-56, -0x1.2d52092ce19f4p-4},
{0x1.912bd0d569a9p-61, -0x1.91f65f10dd814p-5},
{-0x1.f938a73db97fbp-58, -0x1.92155f7a3667cp-6},
#endif // !LIBC_MATH_HAS_SMALL_TABLES
};

LIBC_INLINE unsigned range_reduction_small(double x, DoubleDouble &u) {
constexpr double ONE_TWENTY_EIGHT_OVER_PI = 0x1.45f306dc9c883p5;

// Digits of -pi/128, generated by Sollya with:
// > a = round(-pi/128, 25, RN);
// > b = round(-pi/128 - a, 23, RN);
// > c = round(-pi/128 - a - b, 25, RN);
// > d = round(-pi/128 - a - b - c, D, RN);
// -pi/128 ~ a + b + c + d
// The precisions of the parts are chosen so that:
// 1) k * a, k * b, k * c are exact in double precision
// 2) k * b + (x - (k * a)) is exact in double precsion
constexpr double MPI_OVER_128[4] = {-0x1.921fb5p-6, -0x1.110b48p-32,
+0x1.ee59dap-56, -0x1.98a2e03707345p-83};

double prod_hi = x * ONE_TWENTY_EIGHT_OVER_PI;
double kd = fputil::nearest_integer(prod_hi);

// With -pi/128 ~ a + b + c + d as in MPI_OVER_128 description:
// t = x + k * a
double t = fputil::multiply_add(kd, MPI_OVER_128[0], x); // Exact
// y_hi = t + k * b = (x + k * a) + k * b
double y_hi = fputil::multiply_add(kd, MPI_OVER_128[1], t); // Exact
// y_lo ~ k * c + k * d
double y_lo = fputil::multiply_add(kd, MPI_OVER_128[2], kd * MPI_OVER_128[3]);
// u.hi + u.lo ~ x + k * (a + b + c + d)
u = fputil::exact_add(y_hi, y_lo);
// Error bound: For |x| < 2^-23,
// |(x mod pi/128) - (u_hi + u_lo)| < ulp(y_lo)
// <= ulp(2 * x * c)
// <= ulp(2^24 * 2^-56)
// = 2^(24 - 56 - 52)
// = 2^-84
return static_cast<unsigned>(static_cast<int>(kd));
}

} // namespace nofma

} // namespace LIBC_NAMESPACE_DECL

#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_NOFMA_H
129 changes: 57 additions & 72 deletions libc/src/math/generic/sin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,14 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
#include "src/math/generic/range_reduction_double_common.h"
#include "src/math/generic/sincos_eval.h"

// TODO: We might be able to improve the performance of large range reduction of
// non-FMA targets further by operating directly on 25-bit chunks of 128/pi and
// pre-split SIN_K_PI_OVER_128, but that might double the memory footprint of
// those lookup table.
#include "range_reduction_double_common.h"

#if ((LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS) != 0)
#define LIBC_MATH_SIN_SKIP_ACCURATE_PASS
#endif
#ifdef LIBC_TARGET_CPU_HAS_FMA
#include "range_reduction_double_fma.h"
#else
#include "range_reduction_double_nofma.h"
#endif // LIBC_TARGET_CPU_HAS_FMA

namespace LIBC_NAMESPACE_DECL {

Expand All @@ -43,33 +40,39 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {

DoubleDouble y;
unsigned k;
generic::LargeRangeReduction<NO_FMA> range_reduction_large{};
LargeRangeReduction range_reduction_large{};

// |x| < 2^32 (with FMA) or |x| < 2^23 (w/o FMA)
// |x| < 2^16
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
// |x| < 2^-26
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 26)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0))
return x;
// |x| < 2^-7
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
// |x| < 2^-26, |sin(x) - x| < ulp(x)/2.
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 26)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0))
return x;

// For |x| < 2^-26, |sin(x) - x| < ulp(x)/2.
#ifdef LIBC_TARGET_CPU_HAS_FMA
return fputil::multiply_add(x, -0x1.0p-54, x);
return fputil::multiply_add(x, -0x1.0p-54, x);
#else
if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if (rounding_mode == FE_TOWARDZERO ||
(xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
return FPBits(xbits.uintval() - 1).get_val();
}
return fputil::multiply_add(x, -0x1.0p-54, x);
if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if (rounding_mode == FE_TOWARDZERO ||
(xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
return FPBits(xbits.uintval() - 1).get_val();
}
return fputil::multiply_add(x, -0x1.0p-54, x);
#endif // LIBC_TARGET_CPU_HAS_FMA
}
// No range reduction needed.
k = 0;
y.lo = 0.0;
y.hi = x;
} else {
// Small range reduction.
k = range_reduction_small(x, y);
}

// // Small range reduction.
k = range_reduction_small(x, y);
} else {
// Inf or NaN
if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
Expand All @@ -82,69 +85,51 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
}

// Large range reduction.
k = range_reduction_large.compute_high_part(x);
y = range_reduction_large.fast();
k = range_reduction_large.fast(x, y);
}

DoubleDouble sin_y, cos_y;

generic::sincos_eval(y, sin_y, cos_y);
[[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y);

// Look up sin(k * pi/128) and cos(k * pi/128)
// Memory saving versions:

// Use 128-entry table instead:
// DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 127];
// uint64_t sin_s = static_cast<uint64_t>(k & 128) << (63 - 7);
// sin_k.hi = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// sin_k.lo = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 127];
// uint64_t cos_s = static_cast<uint64_t>((k + 64) & 128) << (63 - 7);
// cos_k.hi = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();
// cos_k.lo = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();

// Use 64-entry table instead:
// auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
// unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
// DoubleDouble ans = SIN_K_PI_OVER_128[idx];
// if (kk & 128) {
// ans.hi = -ans.hi;
// ans.lo = -ans.lo;
// }
// return ans;
// };
// DoubleDouble sin_k = get_idx_dd(k);
// DoubleDouble cos_k = get_idx_dd(k + 64);

#ifdef LIBC_MATH_HAS_SMALL_TABLES
// Memory saving versions. Use 65-entry table.
auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
DoubleDouble ans = SIN_K_PI_OVER_128[idx];
if (kk & 128) {
ans.hi = -ans.hi;
ans.lo = -ans.lo;
}
return ans;
};
DoubleDouble sin_k = get_idx_dd(k);
DoubleDouble cos_k = get_idx_dd(k + 64);
#else
// Fast look up version, but needs 256-entry table.
// cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 255];
DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
#endif

// After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
// So k is an integer and -pi / 256 <= y <= pi / 256.
// Then sin(x) = sin((k * pi/128 + y)
// = sin(y) * cos(k*pi/128) + cos(y) * sin(k*pi/128)
DoubleDouble sin_k_cos_y = fputil::quick_mult<NO_FMA>(cos_y, sin_k);
DoubleDouble cos_k_sin_y = fputil::quick_mult<NO_FMA>(sin_y, cos_k);
DoubleDouble sin_k_cos_y = fputil::quick_mult(cos_y, sin_k);
DoubleDouble cos_k_sin_y = fputil::quick_mult(sin_y, cos_k);

DoubleDouble rr = fputil::exact_add<false>(sin_k_cos_y.hi, cos_k_sin_y.hi);
rr.lo += sin_k_cos_y.lo + cos_k_sin_y.lo;

#ifdef LIBC_MATH_SIN_SKIP_ACCURATE_PASS
#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
return rr.hi + rr.lo;
#else
// Accurate test and pass for correctly rounded implementation.

#ifdef LIBC_TARGET_CPU_HAS_FMA
constexpr double ERR = 0x1.0p-70;
#else
// TODO: Improve non-FMA fast pass accuracy.
constexpr double ERR = 0x1.0p-66;
#endif // LIBC_TARGET_CPU_HAS_FMA

double rlp = rr.lo + ERR;
double rlm = rr.lo - ERR;
double rlp = rr.lo + err;
double rlm = rr.lo - err;

double r_upper = rr.hi + rlp; // (rr.lo + ERR);
double r_lower = rr.hi + rlm; // (rr.lo - ERR);
Expand All @@ -155,15 +140,15 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {

Float128 u_f128, sin_u, cos_u;
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
u_f128 = generic::range_reduction_small_f128(x);
u_f128 = range_reduction_small_f128(x);
else
u_f128 = range_reduction_large.accurate();

generic::sincos_eval(u_f128, sin_u, cos_u);

auto get_sin_k = [](unsigned kk) -> Float128 {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
Float128 ans = generic::SIN_K_PI_OVER_128_F128[idx];
Float128 ans = SIN_K_PI_OVER_128_F128[idx];
if (kk & 128)
ans.sign = Sign::NEG;
return ans;
Expand All @@ -182,7 +167,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
// https://github.com/llvm/llvm-project/issues/96452.

return static_cast<double>(r);
#endif // !LIBC_MATH_SIN_SKIP_ACCURATE_PASS
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
}

} // namespace LIBC_NAMESPACE_DECL
155 changes: 71 additions & 84 deletions libc/src/math/generic/sincos.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,14 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
#include "src/math/generic/range_reduction_double_common.h"
#include "src/math/generic/sincos_eval.h"

// TODO: We might be able to improve the performance of large range reduction of
// non-FMA targets further by operating directly on 25-bit chunks of 128/pi and
// pre-split SIN_K_PI_OVER_128, but that might double the memory footprint of
// those lookup table.
#include "range_reduction_double_common.h"

#if ((LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS) != 0)
#define LIBC_MATH_SINCOS_SKIP_ACCURATE_PASS
#endif
#ifdef LIBC_TARGET_CPU_HAS_FMA
#include "range_reduction_double_fma.h"
#else
#include "range_reduction_double_nofma.h"
#endif // LIBC_TARGET_CPU_HAS_FMA

namespace LIBC_NAMESPACE_DECL {

Expand All @@ -44,40 +41,47 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {

DoubleDouble y;
unsigned k;
generic::LargeRangeReduction<NO_FMA> range_reduction_large{};
LargeRangeReduction range_reduction_large{};

// |x| < 2^32 (with FMA) or |x| < 2^23 (w/o FMA)
// |x| < 2^16
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
// |x| < 2^-27
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0)) {
*sin_x = x;
*cos_x = 1.0;
return;
}

// For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2.
// |x| < 2^-7
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
// |x| < 2^-27
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0)) {
*sin_x = x;
*cos_x = 1.0;
return;
}

// For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2.
#ifdef LIBC_TARGET_CPU_HAS_FMA
*sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
*cos_x = fputil::multiply_add(x, -x, 1.0);
*sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
*cos_x = fputil::multiply_add(x, -x, 1.0);
#else
*cos_x = fputil::round_result_slightly_down(1.0);

if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if (rounding_mode == FE_TOWARDZERO ||
(xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
*sin_x = FPBits(xbits.uintval() - 1).get_val();
}
*sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
*cos_x = fputil::round_result_slightly_down(1.0);

if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if (rounding_mode == FE_TOWARDZERO ||
(xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
*sin_x = FPBits(xbits.uintval() - 1).get_val();
}
*sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
#endif // LIBC_TARGET_CPU_HAS_FMA
return;
return;
}
// No range reduction needed.
k = 0;
y.lo = 0.0;
y.hi = x;
} else {
// Small range reduction.
k = range_reduction_small(x, y);
}

// // Small range reduction.
k = range_reduction_small(x, y);
} else {
// Inf or NaN
if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
Expand All @@ -91,56 +95,46 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
}

// Large range reduction.
k = range_reduction_large.compute_high_part(x);
y = range_reduction_large.fast();
k = range_reduction_large.fast(x, y);
}

DoubleDouble sin_y, cos_y;

generic::sincos_eval(y, sin_y, cos_y);
[[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y);

// Look up sin(k * pi/128) and cos(k * pi/128)
// Memory saving versions:

// Use 128-entry table instead:
// DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 127];
// uint64_t sin_s = static_cast<uint64_t>(k & 128) << (63 - 7);
// sin_k.hi = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// sin_k.lo = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 127];
// uint64_t cos_s = static_cast<uint64_t>((k + 64) & 128) << (63 - 7);
// cos_k.hi = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();
// cos_k.lo = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();

// Use 64-entry table instead:
// auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
// unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
// DoubleDouble ans = SIN_K_PI_OVER_128[idx];
// if (kk & 128) {
// ans.hi = -ans.hi;
// ans.lo = -ans.lo;
// }
// return ans;
// };
// DoubleDouble sin_k = get_idx_dd(k);
// DoubleDouble cos_k = get_idx_dd(k + 64);

#ifdef LIBC_MATH_HAS_SMALL_TABLES
// Memory saving versions. Use 65-entry table.
auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
DoubleDouble ans = SIN_K_PI_OVER_128[idx];
if (kk & 128) {
ans.hi = -ans.hi;
ans.lo = -ans.lo;
}
return ans;
};
DoubleDouble sin_k = get_idx_dd(k);
DoubleDouble cos_k = get_idx_dd(k + 64);
#else
// Fast look up version, but needs 256-entry table.
// cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 255];
DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
#endif // LIBC_MATH_HAS_SMALL_TABLES

DoubleDouble msin_k{-sin_k.lo, -sin_k.hi};

// After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
// So k is an integer and -pi / 256 <= y <= pi / 256.
// Then sin(x) = sin((k * pi/128 + y)
// = sin(y) * cos(k*pi/128) + cos(y) * sin(k*pi/128)
DoubleDouble sin_k_cos_y = fputil::quick_mult<NO_FMA>(cos_y, sin_k);
DoubleDouble cos_k_sin_y = fputil::quick_mult<NO_FMA>(sin_y, cos_k);
DoubleDouble sin_k_cos_y = fputil::quick_mult(cos_y, sin_k);
DoubleDouble cos_k_sin_y = fputil::quick_mult(sin_y, cos_k);
// cos(x) = cos((k * pi/128 + y)
// = cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128)
DoubleDouble cos_k_cos_y = fputil::quick_mult<NO_FMA>(cos_y, cos_k);
DoubleDouble msin_k_sin_y = fputil::quick_mult<NO_FMA>(sin_y, msin_k);
DoubleDouble cos_k_cos_y = fputil::quick_mult(cos_y, cos_k);
DoubleDouble msin_k_sin_y = fputil::quick_mult(sin_y, msin_k);

DoubleDouble sin_dd =
fputil::exact_add<false>(sin_k_cos_y.hi, cos_k_sin_y.hi);
Expand All @@ -149,24 +143,17 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
sin_dd.lo += sin_k_cos_y.lo + cos_k_sin_y.lo;
cos_dd.lo += msin_k_sin_y.lo + cos_k_cos_y.lo;

#ifdef LIBC_MATH_SINCOS_SKIP_ACCURATE_PASS
#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
*sin_x = sin_dd.hi + sin_dd.lo;
*cos_x = cos_dd.hi + cos_dd.lo;
return;
#else
// Accurate test and pass for correctly rounded implementation.

#ifdef LIBC_TARGET_CPU_HAS_FMA
constexpr double ERR = 0x1.0p-70;
#else
// TODO: Improve non-FMA fast pass accuracy.
constexpr double ERR = 0x1.0p-66;
#endif // LIBC_TARGET_CPU_HAS_FMA

double sin_lp = sin_dd.lo + ERR;
double sin_lm = sin_dd.lo - ERR;
double cos_lp = cos_dd.lo + ERR;
double cos_lm = cos_dd.lo - ERR;
double sin_lp = sin_dd.lo + err;
double sin_lm = sin_dd.lo - err;
double cos_lp = cos_dd.lo + err;
double cos_lm = cos_dd.lo - err;

double sin_upper = sin_dd.hi + sin_lp;
double sin_lower = sin_dd.hi + sin_lm;
Expand All @@ -182,15 +169,15 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {

Float128 u_f128, sin_u, cos_u;
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
u_f128 = generic::range_reduction_small_f128(x);
u_f128 = range_reduction_small_f128(x);
else
u_f128 = range_reduction_large.accurate();

generic::sincos_eval(u_f128, sin_u, cos_u);

auto get_sin_k = [](unsigned kk) -> Float128 {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
Float128 ans = generic::SIN_K_PI_OVER_128_F128[idx];
Float128 ans = SIN_K_PI_OVER_128_F128[idx];
if (kk & 128)
ans.sign = Sign::NEG;
return ans;
Expand Down Expand Up @@ -222,7 +209,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
fputil::quick_add(fputil::quick_mul(cos_k_f128, cos_u),
fputil::quick_mul(msin_k_f128, sin_u)));

#endif // !LIBC_MATH_SINCOS_SKIP_ACCURATE_PASS
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
}

} // namespace LIBC_NAMESPACE_DECL
27 changes: 20 additions & 7 deletions libc/src/math/generic/sincos_eval.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ namespace generic {
using fputil::DoubleDouble;
using Float128 = fputil::DyadicFloat<128>;

LIBC_INLINE void sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
DoubleDouble &cos_u) {
LIBC_INLINE double sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
DoubleDouble &cos_u) {
// Evaluate sin(y) = sin(x - k * (pi/128))
// We use the degree-7 Taylor approximation:
// sin(y) ~ y - y^3/3! + y^5/5! - y^7/7!
Expand Down Expand Up @@ -61,9 +61,19 @@ LIBC_INLINE void sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
// + u_hi u_lo (-1 + u_hi^2/6)
// We compute 1 - u_hi^2 accurately:
// v_hi + v_lo ~ 1 - u_hi^2/2
double v_hi = fputil::multiply_add(u.hi, u.hi * (-0.5), 1.0);
double v_lo = 1.0 - v_hi; // Exact
v_lo = fputil::multiply_add(u.hi, u.hi * (-0.5), v_lo);
// with error <= 2^-105.
double u_hi_neg_half = (-0.5) * u.hi;
DoubleDouble v;

#ifdef LIBC_TARGET_CPU_HAS_FMA
v.hi = fputil::multiply_add(u.hi, u_hi_neg_half, 1.0);
v.lo = 1.0 - v.hi; // Exact
v.lo = fputil::multiply_add(u.hi, u_hi_neg_half, v.lo);
#else
DoubleDouble u_hi_sq_neg_half = fputil::exact_mult(u.hi, u_hi_neg_half);
v = fputil::exact_add(1.0, u_hi_sq_neg_half.hi);
v.lo += u_hi_sq_neg_half.lo;
#endif // LIBC_TARGET_CPU_HAS_FMA

// r1 ~ -1/720 + u_hi^2 / 40320
double r1 = fputil::multiply_add(u_hi_sq, 0x1.a01a01a01a01ap-16,
Expand All @@ -75,12 +85,15 @@ LIBC_INLINE void sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
// r2 ~ 1/24 + u_hi^2 (-1/720 + u_hi^2 / 40320)
double r2 = fputil::multiply_add(u_hi_sq, r1, 0x1.5555555555555p-5);
// s2 ~ v_lo + u_hi * u_lo * (-1 + u_hi^2 / 6)
double s2 = fputil::multiply_add(u_hi_u_lo, s1, v_lo);
double s2 = fputil::multiply_add(u_hi_u_lo, s1, v.lo);
double cos_lo = fputil::multiply_add(u_hi_4, r2, s2);
// Overall, |cos(y) - (v_hi + cos_lo)| < 2*ulp(u_hi^4) < 2^-75.

sin_u = fputil::exact_add(u.hi, sin_lo);
cos_u = fputil::exact_add(v_hi, cos_lo);
cos_u = fputil::exact_add(v.hi, cos_lo);

return fputil::multiply_add(fputil::FPBits<double>(u_hi_3).abs().get_val(),
0x1.0p-51, 0x1.0p-105);
}

LIBC_INLINE void sincos_eval(const Float128 &u, Float128 &sin_u,
Expand Down
147 changes: 70 additions & 77 deletions libc/src/math/generic/tan.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,13 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
#include "src/math/generic/range_reduction_double_common.h"

// TODO: We might be able to improve the performance of large range reduction of
// non-FMA targets further by operating directly on 25-bit chunks of 128/pi and
// pre-split SIN_K_PI_OVER_128, but that might double the memory footprint of
// those lookup table.
#include "range_reduction_double_common.h"

#if ((LIBC_MATH & LIBC_MATH_SKIP_ACCURATE_PASS) != 0)
#define LIBC_MATH_TAN_SKIP_ACCURATE_PASS
#endif
#ifdef LIBC_TARGET_CPU_HAS_FMA
#include "range_reduction_double_fma.h"
#else
#include "range_reduction_double_nofma.h"
#endif // LIBC_TARGET_CPU_HAS_FMA

namespace LIBC_NAMESPACE_DECL {

Expand All @@ -38,7 +35,7 @@ using Float128 = typename fputil::DyadicFloat<128>;

namespace {

LIBC_INLINE DoubleDouble tan_eval(const DoubleDouble &u) {
LIBC_INLINE double tan_eval(const DoubleDouble &u, DoubleDouble &result) {
// Evaluate tan(y) = tan(x - k * (pi/128))
// We use the degree-9 Taylor approximation:
// tan(y) ~ P(y) = y + y^3/3 + 2*y^5/15 + 17*y^7/315 + 62*y^9/2835
Expand Down Expand Up @@ -69,10 +66,12 @@ LIBC_INLINE DoubleDouble tan_eval(const DoubleDouble &u) {
// Overall, |tan(y) - (u_hi + tan_lo)| < ulp(u_hi^3) <= 2^-71.
// And the relative errors is:
// |(tan(y) - (u_hi + tan_lo)) / tan(y) | <= 2*ulp(u_hi^2) < 2^-64

return fputil::exact_add(u.hi, tan_lo);
result = fputil::exact_add(u.hi, tan_lo);
return fputil::multiply_add(fputil::FPBits<double>(u_hi_3).abs().get_val(),
0x1.0p-51, 0x1.0p-102);
}

#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
// Accurate evaluation of tan for small u.
[[maybe_unused]] Float128 tan_eval(const Float128 &u) {
Float128 u_sq = fputil::quick_mul(u, u);
Expand Down Expand Up @@ -117,6 +116,7 @@ LIBC_INLINE DoubleDouble tan_eval(const DoubleDouble &u) {
fputil::quick_mul(q1, fputil::quick_add(TWO, fputil::quick_mul(b, q1)));
return fputil::quick_mul(a, q2);
}
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS

} // anonymous namespace

Expand All @@ -128,33 +128,38 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {

DoubleDouble y;
unsigned k;
generic::LargeRangeReduction<NO_FMA> range_reduction_large{};
LargeRangeReduction range_reduction_large{};

// |x| < 2^32 (with FMA) or |x| < 2^23 (w/o FMA)
// |x| < 2^16
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
// |x| < 2^-27
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0))
return x;
// |x| < 2^-7
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
// |x| < 2^-27, |tan(x) - x| < ulp(x)/2.
if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
// Signed zeros.
if (LIBC_UNLIKELY(x == 0.0))
return x;

// For |x| < 2^-27, |tan(x) - x| < ulp(x)/2.
#ifdef LIBC_TARGET_CPU_HAS_FMA
return fputil::multiply_add(x, 0x1.0p-54, x);
return fputil::multiply_add(x, 0x1.0p-54, x);
#else
if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if (rounding_mode == FE_TOWARDZERO ||
(xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
return FPBits(xbits.uintval() + 1).get_val();
}
return fputil::multiply_add(x, 0x1.0p-54, x);
if (LIBC_UNLIKELY(x_e < 4)) {
int rounding_mode = fputil::quick_get_round();
if ((xbits.sign() == Sign::POS && rounding_mode == FE_UPWARD) ||
(xbits.sign() == Sign::NEG && rounding_mode == FE_DOWNWARD))
return FPBits(xbits.uintval() + 1).get_val();
}
return fputil::multiply_add(x, 0x1.0p-54, x);
#endif // LIBC_TARGET_CPU_HAS_FMA
}
// No range reduction needed.
k = 0;
y.lo = 0.0;
y.hi = x;
} else {
// Small range reduction.
k = range_reduction_small(x, y);
}

// // Small range reduction.
k = range_reduction_small(x, y);
} else {
// Inf or NaN
if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
Expand All @@ -167,42 +172,32 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
}

// Large range reduction.
k = range_reduction_large.compute_high_part(x);
y = range_reduction_large.fast();
k = range_reduction_large.fast(x, y);
}

DoubleDouble tan_y = tan_eval(y);
DoubleDouble tan_y;
[[maybe_unused]] double err = tan_eval(y, tan_y);

// Look up sin(k * pi/128) and cos(k * pi/128)
// Memory saving versions:

// Use 128-entry table instead:
// DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 127];
// uint64_t sin_s = static_cast<uint64_t>(k & 128) << (63 - 7);
// sin_k.hi = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// sin_k.lo = FPBits(FPBits(sin_k.hi).uintval() ^ sin_s).get_val();
// DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 127];
// uint64_t cos_s = static_cast<uint64_t>((k + 64) & 128) << (63 - 7);
// cos_k.hi = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();
// cos_k.lo = FPBits(FPBits(cos_k.hi).uintval() ^ cos_s).get_val();

// Use 64-entry table instead:
// auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
// unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
// DoubleDouble ans = SIN_K_PI_OVER_128[idx];
// if (kk & 128) {
// ans.hi = -ans.hi;
// ans.lo = -ans.lo;
// }
// return ans;
// };
// DoubleDouble msin_k = get_idx_dd(k + 128);
// DoubleDouble cos_k = get_idx_dd(k + 64);

#ifdef LIBC_MATH_HAS_SMALL_TABLES
// Memory saving versions. Use 65-entry table:
auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
DoubleDouble ans = SIN_K_PI_OVER_128[idx];
if (kk & 128) {
ans.hi = -ans.hi;
ans.lo = -ans.lo;
}
return ans;
};
DoubleDouble msin_k = get_idx_dd(k + 128);
DoubleDouble cos_k = get_idx_dd(k + 64);
#else
// Fast look up version, but needs 256-entry table.
// cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
DoubleDouble msin_k = SIN_K_PI_OVER_128[(k + 128) & 255];
DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
#endif // LIBC_MATH_HAS_SMALL_TABLES

// After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
// So k is an integer and -pi / 256 <= y <= pi / 256.
Expand All @@ -212,8 +207,8 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
// / (cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128))
// = (sin(k*pi/128) + tan(y) * cos(k*pi/128)) /
// / (cos(k*pi/128) - tan(y) * sin(k*pi/128))
DoubleDouble cos_k_tan_y = fputil::quick_mult<NO_FMA>(tan_y, cos_k);
DoubleDouble msin_k_tan_y = fputil::quick_mult<NO_FMA>(tan_y, msin_k);
DoubleDouble cos_k_tan_y = fputil::quick_mult(tan_y, cos_k);
DoubleDouble msin_k_tan_y = fputil::quick_mult(tan_y, msin_k);

// num_dd = sin(k*pi/128) + tan(y) * cos(k*pi/128)
DoubleDouble num_dd = fputil::exact_add<false>(cos_k_tan_y.hi, -msin_k.hi);
Expand All @@ -222,7 +217,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
num_dd.lo += cos_k_tan_y.lo - msin_k.lo;
den_dd.lo += msin_k_tan_y.lo + cos_k.lo;

#ifdef LIBC_MATH_TAN_SKIP_ACCURATE_PASS
#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
double tan_x = (num_dd.hi + num_dd.lo) / (den_dd.hi + den_dd.lo);
return tan_x;
#else
Expand All @@ -231,18 +226,16 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
// Accurate double-double division
DoubleDouble tan_x = fputil::div(num_dd, den_dd);

// Relative errors for k != 0 mod 64 is:
// absolute errors / min(sin(k*pi/128), cos(k*pi/128)) <= 2^-71 / 2^-7
// = 2^-64.
// For k = 0 mod 64, the relative errors is bounded by:
// 2^-71 / 2^(exponent of x).
constexpr int ERR = 64;
// Simple error bound: |1 / den_dd| < 2^(1 + floor(-log2(den_dd)))).
uint64_t den_inv = (static_cast<uint64_t>(FPBits::EXP_BIAS + 1)
<< (FPBits::FRACTION_LEN + 1)) -
(FPBits(den_dd.hi).uintval() & FPBits::EXP_MASK);

int y_exp = 7 + FPBits(y.hi).get_exponent();
int rel_err_exp = ERR + static_cast<int>((k & 63) == 0) * y_exp;
int64_t tan_x_err = static_cast<int64_t>(FPBits(tan_x.hi).uintval()) -
(static_cast<int64_t>(rel_err_exp) << 52);
double tan_err = FPBits(static_cast<uint64_t>(tan_x_err)).get_val();
// For tan_x = (num_dd + err) / (den_dd + err), the error is bounded by:
// | tan_x - num_dd / den_dd | <= err * ( 1 + | tan_x * den_dd | ).
double tan_err =
err * fputil::multiply_add(FPBits(den_inv).get_val(),
FPBits(tan_x.hi).abs().get_val(), 1.0);

double err_higher = tan_x.lo + tan_err;
double err_lower = tan_x.lo - tan_err;
Expand All @@ -256,15 +249,15 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {

Float128 u_f128;
if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
u_f128 = generic::range_reduction_small_f128(x);
u_f128 = range_reduction_small_f128(x);
else
u_f128 = range_reduction_large.accurate();

Float128 tan_u = tan_eval(u_f128);

auto get_sin_k = [](unsigned kk) -> Float128 {
unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
Float128 ans = generic::SIN_K_PI_OVER_128_F128[idx];
Float128 ans = SIN_K_PI_OVER_128_F128[idx];
if (kk & 128)
ans.sign = Sign::NEG;
return ans;
Expand Down Expand Up @@ -292,7 +285,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
// https://github.com/llvm/llvm-project/issues/96452.
return static_cast<double>(result);

#endif // !LIBC_MATH_TAN_SKIP_ACCURATE_PASS
#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
}

} // namespace LIBC_NAMESPACE_DECL
3 changes: 1 addition & 2 deletions libc/test/src/math/cos_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ TEST_F(LlvmLibcCosTest, TrickyInputs) {
0x1.2b5fe88a9d8d5p+903, 0x1.f6d7518808571p+1023,
-0x1.a880417b7b119p+1023, 0x1.00a33764a0a83p-7,
0x1.fe81868fc47fep+1, 0x1.0da8cc189b47dp-10,
0x1.da1838053b866p+5,

0x1.da1838053b866p+5, 0x1.ffffffffe854bp199,
};
constexpr int N = sizeof(INPUTS) / sizeof(INPUTS[0]);

Expand Down
12 changes: 7 additions & 5 deletions libc/test/src/math/sin_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ using LIBC_NAMESPACE::testing::tlog;

TEST_F(LlvmLibcSinTest, TrickyInputs) {
constexpr double INPUTS[] = {
0x1.940c877fb7dacp-7, 0x1.fffffffffdb6p24, 0x1.fd4da4ef37075p29,
0x1.b951f1572eba5p+31, 0x1.55202aefde314p+31, 0x1.85fc0f04c0128p101,
0x1.7776c2343ba4ep101, 0x1.678309fa50d58p110, 0x1.fffffffffef4ep199,
-0x1.ab514bfc61c76p+7, -0x1.f7898d5a756ddp+2, -0x1.f42fb19b5b9b2p-6,
0x1.5f09cad750ab1p+3, -0x1.14823229799c2p+7, -0x1.0285070f9f1bcp-5,
0x1.5f09cad750ab1p+3, 0x1.fff781921b61fp15, -0x1.f635b70b92407p-21,
-0x1.3ecf146c39c0cp-20, 0x1.6ac5b262ca1ffp849, 0x1.6c6cbc45dc8dep5,
0x1.921fb5443p-7, 0x1.940c877fb7dacp-7, 0x1.fffffffffdb6p24,
0x1.fd4da4ef37075p29, 0x1.b951f1572eba5p+31, 0x1.55202aefde314p+31,
0x1.85fc0f04c0128p101, 0x1.7776c2343ba4ep101, 0x1.678309fa50d58p110,
0x1.fffffffffef4ep199, -0x1.ab514bfc61c76p+7, -0x1.f7898d5a756ddp+2,
-0x1.f42fb19b5b9b2p-6, -0x1.14823229799c2p+7, -0x1.0285070f9f1bcp-5,
0x1.23f40dccdef72p+0, 0x1.43cf16358c9d7p+0, 0x1.addf3b9722265p+0,
0x1.48ff1782ca91dp+8, 0x1.a211877de55dbp+4, 0x1.dcbfda0c7559ep+8,
0x1.1ffb509f3db15p+5, 0x1.2345d1e090529p+5, 0x1.ae945054939c2p+10,
Expand Down
21 changes: 12 additions & 9 deletions libc/test/src/math/tan_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,20 @@ using LIBC_NAMESPACE::testing::tlog;

TEST_F(LlvmLibcTanTest, TrickyInputs) {
constexpr double INPUTS[] = {
0x1.d130383d17321p-27, 0x1.8000000000009p-23, 0x1.8000000000024p-22,
0x1.800000000009p-21, 0x1.20000000000f3p-20, 0x1.800000000024p-20,
0x1.e0000000001c2p-20, 0x1.00452f0e0134dp-13, 0x1.0da8cc189b47dp-10,
0x1.00a33764a0a83p-7, 0x1.911a18779813fp-7, 0x1.940c877fb7dacp-7,
0x1.f42fb19b5b9b2p-6, 0x1.0285070f9f1bcp-5, 0x1.89f0f5241255bp-2,
0x0.0000000000001p-1022, 0x1.d130383d17321p-27, 0x1.8000000000009p-23,
0x1.8000000000024p-22, 0x1.800000000009p-21, 0x1.20000000000f3p-20,
0x1.800000000024p-20, 0x1.e0000000001c2p-20, 0x1.00452f0e0134dp-13,
0x1.0da8cc189b47dp-10, 0x1.00a33764a0a83p-7, 0x1.911a18779813fp-7,
0x1.940c877fb7dacp-7, 0x1.f42fb19b5b9b2p-6, 0x1.0285070f9f1bcp-5,
0x1.90e833c6969c7p-4, 0x1.91d4b77c527eap-3, 0x1.89f0f5241255bp-2,
0x1.6ca9ef729af76p-1, 0x1.23f40dccdef72p+0, 0x1.43cf16358c9d7p+0,
0x1.90f422b49115ep+0, 0x1.9220efee9fc7ep+0, 0x1.a224411cdebcep+0,
0x1.addf3b9722265p+0, 0x1.ae78d360afa15p+0, 0x1.fe81868fc47fep+1,
0x1.e31b55306f22cp+2, 0x1.e639103a05997p+2, 0x1.f7898d5a756ddp+2,
0x1.1685973506319p+3, 0x1.5f09cad750ab1p+3, 0x1.aaf85537ea4c7p+3,
0x1.4f2b874135d27p+4, 0x1.13114266f9764p+4, 0x1.a211877de55dbp+4,
0x1.a5eece87e8606p+4, 0x1.a65d441ea6dcep+4, 0x1.045457ae3994p+5,
0x1.e31b55306f22cp+2, 0x1.e639103a05997p+2, 0x1.f69d074a3358fp+2,
0x1.f7898d5a756ddp+2, 0x1.1685973506319p+3, 0x1.5f09cad750ab1p+3,
0x1.aaf85537ea4c7p+3, 0x1.c50ddc4f513b4p+3, 0x1.13114266f9764p+4,
0x1.4f2b874135d27p+4, 0x1.a211877de55dbp+4, 0x1.a5eece87e8606p+4,
0x1.a65d441ea6dcep+4, 0x1.ab8c2f8ab5b7p+4, 0x1.045457ae3994p+5,
0x1.1ffb509f3db15p+5, 0x1.2345d1e090529p+5, 0x1.c96e28eb679f8p+5,
0x1.da1838053b866p+5, 0x1.be886d9c2324dp+6, 0x1.ab514bfc61c76p+7,
0x1.14823229799c2p+7, 0x1.48ff1782ca91dp+8, 0x1.dcbfda0c7559ep+8,
Expand Down