Merge bitcoin/bitcoin#25001: Modernize util/strencodings and util/str…

…ing: `string_view` and `optional`

fa7078d scripted-diff: Rename ValidAsCString to ContainsNoNUL (MacroFake)
e7d2fbd Use std::string_view throughout util strencodings/string (Pieter Wuille)
8ffbd14 Make DecodeBase{32,64} take string_view arguments (Pieter Wuille)
1a72d62 Generalize ConvertBits to permit transforming the input (Pieter Wuille)
78f3ac5 Make DecodeBase{32,64} return optional instead of taking bool* (Pieter Wuille)
a65931e Make DecodeBase{32,64} always return vector, not string (Pieter Wuille)
a4377a0 Reject incorrect base64 in HTTP auth (Pieter Wuille)
d648b51 Make SanitizeString use string_view (Pieter Wuille)
963bc9b Make IsHexNumber use string_view (Pieter Wuille)
4006299 Make IsHex use string_view (Pieter Wuille)
c1d165a Make ParseHex use string_view (Pieter Wuille)

Pull request description:

  Make use of `std::string_view` and `std::optional` in the util/{strencodings, string} files.

  This avoids many temporary string/vector objects being created, while making the interface easier to read. Changes include:
  * Make all input arguments in functions in util/strencodings and util/string take `std::string_view` instead of `std::string`.
  * Add `RemovePrefixView` and `TrimStringView` which also *return* `std::string_view` objects (the corresponding `RemovePrefix` and `TrimString` keep returning an `std::string`, as that's needed in many call sites still).
  * Stop returning `std::string` from `DecodeBase32` and `DecodeBase64`, but return vectors. Base32/64 are fundamentally algorithms for encoding bytes as strings; returning `std::string` from those (especially doing it conditionally based on the input arguments/types) is just bizarre.
  * Stop taking a `bool* pf_invalid` output argument pointer in `DecodeBase32` and `DecodeBase64`; return an `std::optional` instead.
  * Make `DecodeBase32` and `DecodeBase64` more efficient by doing the conversion from characters to integer symbols on-the-fly rather than through a temporary vector.

ACKs for top commit:
  MarcoFalke:
    re-ACK fa7078d only change is rebase and adding a scripted-diff 🍲
  martinus:
    Code review ACK fa7078d, found no issue
  laanwj:
    Code review ACK  fa7078d
  sipa:
    utACK fa7078d (as far as the commit that isn't mine goes)

Tree-SHA512: 5cf02e541caef0bcd100466747664bdb828a68a05dae568cbcd0632a53dd3a4c4e85cd8c48ebbd168d4247d5c9666689c16005f1c8ad75b0f057d8683931f664

src/base58.cpp

@@ -126,7 +126,7 @@ std::string EncodeBase58(Span<const unsigned char> input)
 
 bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len)
 {
-    if (!ValidAsCString(str)) {
+    if (!ContainsNoNUL(str)) {
         return false;
     }
     return DecodeBase58(str.c_str(), vchRet, max_ret_len);
@@ -160,7 +160,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input)
 
 bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret)
 {
-    if (!ValidAsCString(str)) {
+    if (!ContainsNoNUL(str)) {
         return false;
     }
     return DecodeBase58Check(str.c_str(), vchRet, max_ret);

src/bitcoin-tx.cpp

@@ -240,7 +240,7 @@ static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInId
 template <typename T>
 static T TrimAndParse(const std::string& int_str, const std::string& err)
 {
-    const auto parsed{ToIntegral<T>(TrimString(int_str))};
+    const auto parsed{ToIntegral<T>(TrimStringView(int_str))};
     if (!parsed.has_value()) {
         throw std::runtime_error(err + " '" + int_str + "'");
     }

src/httprpc.cpp

@@ -131,8 +131,11 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna
         return false;
     if (strAuth.substr(0, 6) != "Basic ")
         return false;
-    std::string strUserPass64 = TrimString(strAuth.substr(6));
-    std::string strUserPass = DecodeBase64(strUserPass64);
+    std::string_view strUserPass64 = TrimStringView(std::string_view{strAuth}.substr(6));
+    auto userpass_data = DecodeBase64(strUserPass64);
+    std::string strUserPass;
+    if (!userpass_data) return false;
+    strUserPass.assign(userpass_data->begin(), userpass_data->end());
 
     if (strUserPass.find(':') != std::string::npos)
         strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':'));

src/i2p.cpp

@@ -69,12 +69,11 @@ static std::string SwapBase64(const std::string& from)
 static Binary DecodeI2PBase64(const std::string& i2p_b64)
 {
     const std::string& std_b64 = SwapBase64(i2p_b64);
-    bool invalid;
-    Binary decoded = DecodeBase64(std_b64.c_str(), &invalid);
-    if (invalid) {
+    auto decoded = DecodeBase64(std_b64);
+    if (!decoded) {
         throw std::runtime_error(strprintf("Cannot decode Base64: \"%s\"", i2p_b64));
     }
-    return decoded;
+    return std::move(*decoded);
 }
 
 /**

src/netaddress.cpp

@@ -210,7 +210,7 @@ static void Checksum(Span<const uint8_t> addr_pubkey, uint8_t (&checksum)[CHECKS
 
 bool CNetAddr::SetSpecial(const std::string& addr)
 {
-    if (!ValidAsCString(addr)) {
+    if (!ContainsNoNUL(addr)) {
         return false;
     }
 
@@ -234,17 +234,16 @@ bool CNetAddr::SetTor(const std::string& addr)
         return false;
     }
 
-    bool invalid;
-    const auto& input = DecodeBase32(addr.substr(0, addr.size() - suffix_len).c_str(), &invalid);
+    auto input = DecodeBase32(std::string_view{addr}.substr(0, addr.size() - suffix_len));
 
-    if (invalid) {
+    if (!input) {
         return false;
     }
 
-    if (input.size() == torv3::TOTAL_LEN) {
-        Span<const uint8_t> input_pubkey{input.data(), ADDR_TORV3_SIZE};
-        Span<const uint8_t> input_checksum{input.data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN};
-        Span<const uint8_t> input_version{input.data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)};
+    if (input->size() == torv3::TOTAL_LEN) {
+        Span<const uint8_t> input_pubkey{input->data(), ADDR_TORV3_SIZE};
+        Span<const uint8_t> input_checksum{input->data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN};
+        Span<const uint8_t> input_version{input->data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)};
 
         if (input_version != torv3::VERSION) {
             return false;
@@ -280,15 +279,14 @@ bool CNetAddr::SetI2P(const std::string& addr)
     // can decode it.
     const std::string b32_padded = addr.substr(0, b32_len) + "====";
 
-    bool invalid;
-    const auto& address_bytes = DecodeBase32(b32_padded.c_str(), &invalid);
+    auto address_bytes = DecodeBase32(b32_padded);
 
-    if (invalid || address_bytes.size() != ADDR_I2P_SIZE) {
+    if (!address_bytes || address_bytes->size() != ADDR_I2P_SIZE) {
         return false;
     }
 
     m_net = NET_I2P;
-    m_addr.assign(address_bytes.begin(), address_bytes.end());
+    m_addr.assign(address_bytes->begin(), address_bytes->end());
 
     return true;
 }

src/netbase.cpp

@@ -136,7 +136,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un
 {
     vIP.clear();
 
-    if (!ValidAsCString(name)) {
+    if (!ContainsNoNUL(name)) {
         return false;
     }
 
@@ -169,7 +169,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un
 
 bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function)
 {
-    if (!ValidAsCString(name)) {
+    if (!ContainsNoNUL(name)) {
         return false;
     }
     std::string strHost = name;
@@ -184,7 +184,7 @@ bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned in
 
 bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function)
 {
-    if (!ValidAsCString(name)) {
+    if (!ContainsNoNUL(name)) {
         return false;
     }
     std::vector<CNetAddr> vIP;
@@ -197,7 +197,7 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL
 
 bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function)
 {
-    if (name.empty() || !ValidAsCString(name)) {
+    if (name.empty() || !ContainsNoNUL(name)) {
         return false;
     }
     uint16_t port{portDefault};
@@ -216,7 +216,7 @@ bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t port
 
 bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function)
 {
-    if (!ValidAsCString(name)) {
+    if (!ContainsNoNUL(name)) {
         return false;
     }
     std::vector<CService> vService;
@@ -229,7 +229,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool
 
 CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function)
 {
-    if (!ValidAsCString(name)) {
+    if (!ContainsNoNUL(name)) {
         return {};
     }
     CService addr;
@@ -684,7 +684,7 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_
 
 bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out)
 {
-    if (!ValidAsCString(subnet_str)) {
+    if (!ContainsNoNUL(subnet_str)) {
         return false;
     }
 

src/psbt.cpp

@@ -388,18 +388,17 @@ std::string PSBTRoleName(PSBTRole role) {
 
 bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error)
 {
-    bool invalid;
-    std::string tx_data = DecodeBase64(base64_tx, &invalid);
-    if (invalid) {
+    auto tx_data = DecodeBase64(base64_tx);
+    if (!tx_data) {
         error = "invalid base64";
         return false;
     }
-    return DecodeRawPSBT(psbt, tx_data, error);
+    return DecodeRawPSBT(psbt, MakeByteSpan(*tx_data), error);
 }
 
-bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error)
+bool DecodeRawPSBT(PartiallySignedTransaction& psbt, Span<const std::byte> tx_data, std::string& error)
 {
-    CDataStream ss_data(MakeByteSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION);
+    CDataStream ss_data(tx_data, SER_NETWORK, PROTOCOL_VERSION);
     try {
         ss_data >> psbt;
         if (!ss_data.empty()) {

src/psbt.h

@@ -988,6 +988,6 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti
 //! Decode a base64ed PSBT into a PartiallySignedTransaction
 [[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error);
 //! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction
-[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error);
+[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, Span<const std::byte> raw_psbt, std::string& error);
 
 #endif // BITCOIN_PSBT_H

src/qt/walletframe.cpp

@@ -194,16 +194,16 @@ void WalletFrame::gotoVerifyMessageTab(QString addr)
 
 void WalletFrame::gotoLoadPSBT(bool from_clipboard)
 {
-    std::string data;
+    std::vector<unsigned char> data;
 
     if (from_clipboard) {
         std::string raw = QApplication::clipboard()->text().toStdString();
-        bool invalid;
-        data = DecodeBase64(raw, &invalid);
-        if (invalid) {
+        auto result = DecodeBase64(raw);
+        if (!result) {
             Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR);
             return;
         }
+        data = std::move(*result);
     } else {
         QString filename = GUIUtil::getOpenFileName(this,
             tr("Load Transaction Data"), QString(),
@@ -214,12 +214,12 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard)
             return;
         }
         std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary};
-        data = std::string(std::istreambuf_iterator<char>{in}, {});
+        data.assign(std::istream_iterator<unsigned char>{in}, {});
     }
 
     std::string error;
     PartiallySignedTransaction psbtx;
-    if (!DecodeRawPSBT(psbtx, data, error)) {
+    if (!DecodeRawPSBT(psbtx, MakeByteSpan(data), error)) {
         Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR);
         return;
     }

src/test/base32_tests.cpp

@@ -22,20 +22,16 @@ BOOST_AUTO_TEST_CASE(base32_testvectors)
         BOOST_CHECK_EQUAL(strEnc, vstrOut[i]);
         strEnc = EncodeBase32(vstrIn[i], false);
         BOOST_CHECK_EQUAL(strEnc, vstrOutNoPadding[i]);
-        std::string strDec = DecodeBase32(vstrOut[i]);
-        BOOST_CHECK_EQUAL(strDec, vstrIn[i]);
+        auto dec = DecodeBase32(vstrOut[i]);
+        BOOST_REQUIRE(dec);
+        BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]);
     }
 
     // Decoding strings with embedded NUL characters should fail
-    bool failure;
-    (void)DecodeBase32("invalid\0"s, &failure); // correct size, invalid due to \0
-    BOOST_CHECK(failure);
-    (void)DecodeBase32("AWSX3VPP"s, &failure); // valid
-    BOOST_CHECK(!failure);
-    (void)DecodeBase32("AWSX3VPP\0invalid"s, &failure); // correct size, invalid due to \0
-    BOOST_CHECK(failure);
-    (void)DecodeBase32("AWSX3VPPinvalid"s, &failure); // invalid size
-    BOOST_CHECK(failure);
+    BOOST_CHECK(!DecodeBase32("invalid\0"s)); // correct size, invalid due to \0
+    BOOST_CHECK(DecodeBase32("AWSX3VPP"s)); // valid
+    BOOST_CHECK(!DecodeBase32("AWSX3VPP\0invalid"s)); // correct size, invalid due to \0
+    BOOST_CHECK(!DecodeBase32("AWSX3VPPinvalid"s)); // invalid size
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/base64_tests.cpp

@@ -19,8 +19,9 @@ BOOST_AUTO_TEST_CASE(base64_testvectors)
     {
         std::string strEnc = EncodeBase64(vstrIn[i]);
         BOOST_CHECK_EQUAL(strEnc, vstrOut[i]);
-        std::string strDec = DecodeBase64(strEnc);
-        BOOST_CHECK_EQUAL(strDec, vstrIn[i]);
+        auto dec = DecodeBase64(strEnc);
+        BOOST_REQUIRE(dec);
+        BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]);
     }
 
     {
@@ -34,15 +35,10 @@ BOOST_AUTO_TEST_CASE(base64_testvectors)
     }
 
     // Decoding strings with embedded NUL characters should fail
-    bool failure;
-    (void)DecodeBase64("invalid\0"s, &failure);
-    BOOST_CHECK(failure);
-    (void)DecodeBase64("nQB/pZw="s, &failure);
-    BOOST_CHECK(!failure);
-    (void)DecodeBase64("nQB/pZw=\0invalid"s, &failure);
-    BOOST_CHECK(failure);
-    (void)DecodeBase64("nQB/pZw=invalid\0"s, &failure);
-    BOOST_CHECK(failure);
+    BOOST_CHECK(!DecodeBase64("invalid\0"s));
+    BOOST_CHECK(DecodeBase64("nQB/pZw="s));
+    BOOST_CHECK(!DecodeBase64("nQB/pZw=\0invalid"s));
+    BOOST_CHECK(!DecodeBase64("nQB/pZw=invalid\0"s));
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/fuzz/base_encode_decode.cpp

@@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode)
     std::vector<unsigned char> decoded;
     if (DecodeBase58(random_encoded_string, decoded, 100)) {
         const std::string encoded_string = EncodeBase58(decoded);
-        assert(encoded_string == TrimString(encoded_string));
+        assert(encoded_string == TrimStringView(encoded_string));
         assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string)));
     }
 
@@ -36,17 +36,16 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode)
         assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string)));
     }
 
-    bool pf_invalid;
-    std::string decoded_string = DecodeBase32(random_encoded_string, &pf_invalid);
-    if (!pf_invalid) {
-        const std::string encoded_string = EncodeBase32(decoded_string);
-        assert(encoded_string == TrimString(encoded_string));
+    auto result = DecodeBase32(random_encoded_string);
+    if (result) {
+        const std::string encoded_string = EncodeBase32(*result);
+        assert(encoded_string == TrimStringView(encoded_string));
         assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string)));
     }
 
-    decoded_string = DecodeBase64(random_encoded_string, &pf_invalid);
-    if (!pf_invalid) {
-        const std::string encoded_string = EncodeBase64(decoded_string);
+    result = DecodeBase64(random_encoded_string);
+    if (result) {
+        const std::string encoded_string = EncodeBase64(*result);
         assert(encoded_string == TrimString(encoded_string));
         assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string)));
     }

src/test/fuzz/http_request.cpp

@@ -39,7 +39,7 @@ FUZZ_TARGET(http_request)
     // and is a consequence of our hacky but necessary use of the internal function evhttp_parse_firstline_ in
     // this fuzzing harness. The workaround is not aesthetically pleasing, but it successfully avoids the troublesome
     // code path. " http:// HTTP/1.1\n" was a crashing input prior to this workaround.
-    const std::string http_buffer_str = ToLower({http_buffer.begin(), http_buffer.end()});
+    const std::string http_buffer_str = ToLower(std::string{http_buffer.begin(), http_buffer.end()});
     if (http_buffer_str.find(" http://") != std::string::npos || http_buffer_str.find(" https://") != std::string::npos ||
         evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) {
         evbuffer_free(evbuf);

src/test/fuzz/psbt.cpp

@@ -32,7 +32,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt)
     FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
     PartiallySignedTransaction psbt_mut;
     std::string error;
-    if (!DecodeRawPSBT(psbt_mut, fuzzed_data_provider.ConsumeRandomLengthString(), error)) {
+    auto str = fuzzed_data_provider.ConsumeRandomLengthString();
+    if (!DecodeRawPSBT(psbt_mut, MakeByteSpan(str), error)) {
         return;
     }
     const PartiallySignedTransaction psbt = psbt_mut;
@@ -79,7 +80,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt)
     }
 
     PartiallySignedTransaction psbt_merge;
-    if (!DecodeRawPSBT(psbt_merge, fuzzed_data_provider.ConsumeRandomLengthString(), error)) {
+    str = fuzzed_data_provider.ConsumeRandomLengthString();
+    if (!DecodeRawPSBT(psbt_merge, MakeByteSpan(str), error)) {
         psbt_merge = psbt;
     }
     psbt_mut = psbt;

src/test/fuzz/string.cpp

@@ -42,7 +42,7 @@ bool LegacyParsePrechecks(const std::string& str)
         return false;
     if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size() - 1]))) // No padding allowed
         return false;
-    if (!ValidAsCString(str)) // No embedded NUL characters allowed
+    if (!ContainsNoNUL(str)) // No embedded NUL characters allowed
         return false;
     return true;
 }
@@ -188,7 +188,7 @@ FUZZ_TARGET(string)
     (void)TrimString(random_string_1);
     (void)TrimString(random_string_1, random_string_2);
     (void)urlDecode(random_string_1);
-    (void)ValidAsCString(random_string_1);
+    (void)ContainsNoNUL(random_string_1);
     (void)_(random_string_1.c_str());
     try {
         throw scriptnum_error{random_string_1};