Add RSA-OAEP encrypt/decrypt across all crypto backends

src/core/crypto/CMakeLists.txt

@@ -3,7 +3,7 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME crypto
     sha512.h sha1.h
     fnv128.h
     uuid.h crc32.h base64.h secure.h verify.h sign.h aes_gcm.h aes_cbc_hmac.h
-    aes_kw.h
+    aes_kw.h rsa_oaep.h
   SOURCES crypto_sha256.cc crypto_hmac_sha256.cc crypto_hmac_sha384.cc
     crypto_hmac_sha512.cc
     crypto_sha384.cc
@@ -26,7 +26,7 @@ if(SOURCEMETA_CORE_CRYPTO_USE_SYSTEM_OPENSSL)
     crypto_sha512_openssl.cc crypto_random_openssl.cc
     crypto_verify_openssl.cc crypto_secure_equals_openssl.cc
     crypto_sign_openssl.cc crypto_aes_gcm_openssl.cc crypto_aes_cbc_openssl.cc
-    crypto_aes_kw_openssl.cc
+    crypto_aes_kw_openssl.cc crypto_rsa_oaep_openssl.cc
     crypto_openssl.h crypto_pkcs8.h)
   target_link_libraries(sourcemeta_core_crypto PRIVATE OpenSSL::Crypto)
 elseif(APPLE)
@@ -117,7 +117,7 @@ elseif(APPLE)
     crypto_sha512_apple.cc crypto_random_apple.cc
     crypto_verify_apple.cc crypto_secure_equals_apple.cc
     crypto_sign_apple.cc crypto_aes_gcm_apple.cc crypto_aes_cbc_apple.cc
-    crypto_aes_kw_apple.cc
+    crypto_aes_kw_apple.cc crypto_rsa_oaep_apple.cc
     crypto_eddsa_cryptokit.mm "${CRYPTOKIT_OBJECT}"
     crypto_eddsa.h crypto_eddsa_apple.h crypto_apple.h crypto_aes_apple.h
     crypto_bignum.h crypto_shake256.h crypto_pkcs8.h)
@@ -148,7 +148,7 @@ elseif(WIN32)
     crypto_sha512_windows.cc crypto_random_windows.cc
     crypto_verify_windows.cc crypto_secure_equals_generic.cc
     crypto_sign_windows.cc crypto_aes_gcm_windows.cc crypto_aes_cbc_windows.cc
-    crypto_aes_kw_windows.cc
+    crypto_aes_kw_windows.cc crypto_rsa_oaep_windows.cc
     crypto_eddsa.h
     crypto_windows.h crypto_aes_kw_loop.h
     crypto_bignum.h crypto_ecc.h crypto_shake256.h crypto_pkcs8.h)
@@ -163,7 +163,8 @@ else()
     crypto_sha512_other.cc crypto_random_other.cc crypto_verify_other.cc
     crypto_secure_equals_generic.cc crypto_sign_other.cc
     crypto_aes_gcm_other.cc crypto_aes_cbc_other.cc crypto_aes_kw_other.cc
-    crypto_aes_block.h crypto_aes_kw_loop.h
+    crypto_rsa_oaep_other.cc
+    crypto_aes_block.h crypto_aes_kw_loop.h crypto_sha1_other.h
     crypto_bignum.h crypto_ecc.h crypto_eddsa.h crypto_shake256.h
     crypto_pkcs8.h)
 endif()

src/core/crypto/crypto_rsa_oaep_apple.cc

@@ -0,0 +1,109 @@
+#include <sourcemeta/core/crypto_rsa_oaep.h>
+
+#include <CoreFoundation/CoreFoundation.h> // CF*, kCF*
+#include <Security/Security.h>             // Sec*, kSec*
+
+#include <cstddef>     // std::size_t
+#include <cstdint>     // std::uint8_t
+#include <optional>    // std::optional, std::nullopt
+#include <string>      // std::string
+#include <string_view> // std::string_view
+#include <utility>     // std::unreachable
+
+namespace sourcemeta::core {
+
+// The layout matches the definitions in the sibling Apple key backends, since
+// each translation unit that reads a key redeclares its file-private members
+struct PublicKey::Internal {
+  PublicKey::Type kind;
+  SecKeyRef key;
+  std::string modulus;
+  std::size_t field_bytes;
+  std::string edwards_point;
+  EdwardsCurve edwards_curve;
+};
+
+struct PrivateKey::Internal {
+  PrivateKey::Type kind;
+  SecKeyRef key;
+  std::size_t field_bytes;
+  std::string edwards_seed;
+  EdwardsCurve edwards_curve;
+  bool rsa_pss_restricted{false};
+};
+
+namespace {
+auto make_data(const std::string_view value) -> CFDataRef {
+  return CFDataCreate(kCFAllocatorDefault,
+                      reinterpret_cast<const std::uint8_t *>(value.data()),
+                      static_cast<CFIndex>(value.size()));
+}
+
+auto to_oaep_algorithm(const RSAOAEPHash hash) -> SecKeyAlgorithm {
+  switch (hash) {
+    case RSAOAEPHash::SHA1:
+      return kSecKeyAlgorithmRSAEncryptionOAEPSHA1;
+    case RSAOAEPHash::SHA256:
+      return kSecKeyAlgorithmRSAEncryptionOAEPSHA256;
+  }
+
+  std::unreachable();
+}
+} // namespace
+
+auto rsa_oaep_encrypt(const PublicKey &key, const RSAOAEPHash hash,
+                      const std::string_view plaintext)
+    -> std::optional<std::string> {
+  const auto *const internal{key.internal()};
+  if (internal == nullptr || internal->kind != PublicKey::Type::RSA) {
+    return std::nullopt;
+  }
+
+  auto plaintext_data{make_data(plaintext)};
+  if (plaintext_data == nullptr) {
+    return std::nullopt;
+  }
+
+  auto ciphertext{SecKeyCreateEncryptedData(
+      internal->key, to_oaep_algorithm(hash), plaintext_data, nullptr)};
+  CFRelease(plaintext_data);
+  if (ciphertext == nullptr) {
+    return std::nullopt;
+  }
+
+  std::string result{
+      reinterpret_cast<const char *>(CFDataGetBytePtr(ciphertext)),
+      static_cast<std::size_t>(CFDataGetLength(ciphertext))};
+  CFRelease(ciphertext);
+  return result;
+}
+
+auto rsa_oaep_decrypt(const PrivateKey &key, const RSAOAEPHash hash,
+                      const std::string_view ciphertext)
+    -> std::optional<std::string> {
+  const auto *const internal{key.internal()};
+  if (internal == nullptr || internal->kind != PrivateKey::Type::RSA ||
+      internal->rsa_pss_restricted) {
+    return std::nullopt;
+  }
+
+  auto ciphertext_data{make_data(ciphertext)};
+  if (ciphertext_data == nullptr) {
+    return std::nullopt;
+  }
+
+  auto plaintext{SecKeyCreateDecryptedData(
+      internal->key, to_oaep_algorithm(hash), ciphertext_data, nullptr)};
+  CFRelease(ciphertext_data);
+  if (plaintext == nullptr) {
+    return std::nullopt;
+  }
+
+  std::string result{
+      reinterpret_cast<const char *>(CFDataGetBytePtr(plaintext)),
+      static_cast<std::size_t>(CFDataGetLength(plaintext))};
+  CFRelease(plaintext);
+  return result;
+}
+
+} // namespace sourcemeta::core

src/core/crypto/crypto_rsa_oaep_openssl.cc

@@ -0,0 +1,126 @@
+#include <sourcemeta/core/crypto_rsa_oaep.h>
+
+#include <openssl/evp.h> // EVP_*
+#include <openssl/rsa.h> // RSA_PKCS1_OAEP_PADDING
+
+#include <cstddef>     // std::size_t
+#include <optional>    // std::optional, std::nullopt
+#include <string>      // std::string
+#include <string_view> // std::string_view
+#include <utility>     // std::move, std::unreachable
+
+namespace sourcemeta::core {
+
+// The layout matches the definitions in the sibling OpenSSL key backends, since
+// each translation unit that reads a key redeclares its file-private members
+struct PublicKey::Internal {
+  PublicKey::Type kind;
+  EVP_PKEY *key;
+  std::string modulus;
+  std::size_t field_bytes;
+  std::size_t signature_bytes;
+};
+
+struct PrivateKey::Internal {
+  PrivateKey::Type kind;
+  EVP_PKEY *key;
+  std::size_t field_bytes;
+  bool rsa_pss_restricted{false};
+};
+
+namespace {
+auto to_message_digest(const RSAOAEPHash hash) -> const EVP_MD * {
+  switch (hash) {
+    case RSAOAEPHash::SHA1:
+      return EVP_sha1();
+    case RSAOAEPHash::SHA256:
+      return EVP_sha256();
+  }
+
+  std::unreachable();
+}
+
+auto configure(EVP_PKEY_CTX *context, const RSAOAEPHash hash) -> bool {
+  return EVP_PKEY_CTX_set_rsa_padding(context, RSA_PKCS1_OAEP_PADDING) == 1 &&
+         EVP_PKEY_CTX_set_rsa_oaep_md(context, to_message_digest(hash)) == 1 &&
+         EVP_PKEY_CTX_set_rsa_mgf1_md(context, to_message_digest(hash)) == 1;
+}
+} // namespace
+
+auto rsa_oaep_encrypt(const PublicKey &key, const RSAOAEPHash hash,
+                      const std::string_view plaintext)
+    -> std::optional<std::string> {
+  const auto *const internal{key.internal()};
+  if (internal == nullptr || internal->kind != PublicKey::Type::RSA) {
+    return std::nullopt;
+  }
+
+  auto *context{EVP_PKEY_CTX_new(internal->key, nullptr)};
+  if (context == nullptr) {
+    return std::nullopt;
+  }
+
+  const auto *const input{
+      reinterpret_cast<const unsigned char *>(plaintext.data())};
+  std::optional<std::string> result;
+  std::size_t length{0};
+  if (EVP_PKEY_encrypt_init(context) == 1 && configure(context, hash) &&
+      EVP_PKEY_encrypt(context, nullptr, &length, input, plaintext.size()) ==
+          1) {
+    std::string ciphertext(length, '\x00');
+    if (EVP_PKEY_encrypt(context,
+                         reinterpret_cast<unsigned char *>(ciphertext.data()),
+                         &length, input, plaintext.size()) == 1) {
+      ciphertext.resize(length);
+      result = std::move(ciphertext);
+    }
+  }
+
+  EVP_PKEY_CTX_free(context);
+  return result;
+}
+
+auto rsa_oaep_decrypt(const PrivateKey &key, const RSAOAEPHash hash,
+                      const std::string_view ciphertext)
+    -> std::optional<std::string> {
+  const auto *const internal{key.internal()};
+  if (internal == nullptr || internal->kind != PrivateKey::Type::RSA ||
+      internal->rsa_pss_restricted) {
+    return std::nullopt;
+  }
+
+  // OpenSSL reads the ciphertext as a raw integer, so a shorter input with a
+  // dropped leading zero would decrypt as the same value. Require the exact
+  // modulus size so that a malformed ciphertext is rejected
+  if (ciphertext.size() !=
+      static_cast<std::size_t>(EVP_PKEY_get_size(internal->key))) {
+    return std::nullopt;
+  }
+
+  auto *context{EVP_PKEY_CTX_new(internal->key, nullptr)};
+  if (context == nullptr) {
+    return std::nullopt;
+  }
+
+  const auto *const input{
+      reinterpret_cast<const unsigned char *>(ciphertext.data())};
+  std::optional<std::string> result;
+  std::size_t length{0};
+  if (EVP_PKEY_decrypt_init(context) == 1 && configure(context, hash) &&
+      EVP_PKEY_decrypt(context, nullptr, &length, input, ciphertext.size()) ==
+          1) {
+    std::string plaintext(length, '\x00');
+    // A failed OAEP check surfaces as a non-positive return here
+    if (EVP_PKEY_decrypt(context,
+                         reinterpret_cast<unsigned char *>(plaintext.data()),
+                         &length, input, ciphertext.size()) == 1) {
+      plaintext.resize(length);
+      result = std::move(plaintext);
+    }
+  }
+
+  EVP_PKEY_CTX_free(context);
+  return result;
+}
+
+} // namespace sourcemeta::core