Skip to content

Commit

Permalink
code change for debugging corruption
Browse files Browse the repository at this point in the history
  • Loading branch information
@kakaiu
kakaiu committed Nov 4, 2023
1 parent 361af9e commit b6b621e
Show file tree
Hide file tree
Showing 10 changed files with 286 additions and 113 deletions.
229 changes: 128 additions & 101 deletions fdbclient/BlobCipher.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
#endif

#define BLOB_CIPHER_DEBUG DEBUG_ENCRYPT_KEY_CIPHER
#define DEBUG_WITH_MUTATION_TRACKING 1

namespace {
void validateEncryptHeaderFlagVersion(const int flagsVersion) {
Expand Down Expand Up @@ -1114,20 +1115,24 @@ StringRef EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plaintext,
StringRef encryptBuf = makeString(plaintextLen, arena);
uint8_t* ciphertext = mutateString(encryptBuf);

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding is not needed for AES CTR mode, so EncryptUpdate() should encrypt all the data at once.
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding is not needed for AES CTR mode, so EncryptUpdate() should encrypt all the data at once.
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
} else {
memcpy(ciphertext, plaintext, plaintextLen);
}

// EVP_CIPHER_CTX_reset(ctx) is called after EncryptUpdate() to make sure the same encryptor
Expand Down Expand Up @@ -1160,20 +1165,22 @@ void EncryptBlobCipherAes265Ctr::encryptInplace(uint8_t* plaintext,
startTime = timer_monotonic();
}

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
}

// Ensure encryption header authToken details sanity
Expand Down Expand Up @@ -1208,19 +1215,23 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
Reference<EncryptBuf> encryptBuf = makeReference<EncryptBuf>(plaintextLen, arena);
uint8_t* ciphertext = encryptBuf->begin();

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
} else {
memcpy(ciphertext, plaintext, plaintextLen);
}

updateEncryptHeader(ciphertext, plaintextLen, header);
Expand Down Expand Up @@ -1253,20 +1264,22 @@ void EncryptBlobCipherAes265Ctr::encryptInplace(uint8_t* plaintext,

memset(reinterpret_cast<uint8_t*>(header), 0, sizeof(BlobCipherEncryptHeader));

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
}

updateEncryptHeader(plaintext, plaintextLen, header);
Expand Down Expand Up @@ -1452,19 +1465,24 @@ StringRef DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphertext,
StringRef decrypted = makeString(ciphertextLen, arena);

uint8_t* plaintext = mutateString(decrypted);
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
} else {
memcpy(plaintext, ciphertext, ciphertextLen);
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down Expand Up @@ -1580,19 +1598,24 @@ Reference<EncryptBuf> DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphert
}

uint8_t* plaintext = decrypted->begin();
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
} else {
memcpy(plaintext, ciphertext, ciphertextLen);
}

decrypted->setLogicalSize(ciphertextLen);
Expand Down Expand Up @@ -1635,20 +1658,22 @@ void DecryptBlobCipherAes256Ctr::decryptInplace(uint8_t* ciphertext,
ASSERT(authTokensValidationDone);
}

int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down Expand Up @@ -1677,20 +1702,22 @@ void DecryptBlobCipherAes256Ctr::decryptInplace(uint8_t* ciphertext,
EncryptAuthTokenAlgo authTokenAlgo;
validateEncryptHeader(ciphertext, ciphertextLen, headerRef, &authTokenMode, &authTokenAlgo);

int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down
7 changes: 7 additions & 0 deletions fdbserver/ConsistencyScan.actor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,13 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
matchingKVPairs++;
else {
valueMismatchKey = currentKV.key;
TraceEvent("ValueMismatchDetail")
.detail("Key", currentKV.key)
.detail("ValueCurrent", currentKV.value)
.detail("ValueReference", referenceKV.value)
.detail("CurrentServer", (*storageServerInterfaces)[j].id())
.detail("ReferenceServer",
(*storageServerInterfaces)[firstValidServer->get()].id());
valueMismatches++;
}

Expand Down

0 comments on commit b6b621e

Please sign in to comment.