Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DO NOT MERGE] Code change for debugging corruption #11055

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+286 −113
Draft

[DO NOT MERGE] Code change for debugging corruption #11055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
229 changes: 128 additions & 101 deletions fdbclient/BlobCipher.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
#endif

#define BLOB_CIPHER_DEBUG DEBUG_ENCRYPT_KEY_CIPHER
#define DEBUG_WITH_MUTATION_TRACKING 1

namespace {
void validateEncryptHeaderFlagVersion(const int flagsVersion) {
Expand Down Expand Up @@ -1114,20 +1115,24 @@ StringRef EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plaintext,
StringRef encryptBuf = makeString(plaintextLen, arena);
uint8_t* ciphertext = mutateString(encryptBuf);

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding is not needed for AES CTR mode, so EncryptUpdate() should encrypt all the data at once.
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding is not needed for AES CTR mode, so EncryptUpdate() should encrypt all the data at once.
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
} else {
memcpy(ciphertext, plaintext, plaintextLen);
}

// EVP_CIPHER_CTX_reset(ctx) is called after EncryptUpdate() to make sure the same encryptor
Expand Down Expand Up @@ -1160,20 +1165,22 @@ void EncryptBlobCipherAes265Ctr::encryptInplace(uint8_t* plaintext,
startTime = timer_monotonic();
}

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
}

// Ensure encryption header authToken details sanity
Expand Down Expand Up @@ -1208,19 +1215,23 @@ Reference<EncryptBuf> EncryptBlobCipherAes265Ctr::encrypt(const uint8_t* plainte
Reference<EncryptBuf> encryptBuf = makeReference<EncryptBuf>(plaintextLen, arena);
uint8_t* ciphertext = encryptBuf->begin();

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
} else {
memcpy(ciphertext, plaintext, plaintextLen);
}

updateEncryptHeader(ciphertext, plaintextLen, header);
Expand Down Expand Up @@ -1253,20 +1264,22 @@ void EncryptBlobCipherAes265Ctr::encryptInplace(uint8_t* plaintext,

memset(reinterpret_cast<uint8_t*>(header), 0, sizeof(BlobCipherEncryptHeader));

int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytes{ 0 };
if (EVP_EncryptUpdate(ctx, plaintext, &bytes, plaintext, plaintextLen) != 1) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so encryptUpdate() should encrypt all the data
if (bytes != plaintextLen) {
TraceEvent(SevWarn, "BlobCipherInplaceEncryptUnexpectedCipherLen")
.detail("PlaintextLen", plaintextLen)
.detail("EncryptedBufLen", bytes);
throw encrypt_ops_error();
}
}

updateEncryptHeader(plaintext, plaintextLen, header);
Expand Down Expand Up @@ -1452,19 +1465,24 @@ StringRef DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphertext,
StringRef decrypted = makeString(ciphertextLen, arena);

uint8_t* plaintext = mutateString(decrypted);
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
} else {
memcpy(plaintext, ciphertext, ciphertextLen);
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down Expand Up @@ -1580,19 +1598,24 @@ Reference<EncryptBuf> DecryptBlobCipherAes256Ctr::decrypt(const uint8_t* ciphert
}

uint8_t* plaintext = decrypted->begin();
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
} else {
memcpy(plaintext, ciphertext, ciphertextLen);
}

decrypted->setLogicalSize(ciphertextLen);
Expand Down Expand Up @@ -1635,20 +1658,22 @@ void DecryptBlobCipherAes256Ctr::decryptInplace(uint8_t* ciphertext,
ASSERT(authTokensValidationDone);
}

int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", header.cipherTextDetails.baseCipherId)
.detail("EncryptDomainId", header.cipherTextDetails.encryptDomainId);
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down Expand Up @@ -1677,20 +1702,22 @@ void DecryptBlobCipherAes256Ctr::decryptInplace(uint8_t* ciphertext,
EncryptAuthTokenAlgo authTokenAlgo;
validateEncryptHeader(ciphertext, ciphertextLen, headerRef, &authTokenMode, &authTokenAlgo);

int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}
if (!g_network->isSimulated() || !DEBUG_WITH_MUTATION_TRACKING) {
int bytesDecrypted{ 0 };
if (!EVP_DecryptUpdate(ctx, ciphertext, &bytesDecrypted, ciphertext, ciphertextLen)) {
TraceEvent(SevWarn, "BlobCipherDecryptUpdateFailed")
.detail("BaseCipherId", textCipherKey->getBaseCipherId())
.detail("EncryptDomainId", textCipherKey->getDomainId());
throw encrypt_ops_error();
}

// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
// Padding should be 0 for AES CTR mode, so DecryptUpdate() should decrypt all the data
if (bytesDecrypted != ciphertextLen) {
TraceEvent(SevWarn, "BlobCipherDecryptUnexpectedPlaintextLen")
.detail("CiphertextLen", ciphertextLen)
.detail("DecryptedBufLen", bytesDecrypted);
throw encrypt_ops_error();
}
}

if (CLIENT_KNOBS->ENABLE_ENCRYPTION_CPU_TIME_LOGGING && decryptTime) {
Expand Down
7 changes: 7 additions & 0 deletions fdbserver/ConsistencyScan.actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,13 @@ ACTOR Future<int> consistencyCheckReadData(UID myId,
matchingKVPairs++;
else {
valueMismatchKey = currentKV.key;
TraceEvent("ValueMismatchDetail")
.detail("Key", currentKV.key)
.detail("ValueCurrent", currentKV.value)
.detail("ValueReference", referenceKV.value)
.detail("CurrentServer", (*storageServerInterfaces)[j].id())
.detail("ReferenceServer",
(*storageServerInterfaces)[firstValidServer->get()].id());
valueMismatches++;
}

Expand Down