Added ledger verification test cases

src/history/HistoryTests.cpp

@@ -346,6 +346,113 @@ TEST_CASE("Work batching", "[batching]")
     REQUIRE(verify->getState() == Work::WORK_SUCCESS);
 }
 
+TEST_CASE("Ledger chain verification", "[ledgerheaderverification]")
+{
+    Config cfg(getTestConfig(0));
+    VirtualClock clock;
+    auto cg = std::make_shared<TmpDirHistoryConfigurator>();
+    cg->configure(cfg, true);
+    Application::pointer app = createTestApplication(clock, cfg);
+    CHECK(app->getHistoryArchiveManager().initializeHistoryArchive("test"));
+
+    auto tmpDir = app->getTmpDirManager().tmpDir("tmp-chain-test");
+    auto& wm = app->getWorkManager();
+
+    LedgerHeaderHistoryEntry firstVerified{};
+    LedgerHeaderHistoryEntry verifiedAhead{};
+
+    uint32_t initLedger = 127;
+    LedgerRange ledgerRange{
+        initLedger,
+        initLedger + app->getHistoryManager().getCheckpointFrequency() * 10};
+    CheckpointRange checkpointRange{ledgerRange, app->getHistoryManager()};
+    auto ledgerChainGenerator = TestLedgerChainGenerator{
+        *app, app->getHistoryArchiveManager().getHistoryArchive("test"),
+        checkpointRange, tmpDir};
+
+    auto checkExpectedBehavior = [&](Work::State expectedState,
+                                     LedgerHeaderHistoryEntry lcl,
+                                     LedgerHeaderHistoryEntry last) {
+        auto lclPair = LedgerNumHashPair(lcl.header.ledgerSeq,
+                                         make_optional<Hash>(lcl.hash));
+        auto ledgerRangeEnd = LedgerNumHashPair(last.header.ledgerSeq,
+                                                make_optional<Hash>(last.hash));
+        auto w = wm.executeWork<VerifyLedgerChainWork>(tmpDir, ledgerRange,
+                                                       lclPair, ledgerRangeEnd);
+        REQUIRE(expectedState == w->getState());
+    };
+
+    LedgerHeaderHistoryEntry lcl, last;
+    SECTION("fully valid")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_OK);
+        checkExpectedBehavior(Work::WORK_SUCCESS, lcl, last);
+    }
+    SECTION("invalid link due to bad hash")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_ERR_BAD_HASH);
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("invalid ledger version")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_ERR_BAD_LEDGER_VERSION);
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("overshot")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_ERR_OVERSHOT);
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("undershot")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_ERR_UNDERSHOT);
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("missing entries")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_ERR_MISSING_ENTRIES);
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("chain does not agree with LCL")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_OK);
+        lcl.hash = HashUtils::random();
+
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("chain does not agree with LCL on checkpoint boundary")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_OK);
+        lcl.header.ledgerSeq +=
+            app->getHistoryManager().getCheckpointFrequency() - 1;
+        lcl.hash = HashUtils::random();
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("chain does not agree with LCL outside of range")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_OK);
+        lcl.header.ledgerSeq -= 1;
+        lcl.hash = HashUtils::random();
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+    SECTION("chain does not agree with trusted hash")
+    {
+        std::tie(lcl, last) = ledgerChainGenerator.makeLedgerChainFiles(
+            HistoryManager::VERIFY_STATUS_OK);
+        last.hash = HashUtils::random();
+        checkExpectedBehavior(Work::WORK_FAILURE_FATAL, lcl, last);
+    }
+}
+
 TEST_CASE("History prefix catchup", "[history][historycatchup][prefixcatchup]")
 {
     CatchupSimulation catchupSimulation{};

src/history/HistoryTestsUtils.cpp

@@ -10,9 +10,9 @@
 #include "herder/TxSetFrame.h"
 #include "history/HistoryArchiveManager.h"
 #include "ledger/CheckpointRange.h"
+#include "ledger/LedgerRange.h"
 #include "ledger/LedgerTxn.h"
 #include "ledger/LedgerTxnHeader.h"
-#include "ledger/LedgerRange.h"
 #include "lib/catch.hpp"
 #include "test/TestAccount.h"
 #include "test/TestUtils.h"
@@ -181,6 +181,90 @@ TestBucketGenerator::generateBucket(TestBucketState state)
     return binToHex(hash);
 }
 
+TestLedgerChainGenerator::TestLedgerChainGenerator(
+    Application& app, std::shared_ptr<HistoryArchive> archive,
+    CheckpointRange range, TmpDir const& tmpDir)
+    : mApp{app}, mArchive{archive}, mCheckpointRange{range}, mTmpDir{tmpDir}
+{
+}
+
+void
+TestLedgerChainGenerator::createHistoryFiles(
+    std::vector<LedgerHeaderHistoryEntry> const& lhv,
+    LedgerHeaderHistoryEntry& first, LedgerHeaderHistoryEntry& last,
+    uint32_t checkpoint)
+{
+    FileTransferInfo ft{mTmpDir, HISTORY_FILE_TYPE_LEDGER, checkpoint};
+    XDROutputFileStream ledgerOut;
+    ledgerOut.open(ft.localPath_nogz());
+
+    for (auto& ledger : lhv)
+    {
+        if (first.header.ledgerSeq == 0)
+        {
+            first = ledger;
+        }
+        REQUIRE(ledgerOut.writeOne(ledger));
+        last = ledger;
+    }
+    ledgerOut.close();
+}
+
+TestLedgerChainGenerator::CheckpointEnds
+TestLedgerChainGenerator::makeOneLedgerFile(
+    uint32_t currCheckpoint, Hash prevHash,
+    HistoryManager::LedgerVerificationStatus state)
+{
+    auto initLedger =
+        mApp.getHistoryManager().prevCheckpointLedger(currCheckpoint);
+    auto frequency = mApp.getHistoryManager().getCheckpointFrequency();
+    if (initLedger == 0)
+    {
+        initLedger = LedgerManager::GENESIS_LEDGER_SEQ;
+        frequency -= 1;
+    }
+
+    LedgerHeaderHistoryEntry first, last, lcl;
+    lcl.header.ledgerSeq = initLedger;
+    lcl.header.previousLedgerHash = prevHash;
+
+    std::vector<LedgerHeaderHistoryEntry> ledgerChain =
+        LedgerTestUtils::generateLedgerHeadersForCheckpoint(lcl, frequency,
+                                                            state);
+
+    createHistoryFiles(ledgerChain, first, last, currCheckpoint);
+    return CheckpointEnds(first, last);
+}
+
+TestLedgerChainGenerator::CheckpointEnds
+TestLedgerChainGenerator::makeLedgerChainFiles(
+    HistoryManager::LedgerVerificationStatus state)
+{
+    Hash hash = HashUtils::random();
+    LedgerHeaderHistoryEntry beginRange;
+
+    LedgerHeaderHistoryEntry first, last;
+    for (auto i = mCheckpointRange.first(); i <= mCheckpointRange.last();
+         i += mApp.getHistoryManager().getCheckpointFrequency())
+    {
+        // Only corrupt first checkpoint (last to be verified)
+        if (i != mCheckpointRange.first())
+        {
+            state = HistoryManager::VERIFY_STATUS_OK;
+        }
+
+        std::tie(first, last) = makeOneLedgerFile(i, hash, state);
+        hash = last.hash;
+
+        if (beginRange.header.ledgerSeq == 0)
+        {
+            beginRange = first;
+        }
+    }
+
+    return CheckpointEnds(beginRange, last);
+}
+
 CatchupMetrics::CatchupMetrics()
     : mHistoryArchiveStatesDownloaded{0}
     , mLedgersDownloaded{0}
@@ -538,7 +622,7 @@ CatchupSimulation::catchupApplication(uint32_t initLedger, uint32_t count,
             std::find(mLedgerSeqs.begin(), mLedgerSeqs.end(), initLedger) -
             mLedgerSeqs.begin());
         catchupConfiguration = {
-                LedgerNumHashPair(initLedger, make_optional<Hash>(hash)), count};
+            LedgerNumHashPair(initLedger, make_optional<Hash>(hash)), count};
         lm.startCatchup(catchupConfiguration, true);
     }
 

src/history/HistoryTestsUtils.h

@@ -4,20 +4,25 @@
 // under the Apache License, Version 2.0. See the COPYING file at the root
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
+#include "FileTransferInfo.h"
 #include "bucket/BucketList.h"
+#include "catchup/VerifyLedgerChainWork.h"
 #include "crypto/Hex.h"
 #include "herder/LedgerCloseData.h"
 #include "history/HistoryArchive.h"
 #include "historywork/GzipFileWork.h"
 #include "historywork/MakeRemoteDirWork.h"
 #include "historywork/PutRemoteFileWork.h"
+#include "ledger/LedgerRange.h"
 #include "ledger/LedgerTestUtils.h"
 #include "main/Application.h"
 #include "main/Config.h"
 #include "util/Timer.h"
 #include "util/TmpDir.h"
 
 #include "bucket/BucketOutputIterator.h"
+#include "ledger/CheckpointRange.h"
+#include "lib/catch.hpp"
 #include <random>
 
 namespace stellar
@@ -93,6 +98,31 @@ class TestBucketGenerator
         TestBucketState desiredState = TestBucketState::CONTENTS_AND_HASH_OK);
 };
 
+class TestLedgerChainGenerator
+{
+    Application& mApp;
+    std::shared_ptr<HistoryArchive> mArchive;
+    CheckpointRange mCheckpointRange;
+    TmpDir const& mTmpDir;
+
+  public:
+    using CheckpointEnds =
+        std::pair<LedgerHeaderHistoryEntry, LedgerHeaderHistoryEntry>;
+    TestLedgerChainGenerator(Application& app,
+                             std::shared_ptr<HistoryArchive> archive,
+                             CheckpointRange range, const TmpDir& tmpDir);
+    void createHistoryFiles(std::vector<LedgerHeaderHistoryEntry> const& lhv,
+                            LedgerHeaderHistoryEntry& first,
+                            LedgerHeaderHistoryEntry& last,
+                            uint32_t checkpoint);
+    CheckpointEnds
+    makeOneLedgerFile(uint32_t currCheckpoint, Hash prevHash,
+                      HistoryManager::LedgerVerificationStatus state);
+    CheckpointEnds
+    makeLedgerChainFiles(HistoryManager::LedgerVerificationStatus state =
+                             HistoryManager::VERIFY_STATUS_OK);
+};
+
 struct CatchupMetrics
 {
     uint64_t mHistoryArchiveStatesDownloaded;

src/ledger/LedgerTestUtils.cpp

@@ -3,11 +3,16 @@
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
 #include "LedgerTestUtils.h"
+#include "crypto/SHA.h"
 #include "crypto/SecretKey.h"
+#include "main/Config.h"
+#include "util/Logging.h"
+#include "util/Math.h"
 #include "util/types.h"
 #include <cctype>
 #include <string>
 #include <xdrpp/autocheck.h>
+#include <xdrpp/xdrpp/marshal.h>
 
 namespace stellar
 {
@@ -155,6 +160,66 @@ makeValid(DataEntry& d)
     replaceControlCharacters(d.dataName, 1);
 }
 
+void
+makeValid(std::vector<LedgerHeaderHistoryEntry>& lhv,
+          LedgerHeaderHistoryEntry firstLedger,
+          HistoryManager::LedgerVerificationStatus state)
+{
+    auto randomIndex = rand_uniform<size_t>(1, lhv.size() - 1);
+    auto prevHash = firstLedger.header.previousLedgerHash;
+    auto ledgerSeq = firstLedger.header.ledgerSeq;
+
+    for (auto i = 0; i < lhv.size(); i++)
+    {
+        auto& lh = lhv[i];
+
+        lh.header.ledgerVersion = Config::CURRENT_LEDGER_PROTOCOL_VERSION;
+        lh.header.previousLedgerHash = prevHash;
+        lh.header.ledgerSeq = ledgerSeq;
+
+        if (i == randomIndex && state != HistoryManager::VERIFY_STATUS_OK)
+        {
+            switch (state)
+            {
+            case HistoryManager::VERIFY_STATUS_ERR_BAD_LEDGER_VERSION:
+                lh.header.ledgerVersion += 1;
+                break;
+            case HistoryManager::VERIFY_STATUS_ERR_BAD_HASH:
+                lh.header.previousLedgerHash = HashUtils::random();
+                break;
+            case HistoryManager::VERIFY_STATUS_ERR_UNDERSHOT:
+                lh.header.ledgerSeq -= 1;
+                break;
+            case HistoryManager::VERIFY_STATUS_ERR_OVERSHOT:
+                lh.header.ledgerSeq += 1;
+                break;
+            default:
+                break;
+            }
+        }
+        // On a coin flip, corrupt header content rather than previous link
+        autocheck::generator<bool> flip;
+        if (i == randomIndex &&
+            state == HistoryManager::VERIFY_STATUS_ERR_BAD_HASH && flip())
+        {
+            lh.hash = HashUtils::random();
+        }
+        else
+        {
+            lh.hash = sha256(xdr::xdr_to_opaque(lh.header));
+        }
+
+        prevHash = lh.hash;
+        ledgerSeq = lh.header.ledgerSeq + 1;
+    }
+
+    if (state == HistoryManager::VERIFY_STATUS_ERR_MISSING_ENTRIES)
+    {
+        // Delete last element
+        lhv.erase(lhv.begin() + lhv.size() - 1);
+    }
+}
+
 static auto validLedgerEntryGenerator = autocheck::map(
     [](LedgerEntry&& le, size_t s) {
         auto& led = le.data;
@@ -273,5 +338,17 @@ generateValidDataEntries(size_t n)
     static auto vecgen = autocheck::list_of(validDataEntryGenerator);
     return vecgen(n);
 }
+
+std::vector<LedgerHeaderHistoryEntry>
+generateLedgerHeadersForCheckpoint(
+    LedgerHeaderHistoryEntry firstLedger, uint32_t freq,
+    HistoryManager::LedgerVerificationStatus state)
+{
+    static auto vecgen =
+        autocheck::list_of(autocheck::generator<LedgerHeaderHistoryEntry>());
+    auto res = vecgen(freq);
+    makeValid(res, firstLedger, state);
+    return res;
+}
 }
 }