Merge pull request stellar#2180 from vogel/do-not-run-all-tests-on-tr…

…avis

Do not run all tests on travis

Reviewed-by: MonsieurNicolas

CONTRIBUTING.md

@@ -134,7 +134,9 @@ run one test category with:
 
 Categories (or tags) can be combined: AND-ed (by juxtaposition) or OR-ed (by comma-listing).
 
-Tests tagged as [.] or [hide] are not part of the default test test.
+Tests tagged as [.] or [hide] are not part of the default test.
+
+Tests tagged as [acceptance] are not part of `make check` test runs.
 
 supported test options can be seen with
   `src/stellar-core test --help`

src/herder/test/HerderTests.cpp

@@ -31,7 +31,7 @@
 using namespace stellar;
 using namespace stellar::txtest;
 
-TEST_CASE("standalone", "[herder]")
+TEST_CASE("standalone", "[herder][acceptance]")
 {
     SIMULATION_CREATE_NODE(0);
 
@@ -1169,7 +1169,7 @@ testSCPDriver(uint32 protocolVersion, uint32_t maxTxSize, size_t expectedOps,
     }
 }
 
-TEST_CASE("SCP Driver", "[herder]")
+TEST_CASE("SCP Driver", "[herder][acceptance]")
 {
     SECTION("protocol 10")
     {
@@ -1182,7 +1182,7 @@ TEST_CASE("SCP Driver", "[herder]")
     }
 }
 
-TEST_CASE("SCP State", "[herder]")
+TEST_CASE("SCP State", "[herder][acceptance]")
 {
     SecretKey nodeKeys[3];
     PublicKey nodeIDs[3];
@@ -1460,7 +1460,7 @@ TEST_CASE("quick restart", "[herder][quickRestart]")
     simulation->stopAllNodes();
 }
 
-TEST_CASE("In quorum filtering", "[quorum][herder]")
+TEST_CASE("In quorum filtering", "[quorum][herder][acceptance]")
 {
     auto mode = Simulation::OVER_LOOPBACK;
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);

src/herder/test/QuorumTrackerTests.cpp

@@ -13,7 +13,7 @@
 
 using namespace stellar;
 
-TEST_CASE("quorum tracker", "[quorum][herder]")
+TEST_CASE("quorum tracker", "[quorum][herder][acceptance]")
 {
     Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
 

src/herder/test/UpgradesTests.cpp

@@ -1720,7 +1720,7 @@ TEST_CASE("upgrade base reserve", "[upgrades]")
     }
 }
 
-TEST_CASE("simulate upgrades", "[herder][upgrades]")
+TEST_CASE("simulate upgrades", "[herder][upgrades][acceptance]")
 {
     // no upgrade is done
     auto noUpgrade =

src/history/test/HistoryTests.cpp

@@ -98,8 +98,7 @@ TEST_CASE("HistoryArchiveState get_put", "[history]")
     REQUIRE(has2.currentLedger == 0x1234);
 }
 
-TEST_CASE("History bucket verification",
-          "[history][bucketverification][batching]")
+TEST_CASE("History bucket verification", "[history][catchup]")
 {
     /* Tests bucket verification stage of catchup. Assumes ledger chain
      * verification was successful. **/
@@ -283,7 +282,7 @@ TEST_CASE("Ledger chain verification", "[ledgerheaderverification]")
     }
 }
 
-TEST_CASE("History publish", "[history]")
+TEST_CASE("History publish", "[history][publish]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(1);
@@ -322,7 +321,7 @@ dbModeName(Config::TestDbMode mode)
     }
 }
 
-TEST_CASE("History catchup", "[history][catchup]")
+TEST_CASE("History catchup", "[history][catchup][acceptance]")
 {
     // needs REAL_TIME here, as prepare-snapshot works will fail for one of the
     // sections again and again - as it is set to RETRY_FOREVER it can generate
@@ -413,7 +412,8 @@ TEST_CASE("History catchup", "[history][catchup]")
     }
 }
 
-TEST_CASE("History catchup with different modes", "[history][catchup]")
+TEST_CASE("History catchup with different modes",
+          "[history][catchup][acceptance]")
 {
     CatchupSimulation catchupSimulation{};
 
@@ -446,7 +446,7 @@ TEST_CASE("History catchup with different modes", "[history][catchup]")
     }
 }
 
-TEST_CASE("History prefix catchup", "[history][catchup][prefixcatchup]")
+TEST_CASE("History prefix catchup", "[history][catchup]")
 {
     CatchupSimulation catchupSimulation{};
 
@@ -482,7 +482,7 @@ TEST_CASE("History prefix catchup", "[history][catchup][prefixcatchup]")
 }
 
 TEST_CASE("Catchup non-initentry buckets to initentry-supporting works",
-          "[history][historyinitentry]")
+          "[history][bucket][acceptance]")
 {
     uint32_t newProto =
         Bucket::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY;
@@ -564,7 +564,7 @@ TEST_CASE("Catchup non-initentry buckets to initentry-supporting works",
 }
 
 TEST_CASE("Publish catchup alternation with stall",
-          "[history][catchup][catchupalternation]")
+          "[history][catchup][acceptance]")
 {
     CatchupSimulation catchupSimulation{};
     auto& lm = catchupSimulation.getApp().getLedgerManager();
@@ -621,8 +621,7 @@ TEST_CASE("Publish catchup alternation with stall",
     REQUIRE(catchupSimulation.catchupOnline(minimalApp, targetLedger, 5));
 }
 
-TEST_CASE("Repair missing buckets via history",
-          "[history][historybucketrepair]")
+TEST_CASE("Repair missing buckets via history", "[history][bucket]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(1);
@@ -657,7 +656,7 @@ TEST_CASE("Repair missing buckets via history",
     REQUIRE(hash1 == hash2);
 }
 
-TEST_CASE("Repair missing buckets fails", "[history][historybucketrepair]")
+TEST_CASE("Repair missing buckets fails", "[history][bucket]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(1);
@@ -754,7 +753,7 @@ TEST_CASE("HAS in publish queue is resolved", "[history]")
     REQUIRE(has.allBuckets() == pqb);
 }
 
-TEST_CASE("persist publish queue", "[history]")
+TEST_CASE("persist publish queue", "[history][publish][acceptance]")
 {
     Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
     cfg.MAX_CONCURRENT_SUBPROCESSES = 0;
@@ -824,7 +823,7 @@ TEST_CASE("persist publish queue", "[history]")
 // The idea with this test is that we join a network and somehow get a gap
 // in the SCP voting sequence while we're trying to catchup. This will let
 // system catchup just before the gap.
-TEST_CASE("catchup with a gap", "[history][catchup][catchupstall]")
+TEST_CASE("catchup with a gap", "[history][catchup][acceptance]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(1);
@@ -860,7 +859,7 @@ TEST_CASE("catchup with a gap", "[history][catchup][catchupstall]")
  * Test a variety of orderings of CATCHUP_RECENT mode, to shake out boundary
  * cases.
  */
-TEST_CASE("Catchup recent", "[history][catchup][catchuprecent][!hide]")
+TEST_CASE("Catchup recent", "[history][catchup][acceptance]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(3);
@@ -910,7 +909,7 @@ TEST_CASE("Catchup recent", "[history][catchup][catchuprecent][!hide]")
 /*
  * Test a variety of LCL/initLedger/count modes.
  */
-TEST_CASE("Catchup manual", "[history][catchup][catchupmanual]")
+TEST_CASE("Catchup manual", "[history][catchup][acceptance]")
 {
     CatchupSimulation catchupSimulation{};
     auto checkpointLedger = catchupSimulation.getLastCheckpointLedger(6);

src/history/test/SerializeTests.cpp

@@ -10,7 +10,7 @@
 
 using namespace stellar;
 
-TEST_CASE("Serialization round trip", "[history][serialize]")
+TEST_CASE("Serialization round trip", "[history]")
 {
     std::vector<std::string> testFiles = {
         "stellar-history.testnet.6714239.json",

src/invariant/test/BucketListIsConsistentWithDatabaseTests.cpp

@@ -559,7 +559,7 @@ TEST_CASE("BucketListIsConsistentWithDatabase test root account",
 }
 
 TEST_CASE("BucketListIsConsistentWithDatabase added entries",
-          "[invariant][bucketlistconsistent]")
+          "[invariant][bucketlistconsistent][acceptance]")
 {
     for (size_t nTests = 0; nTests < 40; ++nTests)
     {
@@ -577,7 +577,7 @@ TEST_CASE("BucketListIsConsistentWithDatabase added entries",
 }
 
 TEST_CASE("BucketListIsConsistentWithDatabase deleted entries",
-          "[invariant][bucketlistconsistent]")
+          "[invariant][bucketlistconsistent][acceptance]")
 {
     for (auto t : xdr::xdr_traits<LedgerEntryType>::enum_values())
     {
@@ -599,7 +599,7 @@ TEST_CASE("BucketListIsConsistentWithDatabase deleted entries",
 }
 
 TEST_CASE("BucketListIsConsistentWithDatabase modified entries",
-          "[invariant][bucketlistconsistent]")
+          "[invariant][bucketlistconsistent][acceptance]")
 {
     for (auto t : xdr::xdr_traits<LedgerEntryType>::enum_values())
     {
@@ -621,7 +621,7 @@ TEST_CASE("BucketListIsConsistentWithDatabase modified entries",
 }
 
 TEST_CASE("BucketListIsConsistentWithDatabase bucket bounds",
-          "[invariant][bucketlistconsistent]")
+          "[invariant][bucketlistconsistent][acceptance]")
 {
     struct LastModifiedBucketListGenerator : public BucketListGenerator
     {
@@ -703,7 +703,7 @@ TEST_CASE("BucketListIsConsistentWithDatabase bucket bounds",
 }
 
 TEST_CASE("BucketListIsConsistentWithDatabase merged LIVEENTRY and DEADENTRY",
-          "[invariant][bucketlistconsistent]")
+          "[invariant][bucketlistconsistent][acceptance]")
 {
     struct MergeBucketListGenerator : public SelectBucketListGenerator
     {

src/overlay/test/FloodTests.cpp

@@ -26,7 +26,7 @@ namespace stellar
 {
 using namespace txtest;
 
-TEST_CASE("Flooding", "[flood][overlay]")
+TEST_CASE("Flooding", "[flood][overlay][acceptance]")
 {
     Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     Simulation::pointer simulation;

src/overlay/test/OverlayTests.cpp

@@ -959,7 +959,7 @@ TEST_CASE("reject peers with the same nodeid", "[overlay][connections]")
     }
 }
 
-TEST_CASE("connecting to saturated nodes", "[overlay][connections]")
+TEST_CASE("connecting to saturated nodes", "[overlay][connections][acceptance]")
 {
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     auto simulation =
@@ -1040,7 +1040,8 @@ TEST_CASE("connecting to saturated nodes", "[overlay][connections]")
         std::chrono::seconds{15}, true);
 }
 
-TEST_CASE("inbounds nodes can be promoted to ouboundvalid", "[overlay]")
+TEST_CASE("inbounds nodes can be promoted to ouboundvalid",
+          "[overlay][acceptance]")
 {
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     auto simulation =
@@ -1195,7 +1196,8 @@ TEST_CASE("database is purged at overlay start", "[overlay]")
     REQUIRE(!peerManager.load(localhost(5)).second);
 }
 
-TEST_CASE("peer numfailures resets after good connection", "[overlay]")
+TEST_CASE("peer numfailures resets after good connection",
+          "[overlay][acceptance]")
 {
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     auto simulation =
@@ -1231,7 +1233,8 @@ TEST_CASE("peer numfailures resets after good connection", "[overlay]")
     REQUIRE(r.first.mNumFailures == 0);
 }
 
-TEST_CASE("peer is purged from database after few failures", "[overlay]")
+TEST_CASE("peer is purged from database after few failures",
+          "[overlay][acceptance]")
 {
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     auto simulation =

src/overlay/test/TCPPeerTests.cpp

@@ -17,7 +17,7 @@
 namespace stellar
 {
 
-TEST_CASE("TCPPeer can communicate", "[overlay]")
+TEST_CASE("TCPPeer can communicate", "[overlay][acceptance]")
 {
     Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     Simulation::pointer s =

src/simulation/CoreTests.cpp

@@ -51,7 +51,7 @@ printStats(int& nLedgers, std::chrono::system_clock::time_point tBegin,
     LOG(INFO) << sim->metricsSummary("scp");
 }
 
-TEST_CASE("3 nodes 2 running threshold 2", "[simulation][core3]")
+TEST_CASE("3 nodes 2 running threshold 2", "[simulation][core3][acceptance]")
 {
     Simulation::Mode mode = Simulation::OVER_LOOPBACK;
     SECTION("Over loopback")
@@ -104,7 +104,8 @@ TEST_CASE("3 nodes 2 running threshold 2", "[simulation][core3]")
     LOG(DEBUG) << "done with core3 test";
 }
 
-TEST_CASE("core topology 4 ledgers at scales 2 to 4", "[simulation]")
+TEST_CASE("core topology 4 ledgers at scales 2 to 4",
+          "[simulation][acceptance]")
 {
     Simulation::Mode mode = Simulation::OVER_LOOPBACK;
     SECTION("Over loopback")
@@ -245,7 +246,7 @@ hierarchicalTopoTest(int nLedgers, int nBranches, Simulation::Mode mode,
     REQUIRE(sim->haveAllExternalized(nLedgers + 1, 5));
 }
 
-TEST_CASE("hierarchical topology scales 1 to 3", "[simulation]")
+TEST_CASE("hierarchical topology scales 1 to 3", "[simulation][acceptance]")
 {
     Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     Simulation::Mode mode = Simulation::OVER_LOOPBACK;
@@ -289,7 +290,7 @@ hierarchicalSimplifiedTest(int nLedgers, int nbCore, int nbOuterNodes,
     REQUIRE(sim->haveAllExternalized(nLedgers + 1, 3));
 }
 
-TEST_CASE("core nodes with outer nodes", "[simulation]")
+TEST_CASE("core nodes with outer nodes", "[simulation][acceptance]")
 {
     Hash networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
     Simulation::Mode mode = Simulation::OVER_LOOPBACK;

src/test/selftest-parallel

@@ -13,7 +13,7 @@ if [[ -z "$RUN_PARTITIONS" ]]; then
     RUN_PARTITIONS=$(seq 0 $((NUM_PARTITIONS-1)));
 fi
 if [[ -z "$TEST_SPEC" ]]; then
-    TEST_SPEC="~[.]" # All non-hidden tests by default
+    TEST_SPEC="~[acceptance]~[.]" # All non-hidden non-acceptance tests by default
 fi
 if [[ -z "$BATCHSIZE" ]]; then
     # number of tests to run per run per job slot