Merge pull request #1337 from pmconrad/performance_test

Add performance test

tests/common/database_fixture.cpp

@@ -127,6 +127,7 @@ database_fixture::database_fixture()
    }
 
    auto test_name = boost::unit_test::framework::current_test_case().p_name.value;
+   auto test_suite_id = boost::unit_test::framework::current_test_case().p_parent_id;
    if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite") {
       auto esplugin = app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>();
       esplugin->plugin_set_app(&app);
@@ -140,7 +141,8 @@ database_fixture::database_fixture()
       esplugin->plugin_initialize(options);
       esplugin->plugin_startup();
    }
-   else {
+   else if( boost::unit_test::framework::get<boost::unit_test::test_suite>(test_suite_id).p_name.value != "performance_tests" )
+   {
       auto ahplugin = app.register_plugin<graphene::account_history::account_history_plugin>();
       ahplugin->plugin_set_app(&app);
       ahplugin->plugin_initialize(options);

tests/performance/README.md

@@ -0,0 +1,40 @@
+HOW TO
+======
+
+This small test suite serves to demonstrate two key points about the performance
+of our current implementation. The subject was talked about in detail at
+BitFest Amsterdam, Sep 22, 2018.
+
+The original description of the 100,000 transactions per second test can be
+found at https://bitshares.org/blog/2015/06/08/measuring-performance/ .
+
+Prepare
+-------
+
+1. Follow the build instructions in the top-level README file.
+2. Instead of running ``make`` you can run ``make performance_test`` to build
+   only the test suite.
+3. Run ``tests/performance_test -t performance_tests/<testcase>``
+
+
+100k TX/s
+---------
+
+``tests/performance_test -t performance_tests/one_hundred_k_benchmark``
+
+This test will create 200,000 accounts, make two transfers from each account,
+then create an asset and issue tokens to each account, for a total of one
+million operations.
+
+Different operation types have different execution times, but on fairly modern
+off-the-shelf hardware an average of 100,000 transactions per second should be
+achieved.
+
+Signature verification
+----------------------
+
+``tests/performance_test -t performance_tests/sigcheck_benchmark``
+
+This suite pre-creates 100,000 signatures and then measures how long it takes
+to verify them. Results vary depending on CPU type and clockspeed, but should be
+somewhere between 5,000 and 20,000 per second.

tests/performance/performance_tests.cpp

@@ -32,47 +32,183 @@
 #include <graphene/db/simple_index.hpp>
 
 #include <fc/crypto/digest.hpp>
+
 #include "../common/database_fixture.hpp"
+#include <cstdlib>
+#include <iostream>
 
 using namespace graphene::chain;
 
-//BOOST_FIXTURE_TEST_SUITE( performance_tests, database_fixture )
+BOOST_FIXTURE_TEST_SUITE( performance_tests, database_fixture )
 
 BOOST_AUTO_TEST_CASE( sigcheck_benchmark )
 {
    fc::ecc::private_key nathan_key = fc::ecc::private_key::generate();
    auto digest = fc::sha256::hash("hello");
    auto sig = nathan_key.sign_compact( digest );
    auto start = fc::time_point::now();
-   for( uint32_t i = 0; i < 100000; ++i )
-      auto pub = fc::ecc::public_key( sig, digest );
+   const uint64_t cycles = 100000;
+   for( uint32_t i = 0; i < cycles; ++i )
+      fc::ecc::public_key( sig, digest );
    auto end = fc::time_point::now();
    auto elapsed = end-start;
-   wdump( ((100000.0*1000000.0) / elapsed.count()) );
+   wlog( "Benchmark: verify ${sps} signatures/s", ("sps",(cycles*1000000)/elapsed.count()) );
 }
-/*
-BOOST_AUTO_TEST_CASE( transfer_benchmark )
-{
-   fc::ecc::private_key nathan_key = fc::ecc::private_key::generate();
-   const key_object& key = register_key(nathan_key.get_public_key());
+
+// See https://bitshares.org/blog/2015/06/08/measuring-performance/
+// (note this is not the original test mentioned in the above post, but was
+//  recreated later according to the description)
+BOOST_AUTO_TEST_CASE( one_hundred_k_benchmark )
+{ try {
+   ACTORS( (alice) );
+   fund( alice, asset(10000000) );
+   db._undo_db.disable(); // Blog post mentions replay, this implies no undo
+
+   const fc::ecc::private_key nathan_key = fc::ecc::private_key::generate();
+   const fc::ecc::public_key  nathan_pub = nathan_key.get_public_key();;
    const auto& committee_account = account_id_type()(db);
-   auto start = fc::time_point::now();
-   for( uint32_t i = 0; i < 1000*1000; ++i )
+
+   const uint64_t cycles = 200000;
+   uint64_t total_time = 0;
+   uint64_t total_count = 0;
+   std::vector<account_id_type> accounts;
+   accounts.reserve( cycles+1 );
+   std::vector<asset_id_type> assets;
+   assets.reserve( cycles );
+
+   std::vector<signed_transaction> transactions;
+   transactions.reserve( cycles );
+
    {
-      const auto& a = create_account("a"+fc::to_string(i), key.id);
-      transfer( committee_account, a, asset(1000) );
+      account_create_operation aco;
+      aco.name = "a1";
+      aco.registrar = committee_account.id;
+      aco.owner = authority( 1, public_key_type(nathan_pub), 1 );
+      aco.active = authority( 1, public_key_type(nathan_pub), 1 );
+      aco.options.memo_key = nathan_pub;
+      aco.options.voting_account = GRAPHENE_PROXY_TO_SELF_ACCOUNT;
+      aco.options.num_committee = 0;
+      aco.options.num_witness = 0;
+      aco.fee = db.current_fee_schedule().calculate_fee( aco );
+      trx.clear();
+      test::set_expiration( db, trx );
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         aco.name = "a" + fc::to_string(i);
+         trx.operations.push_back( aco );
+         transactions.push_back( trx );
+         trx.operations.clear();
+         ++total_count;
+      }
+
+      auto start = fc::time_point::now();
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         auto result = db.apply_transaction( transactions[i], ~0 );
+         accounts[i] = result.operation_results[0].get<object_id_type>();
+      }
+      auto end = fc::time_point::now();
+      auto elapsed = end - start;
+      total_time += elapsed.count();
+      wlog( "Create ${aps} accounts/s over ${total}ms",
+            ("aps",(cycles*1000000)/elapsed.count())("total",elapsed.count()/1000) );
    }
-   auto end = fc::time_point::now();
-   auto elapsed = end - start;
-   wdump( (elapsed) );
-}
-*/
 
-//BOOST_AUTO_TEST_SUITE_END()
+   {
+      accounts[cycles] = accounts[0];
+      transfer_operation to1;
+      to1.from = committee_account.id;
+      to1.amount = asset( 1000000 );
+      to1.fee = asset( 10 );
+      transfer_operation to2;
+      to2.amount = asset( 100 );
+      to2.fee = asset( 10 );
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         to1.to = accounts[i];
+         to2.from = accounts[i];
+         to2.to = accounts[i+1];
+         trx.operations.push_back( to1 );
+         ++total_count;
+         trx.operations.push_back( to2 );
+         ++total_count;
+         transactions[i] = trx;
+         trx.operations.clear();
+      }
+
+      auto start = fc::time_point::now();
+      for( uint32_t i = 0; i < cycles; ++i )
+         db.apply_transaction( transactions[i], ~0 );
+      auto end = fc::time_point::now();
+      auto elapsed = end - start;
+      total_time += elapsed.count();
+      wlog( "${aps} transfers/s over ${total}ms",
+            ("aps",(2*cycles*1000000)/elapsed.count())("total",elapsed.count()/1000) );
+      trx.clear();
+   }
+
+   {
+      asset_create_operation aco;
+      aco.fee = asset( 100000 );
+      aco.common_options.core_exchange_rate = asset( 1 ) / asset( 1, asset_id_type(1) );
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         aco.issuer = accounts[i];
+         aco.symbol = "ASSET" + fc::to_string( i );
+         trx.operations.push_back( aco );
+         ++total_count;
+         transactions[i] = trx;
+         trx.operations.clear();
+      }
+
+      auto start = fc::time_point::now();
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         auto result = db.apply_transaction( transactions[i], ~0 );
+         assets[i] = result.operation_results[0].get<object_id_type>();
+      }
+      auto end = fc::time_point::now();
+      auto elapsed = end - start;
+      total_time += elapsed.count();
+      wlog( "${aps} asset create/s over ${total}ms",
+            ("aps",(cycles*1000000)/elapsed.count())("total",elapsed.count()/1000) );
+      trx.clear();
+   }
+
+   {
+      asset_issue_operation aio;
+      aio.fee = asset( 10 );
+      for( uint32_t i = 0; i < cycles; ++i )
+      {
+         aio.issuer = accounts[i];
+         aio.issue_to_account = accounts[i+1];
+         aio.asset_to_issue = asset( 10, assets[i] );
+         trx.operations.push_back( aio );
+         ++total_count;
+         transactions[i] = trx;
+         trx.operations.clear();
+      }
+
+      auto start = fc::time_point::now();
+      for( uint32_t i = 0; i < cycles; ++i )
+         db.apply_transaction( transactions[i], ~0 );
+      auto end = fc::time_point::now();
+      auto elapsed = end - start;
+      total_time += elapsed.count();
+      wlog( "${aps} issuances/s over ${total}ms",
+            ("aps",(cycles*1000000)/elapsed.count())("total",elapsed.count()/1000) );
+      trx.clear();
+   }
+
+   wlog( "${total} operations in ${total_time}ms => ${avg} ops/s on average",
+         ("total",total_count)("total_time",total_time/1000)
+         ("avg",(total_count*1000000)/total_time) );
+
+   db._undo_db.enable();
+} FC_LOG_AND_RETHROW() }
+
+BOOST_AUTO_TEST_SUITE_END()
 
-//#define BOOST_TEST_MODULE "C++ Unit Tests for Graphene Blockchain Database"
-#include <cstdlib>
-#include <iostream>
 #include <boost/test/included/unit_test.hpp>
 
 boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) {