From 7eb0a196518d2553ac69716e8a35fd676d14c231 Mon Sep 17 00:00:00 2001 From: ghubstan <36207203+ghubstan@users.noreply.github.com> Date: Thu, 2 Sep 2021 15:20:32 -0300 Subject: [PATCH 1/3] Delete generated rate metering config files in cleanDaoSetup task --- apitest/dao-setup.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apitest/dao-setup.gradle b/apitest/dao-setup.gradle index 5f55ce72e5c..12b9e7188e8 100644 --- a/apitest/dao-setup.gradle +++ b/apitest/dao-setup.gradle @@ -79,5 +79,10 @@ task cleanDaoSetup { delete file(buildResourcesDir + '/bisq-BTC_REGTEST_Arb_dao') delete file(buildResourcesDir + '/bisq-BTC_REGTEST_Alice_dao') delete file(buildResourcesDir + '/bisq-BTC_REGTEST_Bob_dao') + + def mainResourcesDir = layout.projectDirectory.dir('src/main/resources').asFile.path + println "Deleting test call rate metering config files in src main resources dir $mainResourcesDir ..." + delete file(mainResourcesDir + '/dao-setup/bisq-BTC_REGTEST_Alice_dao/ratemeters.json') + delete file(mainResourcesDir + '/dao-setup/bisq-BTC_REGTEST_Bob_dao/ratemeters.json') } } From 4a69b89dc764b0d5e549d9d486142e771a3ef8ec Mon Sep 17 00:00:00 2001 From: ghubstan <36207203+ghubstan@users.noreply.github.com> Date: Thu, 2 Sep 2021 15:26:27 -0300 Subject: [PATCH 2/3] Put ratemeters.json in resources/data dirs before they are copied to build dirs This fixes an inconsistency in the scaffold setup. --- .../src/main/java/bisq/apitest/Scaffold.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/apitest/src/main/java/bisq/apitest/Scaffold.java b/apitest/src/main/java/bisq/apitest/Scaffold.java index d518c0bcb04..df3a8f0ddcd 100644 --- a/apitest/src/main/java/bisq/apitest/Scaffold.java +++ b/apitest/src/main/java/bisq/apitest/Scaffold.java @@ -191,7 +191,7 @@ private Optional shutDownAll(SetupTask[] orderedTasks) { MILLISECONDS.sleep(1000); if (p.hasShutdownExceptions()) { // We log shutdown exceptions, but do not throw any from here - // because all of the background instances must be shut down. + // because all the background instances must be shut down. p.logExceptions(p.getShutdownExceptions(), log); // We cache only the 1st shutdown exception and move on to the @@ -227,27 +227,28 @@ public void installDaoSetupDirectories() { throw new IllegalStateException("Could not install bitcoin regtest dir"); String aliceDataDir = daoSetupDir + "/" + alicedaemon.appName; + if (!config.callRateMeteringConfigPath.isEmpty()) { + installCallRateMeteringConfiguration(aliceDataDir); + } BashCommand copyAliceDataDir = new BashCommand( "cp -rf " + aliceDataDir + " " + config.rootAppDataDir); if (copyAliceDataDir.run().getExitStatus() != 0) throw new IllegalStateException("Could not install alice data dir"); String bobDataDir = daoSetupDir + "/" + bobdaemon.appName; + if (!config.callRateMeteringConfigPath.isEmpty()) { + installCallRateMeteringConfiguration(bobDataDir); + } BashCommand copyBobDataDir = new BashCommand( "cp -rf " + bobDataDir + " " + config.rootAppDataDir); if (copyBobDataDir.run().getExitStatus() != 0) throw new IllegalStateException("Could not install bob data dir"); - log.info("Installed dao-setup files into {}", buildDataDir); - - if (!config.callRateMeteringConfigPath.isEmpty()) { - installCallRateMeteringConfiguration(aliceDataDir); - installCallRateMeteringConfiguration(bobDataDir); - } + log.info("Copied all dao-setup files to {}", buildDataDir); // Copy the blocknotify script from the src resources dir to the build // resources dir. Users may want to edit comment out some lines when all - // of the default block notifcation ports being will not be used (to avoid + // the default block notifcation ports being will not be used (to avoid // seeing rpc notifcation warnings in log files). installBitcoinBlocknotify(); From 8a42109a7ada0413aeed68b4146267531ee92f10 Mon Sep 17 00:00:00 2001 From: ghubstan <36207203+ghubstan@users.noreply.github.com> Date: Thu, 2 Sep 2021 15:37:50 -0300 Subject: [PATCH 3/3] Override/disable gRPC call rate meters in test harness driver Ad-hoc API testers need to be able to run the test harness without interference from the production api method call rate meters. This change overrides and disables most call rate meters when the test harness is run from the ApiTestMain driver (no jupiter tests). --- .../main/java/bisq/apitest/ApiTestMain.java | 22 ++++-- .../bisq/apitest/config/ApiTestConfig.java | 18 ++--- .../ApiTestRateMeterInterceptorConfig.java | 70 +++++++++++++++++++ .../test/java/bisq/apitest/ApiTestCase.java | 42 +---------- .../java/bisq/apitest/method/MethodTest.java | 19 +---- .../bisq/apitest/scenario/StartupTest.java | 3 +- 6 files changed, 103 insertions(+), 71 deletions(-) create mode 100644 apitest/src/main/java/bisq/apitest/config/ApiTestRateMeterInterceptorConfig.java diff --git a/apitest/src/main/java/bisq/apitest/ApiTestMain.java b/apitest/src/main/java/bisq/apitest/ApiTestMain.java index 20a40850ec4..6d50d19edd8 100644 --- a/apitest/src/main/java/bisq/apitest/ApiTestMain.java +++ b/apitest/src/main/java/bisq/apitest/ApiTestMain.java @@ -17,10 +17,15 @@ package bisq.apitest; +import java.io.File; + import lombok.extern.slf4j.Slf4j; import static bisq.apitest.Scaffold.EXIT_FAILURE; import static bisq.apitest.Scaffold.EXIT_SUCCESS; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.appendCallRateMeteringConfigPathOpt; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.getTestRateMeterInterceptorConfig; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.hasCallRateMeteringConfigPathOpt; import static java.lang.System.err; import static java.lang.System.exit; @@ -32,7 +37,7 @@ * ApiTestMain is a placeholder for the gradle build file, which requires a valid * 'mainClassName' property in the :apitest subproject configuration. * - * It does has some uses: + * It has some uses: * * It can be used to print test scaffolding options: bisq-apitest --help. * @@ -41,7 +46,7 @@ * It can be used to run the regtest/dao environment for release testing: * bisq-test --shutdownAfterTests=false * - * All method, scenario and end to end tests are found in the test sources folder. + * All method, scenario and end-to-end tests are found in the test sources folder. * * Requires bitcoind v0.19, v0.20, or v0.21. */ @@ -49,11 +54,15 @@ public class ApiTestMain { public static void main(String[] args) { - new ApiTestMain().execute(args); + if (!hasCallRateMeteringConfigPathOpt(args)) + new ApiTestMain().execute(getAppendedArgs(args)); + else + new ApiTestMain().execute(args); } - public void execute(@SuppressWarnings("unused") String[] args) { + public void execute(String[] args) { try { + log.info("Configuring test harness with options:\n\t{}", String.join("\n\t", args)); Scaffold scaffold = new Scaffold(args).setUp(); ApiTestConfig config = scaffold.config; @@ -77,4 +86,9 @@ public void execute(@SuppressWarnings("unused") String[] args) { exit(EXIT_FAILURE); } } + + private static String[] getAppendedArgs(String[] args) { + File rateMeterInterceptorConfig = getTestRateMeterInterceptorConfig(); + return appendCallRateMeteringConfigPathOpt(args, rateMeterInterceptorConfig); + } } diff --git a/apitest/src/main/java/bisq/apitest/config/ApiTestConfig.java b/apitest/src/main/java/bisq/apitest/config/ApiTestConfig.java index 44897a2da0c..b17b154171a 100644 --- a/apitest/src/main/java/bisq/apitest/config/ApiTestConfig.java +++ b/apitest/src/main/java/bisq/apitest/config/ApiTestConfig.java @@ -149,7 +149,7 @@ public ApiTestConfig(String... args) { ArgumentAcceptingOptionSpec configFileOpt = parser.accepts(CONFIG_FILE, format("Specify configuration file. " + - "Relative paths will be prefixed by %s location.", userDir)) + "Relative paths will be prefixed by %s location.", userDir)) .withRequiredArg() .ofType(String.class) .defaultsTo(DEFAULT_CONFIG_FILE_NAME); @@ -206,55 +206,55 @@ public ApiTestConfig(String... args) { ArgumentAcceptingOptionSpec runSubprojectJarsOpt = parser.accepts(RUN_SUBPROJECT_JARS, - "Run subproject build jars instead of full build jars") + "Run subproject build jars instead of full build jars") .withRequiredArg() .ofType(Boolean.class) .defaultsTo(false); ArgumentAcceptingOptionSpec bisqAppInitTimeOpt = parser.accepts(BISQ_APP_INIT_TIME, - "Amount of time (ms) to wait on a Bisq instance's initialization") + "Amount of time (ms) to wait on a Bisq instance's initialization") .withRequiredArg() .ofType(Long.class) .defaultsTo(5000L); ArgumentAcceptingOptionSpec skipTestsOpt = parser.accepts(SKIP_TESTS, - "Start apps, but skip tests") + "Start apps, but skip tests") .withRequiredArg() .ofType(Boolean.class) .defaultsTo(false); ArgumentAcceptingOptionSpec shutdownAfterTestsOpt = parser.accepts(SHUTDOWN_AFTER_TESTS, - "Terminate all processes after tests") + "Terminate all processes after tests") .withRequiredArg() .ofType(Boolean.class) .defaultsTo(true); ArgumentAcceptingOptionSpec supportingAppsOpt = parser.accepts(SUPPORTING_APPS, - "Comma delimited list of supporting apps (bitcoind,seednode,arbdaemon,...") + "Comma delimited list of supporting apps (bitcoind,seednode,arbdaemon,...") .withRequiredArg() .ofType(String.class) .defaultsTo("bitcoind,seednode,arbdaemon,alicedaemon,bobdaemon"); ArgumentAcceptingOptionSpec callRateMeteringConfigPathOpt = parser.accepts(CALL_RATE_METERING_CONFIG_PATH, - "Install a ratemeters.json file to configure call rate metering interceptors") + "Install a ratemeters.json file to configure call rate metering interceptors") .withRequiredArg() .defaultsTo(EMPTY); ArgumentAcceptingOptionSpec enableBisqDebuggingOpt = parser.accepts(ENABLE_BISQ_DEBUGGING, - "Start Bisq apps with remote debug options") + "Start Bisq apps with remote debug options") .withRequiredArg() .ofType(Boolean.class) .defaultsTo(false); ArgumentAcceptingOptionSpec registerDisputeAgentsOpt = parser.accepts(REGISTER_DISPUTE_AGENTS, - "Register dispute agents in arbitration daemon") + "Register dispute agents in arbitration daemon") .withRequiredArg() .ofType(Boolean.class) .defaultsTo(true); diff --git a/apitest/src/main/java/bisq/apitest/config/ApiTestRateMeterInterceptorConfig.java b/apitest/src/main/java/bisq/apitest/config/ApiTestRateMeterInterceptorConfig.java new file mode 100644 index 00000000000..9655f21dbca --- /dev/null +++ b/apitest/src/main/java/bisq/apitest/config/ApiTestRateMeterInterceptorConfig.java @@ -0,0 +1,70 @@ +package bisq.apitest.config; + +import java.io.File; + +import static bisq.apitest.config.ApiTestConfig.CALL_RATE_METERING_CONFIG_PATH; +import static bisq.proto.grpc.DisputeAgentsGrpc.getRegisterDisputeAgentMethod; +import static bisq.proto.grpc.GetVersionGrpc.getGetVersionMethod; +import static java.lang.System.arraycopy; +import static java.util.Arrays.stream; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + + + +import bisq.daemon.grpc.GrpcVersionService; +import bisq.daemon.grpc.interceptor.GrpcServiceRateMeteringConfig; + +public class ApiTestRateMeterInterceptorConfig { + + public static File getTestRateMeterInterceptorConfig() { + GrpcServiceRateMeteringConfig.Builder builder = new GrpcServiceRateMeteringConfig.Builder(); + builder.addCallRateMeter(GrpcVersionService.class.getSimpleName(), + getGetVersionMethod().getFullMethodName(), + 1, + SECONDS); + // Only GrpcVersionService is @VisibleForTesting, so we need to + // hardcode other grpcServiceClassName parameter values used in + // builder.addCallRateMeter(...). + builder.addCallRateMeter("GrpcDisputeAgentsService", + getRegisterDisputeAgentMethod().getFullMethodName(), + 10, // Same as default. + SECONDS); + // Define rate meters for non-existent method 'disabled', to override other grpc + // services' default rate meters -- defined in their rateMeteringInterceptor() + // methods. + String[] serviceClassNames = new String[]{ + "GrpcGetTradeStatisticsService", + "GrpcHelpService", + "GrpcOffersService", + "GrpcPaymentAccountsService", + "GrpcPriceService", + "GrpcTradesService", + "GrpcWalletsService" + }; + for (String service : serviceClassNames) { + builder.addCallRateMeter(service, "disabled", 1, MILLISECONDS); + } + File file = builder.build(); + file.deleteOnExit(); + return file; + } + + public static boolean hasCallRateMeteringConfigPathOpt(String[] args) { + return stream(args).anyMatch(a -> a.contains("--" + CALL_RATE_METERING_CONFIG_PATH)); + } + + public static String[] appendCallRateMeteringConfigPathOpt(String[] args, File rateMeterInterceptorConfig) { + String[] rateMeteringConfigPathOpt = new String[]{ + "--" + CALL_RATE_METERING_CONFIG_PATH + "=" + rateMeterInterceptorConfig.getAbsolutePath() + }; + if (args.length == 0) { + return rateMeteringConfigPathOpt; + } else { + String[] appendedOpts = new String[args.length + 1]; + arraycopy(args, 0, appendedOpts, 0, args.length); + arraycopy(rateMeteringConfigPathOpt, 0, appendedOpts, args.length, rateMeteringConfigPathOpt.length); + return appendedOpts; + } + } +} diff --git a/apitest/src/test/java/bisq/apitest/ApiTestCase.java b/apitest/src/test/java/bisq/apitest/ApiTestCase.java index 1ecd7e53c30..fb4938d4790 100644 --- a/apitest/src/test/java/bisq/apitest/ApiTestCase.java +++ b/apitest/src/test/java/bisq/apitest/ApiTestCase.java @@ -17,7 +17,6 @@ package bisq.apitest; -import java.io.File; import java.io.IOException; import java.util.concurrent.ExecutionException; @@ -29,23 +28,19 @@ import org.junit.jupiter.api.TestInfo; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.getTestRateMeterInterceptorConfig; import static bisq.apitest.config.BisqAppConfig.alicedaemon; import static bisq.apitest.config.BisqAppConfig.arbdaemon; import static bisq.apitest.config.BisqAppConfig.bobdaemon; -import static bisq.proto.grpc.DisputeAgentsGrpc.getRegisterDisputeAgentMethod; -import static bisq.proto.grpc.GetVersionGrpc.getGetVersionMethod; import static java.net.InetAddress.getLoopbackAddress; import static java.util.Arrays.stream; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; import bisq.apitest.config.ApiTestConfig; import bisq.apitest.method.BitcoinCliHelper; import bisq.cli.GrpcClient; -import bisq.daemon.grpc.GrpcVersionService; -import bisq.daemon.grpc.interceptor.GrpcServiceRateMeteringConfig; /** * Base class for all test types: 'method', 'scenario' and 'e2e'. @@ -90,7 +85,7 @@ public static void setUpScaffold(Enum... supportingApps) throws InterruptedException, ExecutionException, IOException { String[] params = new String[]{ "--supportingApps", stream(supportingApps).map(Enum::name).collect(Collectors.joining(",")), - "--callRateMeteringConfigPath", defaultRateMeterInterceptorConfig().getAbsolutePath(), + "--callRateMeteringConfigPath", getTestRateMeterInterceptorConfig().getAbsolutePath(), "--enableBisqDebugging", "false" }; setUpScaffold(params); @@ -148,37 +143,4 @@ protected final String testName(TestInfo testInfo) { ? testInfo.getTestMethod().get().getName() : "unknown test name"; } - - protected static File defaultRateMeterInterceptorConfig() { - GrpcServiceRateMeteringConfig.Builder builder = new GrpcServiceRateMeteringConfig.Builder(); - builder.addCallRateMeter(GrpcVersionService.class.getSimpleName(), - getGetVersionMethod().getFullMethodName(), - 1, - SECONDS); - // Only GrpcVersionService is @VisibleForTesting, so we need to - // hardcode other grpcServiceClassName parameter values used in - // builder.addCallRateMeter(...). - builder.addCallRateMeter("GrpcDisputeAgentsService", - getRegisterDisputeAgentMethod().getFullMethodName(), - 10, // Same as default. - SECONDS); - // Define rate meters for non-existent method 'disabled', to override other grpc - // services' default rate meters -- defined in their rateMeteringInterceptor() - // methods. - String[] serviceClassNames = new String[]{ - "GrpcGetTradeStatisticsService", - "GrpcHelpService", - "GrpcOffersService", - "GrpcPaymentAccountsService", - "GrpcPriceService", - "GrpcTradesService", - "GrpcWalletsService" - }; - for (String service : serviceClassNames) { - builder.addCallRateMeter(service, "disabled", 1, MILLISECONDS); - } - File file = builder.build(); - file.deleteOnExit(); - return file; - } } diff --git a/apitest/src/test/java/bisq/apitest/method/MethodTest.java b/apitest/src/test/java/bisq/apitest/method/MethodTest.java index 34929a88991..c2701ffc441 100644 --- a/apitest/src/test/java/bisq/apitest/method/MethodTest.java +++ b/apitest/src/test/java/bisq/apitest/method/MethodTest.java @@ -31,6 +31,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.getTestRateMeterInterceptorConfig; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.stream; import static org.junit.jupiter.api.Assertions.fail; @@ -47,15 +48,6 @@ public class MethodTest extends ApiTestCase { private static final Function[], String> toNameList = (enums) -> stream(enums).map(Enum::name).collect(Collectors.joining(",")); - public static void startSupportingApps(File callRateMeteringConfigFile, - boolean generateBtcBlock, - Enum... supportingApps) { - startSupportingApps(callRateMeteringConfigFile, - generateBtcBlock, - false, - supportingApps); - } - public static void startSupportingApps(File callRateMeteringConfigFile, boolean generateBtcBlock, boolean startSupportingAppsInDebugMode, @@ -72,19 +64,12 @@ public static void startSupportingApps(File callRateMeteringConfigFile, } } - public static void startSupportingApps(boolean generateBtcBlock, - Enum... supportingApps) { - startSupportingApps(generateBtcBlock, - false, - supportingApps); - } - public static void startSupportingApps(boolean generateBtcBlock, boolean startSupportingAppsInDebugMode, Enum... supportingApps) { try { // Disable call rate metering where there is no callRateMeteringConfigFile. - File callRateMeteringConfigFile = defaultRateMeterInterceptorConfig(); + File callRateMeteringConfigFile = getTestRateMeterInterceptorConfig(); setUpScaffold(new String[]{ "--supportingApps", toNameList.apply(supportingApps), "--callRateMeteringConfigPath", callRateMeteringConfigFile.getAbsolutePath(), diff --git a/apitest/src/test/java/bisq/apitest/scenario/StartupTest.java b/apitest/src/test/java/bisq/apitest/scenario/StartupTest.java index 8aa93675119..74df4f78a56 100644 --- a/apitest/src/test/java/bisq/apitest/scenario/StartupTest.java +++ b/apitest/src/test/java/bisq/apitest/scenario/StartupTest.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.TestMethodOrder; import static bisq.apitest.Scaffold.BitcoinCoreApp.bitcoind; +import static bisq.apitest.config.ApiTestRateMeterInterceptorConfig.getTestRateMeterInterceptorConfig; import static bisq.apitest.config.BisqAppConfig.alicedaemon; import static bisq.apitest.config.BisqAppConfig.arbdaemon; import static bisq.apitest.config.BisqAppConfig.seednode; @@ -54,7 +55,7 @@ public class StartupTest extends MethodTest { @BeforeAll public static void setUp() { try { - callRateMeteringConfigFile = defaultRateMeterInterceptorConfig(); + callRateMeteringConfigFile = getTestRateMeterInterceptorConfig(); startSupportingApps(callRateMeteringConfigFile, false, false,