diff --git a/night_rally.sh b/night_rally.sh index e26d4c5c7a760..de34f6e2fccaf 100755 --- a/night_rally.sh +++ b/night_rally.sh @@ -37,6 +37,7 @@ MODE="nightly" RELEASE="master" # only needed for ad-hoc benchmarks REVISION="latest" +RUNTIME_JDK="8" TARGET_HOST="localhost:9200" X_PACK="" TRACKS_FILE="" @@ -71,6 +72,10 @@ case ${i} in REVISION="${i#*=}" shift # past argument=value ;; + --runtime-jdk=*) + RUNTIME_JDK="${i#*=}" + shift # past argument=value + ;; --release=*) RELEASE="${i#*=}" shift # past argument=value @@ -186,6 +191,7 @@ es-night-rally \ ${NIGHT_RALLY_DRY_RUN} \ ${SKIP_ANSIBLE_PARAM} \ --fixtures="${FIXTURES}" \ + --runtime-jdk="${RUNTIME_JDK}" \ --revision="${REVISION}" \ --release="${RELEASE}" \ --tracks="${TRACKS_FILE}" \ diff --git a/night_rally/night_rally.py b/night_rally/night_rally.py index 4b414c00735b7..e0e7e1a6f7a01 100644 --- a/night_rally/night_rally.py +++ b/night_rally/night_rally.py @@ -342,9 +342,10 @@ class StandardParams: """ Extracts all parameters that are needed for all Rally invocations. """ - def __init__(self, configuration_name, effective_start_date, user_tag): + def __init__(self, configuration_name, effective_start_date, runtime_jdk, user_tag): self.configuration_name = configuration_name self.effective_start_date = effective_start_date + self.runtime_jdk = runtime_jdk self.user_tag = user_tag def __call__(self, race_config): @@ -358,6 +359,7 @@ def __call__(self, race_config): "car": [race_config.car], "user-tag": self.format_tag(additional_tags={"name": race_config.name}) } + add_if_present(params, "runtime-jdk", self.runtime_jdk) add_if_present(params, "car-params", race_config.car_params) add_if_present(params, "track-params", race_config.track_params) add_if_present(params, "elasticsearch-plugins", race_config.plugins) @@ -680,6 +682,10 @@ def parse_args(): "--release", help="Specify release string to use for comparison reports", default="master") + parser.add_argument( + "--runtime-jdk", + help="The major version of the runtime JDK to use.", + default=None) parser.add_argument( "--tracks", help="Path to the tracks.json file that contains the tracks to run", @@ -737,7 +743,7 @@ def main(): params.append(TelemetryParams(args.telemetry, args.telemetry_params)) if release_mode: - params.append(StandardParams(env_name, start_date, release_tag)) + params.append(StandardParams(env_name, start_date, args.runtime_jdk, release_tag)) if docker_benchmark: if x_pack: raise RuntimeError("User specified x-pack configuration [%s] but this is not supported for Docker benchmarks." % x_pack) @@ -749,11 +755,11 @@ def main(): command = ReleaseCommand(params, x_pack, release) elif adhoc_mode: logger.info("Running adhoc benchmarks for revision [%s] against %s." % (args.revision, target_hosts)) - params.append(StandardParams(env_name, start_date, release_tag)) + params.append(StandardParams(env_name, start_date, args.runtime_jdk, release_tag)) command = AdHocCommand(params, args.revision) else: logger.info("Running nightly benchmarks against %s." % target_hosts) - params.append(StandardParams(env_name, start_date, release_tag)) + params.append(StandardParams(env_name, start_date, args.runtime_jdk, release_tag)) command = NightlyCommand(params, start_date) rally_failure = run_rally(tracks, target_hosts, command, args.dry_run, args.skip_ansible) diff --git a/tests/night_rally_test.py b/tests/night_rally_test.py index 9b2de3f2a2eec..5e7bcc25e058e 100644 --- a/tests/night_rally_test.py +++ b/tests/night_rally_test.py @@ -115,7 +115,7 @@ def test_run_two_challenges_successfully(self, mocked_wait_until_port_is_free): } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("nightly", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("nightly", start_date, 8, {"env": "bare"})] cmd = night_rally.NightlyCommand(params, start_date) night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) self.assertEqual(2, len(system_call.calls)) @@ -123,12 +123,13 @@ def test_run_two_challenges_successfully(self, mocked_wait_until_port_is_free): [ "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --car-params=\"verbose_iw_logging_enabled:true\" " - "--pipeline=\"from-sources-complete\" --revision=\"@2016-01-01T00:00:00Z\"", + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" " + "--car-params=\"verbose_iw_logging_enabled:true\" --pipeline=\"from-sources-complete\" " + "--revision=\"@2016-01-01T00:00:00Z\"", "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --pipeline=\"from-sources-skip-build\" " + "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --runtime-jdk=\"8\" --pipeline=\"from-sources-skip-build\" " "--revision=\"@2016-01-01T00:00:00Z\"" ] , @@ -163,7 +164,7 @@ def test_run_two_tracks_successfully(self, mocked_wait_until_port_is_free): ] start_date = datetime.datetime(2016, 10, 1) - params = [night_rally.StandardParams("nightly", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("nightly", start_date, 8, {"env": "bare"})] cmd = night_rally.NightlyCommand(params, start_date) night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) self.assertEqual(2, len(system_call.calls)) @@ -171,12 +172,12 @@ def test_run_two_tracks_successfully(self, mocked_wait_until_port_is_free): [ "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --pipeline=\"from-sources-complete\" " - "--revision=\"@2016-10-01T00:00:00Z\"", + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" " + "--pipeline=\"from-sources-complete\" --revision=\"@2016-10-01T00:00:00Z\"", "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --pipeline=\"from-sources-skip-build\" " + "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" --pipeline=\"from-sources-skip-build\" " "--revision=\"@2016-10-01T00:00:00Z\"" ] , @@ -211,7 +212,7 @@ def test_run_adhoc_benchmark(self, mocked_wait_until_port_is_free): ] start_date = datetime.datetime(2016, 10, 1) - params = [night_rally.StandardParams("lucene-7", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("lucene-7", start_date, 8, {"env": "bare"})] cmd = night_rally.AdHocCommand(params, "66202dc") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -220,12 +221,13 @@ def test_run_adhoc_benchmark(self, mocked_wait_until_port_is_free): [ "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --pipeline=\"from-sources-complete\" " - "--revision=\"66202dc\"", + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" " + "--pipeline=\"from-sources-complete\" --revision=\"66202dc\"", "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --pipeline=\"from-sources-skip-build\" --revision=\"66202dc\"" + "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" " + "--pipeline=\"from-sources-skip-build\" --revision=\"66202dc\"" ] , system_call.calls @@ -253,7 +255,7 @@ def test_run_release_benchmark_without_plugins(self, mocked_wait_until_port_is_f } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "bare"})] cmd = night_rally.ReleaseCommand(params, None, "5.3.0") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) self.assertEqual(2, len(system_call.calls)) @@ -261,12 +263,12 @@ def test_run_release_benchmark_without_plugins(self, mocked_wait_until_port_is_f [ "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --distribution-version=\"5.3.0\" " + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" --distribution-version=\"5.3.0\" " "--pipeline=\"from-distribution\"", "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --distribution-version=\"5.3.0\" " + "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --runtime-jdk=\"8\" --distribution-version=\"5.3.0\" " "--pipeline=\"from-distribution\"" ] , @@ -306,7 +308,7 @@ def test_run_release_benchmark_with_plugins(self, mocked_wait_until_port_is_free } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "x-pack"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "x-pack"})] cmd = night_rally.ReleaseCommand(params, ["security", "monitoring"], "5.3.0") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -315,15 +317,15 @@ def test_run_release_benchmark_with_plugins(self, mocked_wait_until_port_is_free [ "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:x-pack,name:geonames-defaults,x-pack:true\" --track-params=\"bulk_size:3000\" " - "--client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," + "--car=\"defaults\" --user-tag=\"env:x-pack,name:geonames-defaults,x-pack:true\" --runtime-jdk=\"8\" " + "--track-params=\"bulk_size:3000\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," "basic_auth_password:'rally-password'\" --elasticsearch-plugins=\"x-pack:security,monitoring\" " "--distribution-version=\"5.3.0\" --pipeline=\"from-distribution\"", "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:x-pack,name:geonames-4g,x-pack:true\" --track-params=\"bulk_size:2000\" " - "--client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," + "--car=\"4gheap\" --user-tag=\"env:x-pack,name:geonames-4g,x-pack:true\" --runtime-jdk=\"8\" " + "--track-params=\"bulk_size:2000\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," "basic_auth_password:'rally-password'\" --elasticsearch-plugins=\"x-pack:security,monitoring\" " "--distribution-version=\"5.3.0\" --pipeline=\"from-distribution\"", ] @@ -361,7 +363,7 @@ def test_run_release_benchmark_with_x_pack_module(self, mocked_wait_until_port_i } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "x-pack"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "x-pack"})] # 6.3.0 is the first version to include x-pack as a module cmd = night_rally.ReleaseCommand(params, ["security", "monitoring"], "6.3.0") @@ -372,13 +374,13 @@ def test_run_release_benchmark_with_x_pack_module(self, mocked_wait_until_port_i "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " "--car=\"defaults,x-pack-security,x-pack-monitoring\" --user-tag=\"env:x-pack,name:geonames-defaults,x-pack:true\" " - "--client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," + "--runtime-jdk=\"8\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," "basic_auth_password:'rally-password'\" --distribution-version=\"6.3.0\" --pipeline=\"from-distribution\"", "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " "--car=\"4gheap,x-pack-security,x-pack-monitoring\" --user-tag=\"env:x-pack,name:geonames-4g,x-pack:true\" " - "--client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," + "--runtime-jdk=\"8\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally'," "basic_auth_password:'rally-password'\" --distribution-version=\"6.3.0\" --pipeline=\"from-distribution\"", ] , @@ -403,7 +405,7 @@ def test_run_release_benchmark_with_transport_nio(self, mocked_wait_until_port_i } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "bare"})] cmd = night_rally.ReleaseCommand(params, x_pack_config=None, distribution_version="7.0.0") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -412,8 +414,9 @@ def test_run_release_benchmark_with_transport_nio(self, mocked_wait_until_port_i [ "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --elasticsearch-plugins=\"transport-nio:transport+http\" " - "--distribution-version=\"7.0.0\" --pipeline=\"from-distribution\"", + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" " + "--elasticsearch-plugins=\"transport-nio:transport+http\" --distribution-version=\"7.0.0\" " + "--pipeline=\"from-distribution\"" ] , system_call.calls @@ -437,7 +440,7 @@ def test_do_not_run_release_benchmark_with_transport_nio_and_x_pack(self, mocked } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "x-pack"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "x-pack"})] cmd = night_rally.ReleaseCommand(params, ["security", "monitoring"], "7.0.0") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -461,7 +464,7 @@ def test_do_not_run_release_benchmark_with_transport_nio_on_old_version(self, mo } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "bare"})] # 6.2.0 does not have transport-nio available cmd = night_rally.ReleaseCommand(params, x_pack_config=None, distribution_version="6.2.0") @@ -490,7 +493,7 @@ def test_run_docker_benchmark(self, mocked_wait_until_port_is_free): } ] start_date = datetime.datetime(2016, 1, 1) - params = [night_rally.StandardParams("release", start_date, {"env": "docker"})] + params = [night_rally.StandardParams("release", start_date, 8, {"env": "docker"})] cmd = night_rally.DockerCommand(params, "5.3.0") night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -499,12 +502,12 @@ def test_run_docker_benchmark(self, mocked_wait_until_port_is_free): [ "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:docker,name:geonames-defaults\" " + "--car=\"defaults\" --user-tag=\"env:docker,name:geonames-defaults\" --runtime-jdk=\"8\" " "--distribution-version=\"5.3.0\" --pipeline=\"docker\"", "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:docker,name:geonames-4g\" " + "--car=\"4gheap\" --user-tag=\"env:docker,name:geonames-4g\" --runtime-jdk=\"8\" " "--distribution-version=\"5.3.0\" --pipeline=\"docker\"" ] , @@ -540,7 +543,7 @@ def test_run_continues_on_error(self, mocked_wait_until_port_is_free): ] start_date = datetime.datetime(2016, 10, 1) - params = [night_rally.StandardParams("nightly", start_date, {"env": "bare"})] + params = [night_rally.StandardParams("nightly", start_date, 8, {"env": "bare"})] cmd = night_rally.NightlyCommand(params, start_date) night_rally.run_rally(tracks, ["localhost"], cmd, skip_ansible=True, system=system_call) @@ -549,13 +552,13 @@ def test_run_continues_on_error(self, mocked_wait_until_port_is_free): [ "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" " - "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --pipeline=\"from-sources-complete\" " - "--revision=\"@2016-10-01T00:00:00Z\"", + "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" " + "--pipeline=\"from-sources-complete\" --revision=\"@2016-10-01T00:00:00Z\"", "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" " "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" " - "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --pipeline=\"from-sources-skip-build\" " - "--revision=\"@2016-10-01T00:00:00Z\"" + "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" " + "--pipeline=\"from-sources-skip-build\" --revision=\"@2016-10-01T00:00:00Z\"" ] , system_call.calls @@ -583,7 +586,7 @@ def test_run_with_telemetry(self, mocked_wait_until_port_is_free): telemetry="jfr,gc,jit", telemetry_params="recording-template:profile" ), - night_rally.StandardParams("nightly", start_date, {"env": "bare"})] + night_rally.StandardParams("nightly", start_date, 8, {"env": "bare"})] cmd = night_rally.NightlyCommand(params, start_date) @@ -594,7 +597,7 @@ def test_run_with_telemetry(self, mocked_wait_until_port_is_free): "rally --skip-update --telemetry=\"jfr,gc,jit\" --telemetry-params=\"recording-template:profile\" " "--configuration-name=\"nightly\" --quiet --target-host=\"localhost\" --effective-start-date=\"2016-01-01 00:00:00\" " "--track=\"geonames\" --challenge=\"append-no-conflicts\" --car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\"" - " --pipeline=\"from-sources-complete\" --revision=\"@2016-01-01T00:00:00Z\"" + " --runtime-jdk=\"8\" --pipeline=\"from-sources-complete\" --revision=\"@2016-01-01T00:00:00Z\"" ] , system_call.calls