Use track-params instead of specific challenges

With this commit we get one step closer to removing the challenges
"append-no-conflicts-index-only-1-replica" by using track parameters.

Relates elastic#44

night_rally.py

@@ -59,7 +59,7 @@ def __init__(self, effective_start_date, user_tag):
     def runnable(self, track, challenge, car, plugins, target_hosts):
         return True
 
-    def command_line(self, track, challenge, car, name, plugins, target_hosts):
+    def command_line(self, track, track_params, challenge, car, name, plugins, target_hosts):
         raise NotImplementedError("abstract method")
 
     def format_tag(self, additional_tags=None):
@@ -78,7 +78,7 @@ def __init__(self, effective_start_date, revision, configuration_name, user_tag)
         self.pipeline = "from-sources-complete"
         self.pipeline_with_plugins = "from-sources-complete"
 
-    def command_line(self, track, challenge, car, name, plugins, target_hosts):
+    def command_line(self, track, track_params, challenge, car, name, plugins, target_hosts):
         cmd = RALLY_BINARY
         cmd += " --configuration-name=%s" % self.configuration_name
         cmd += " --target-host=\"%s\"" % ",".join(target_hosts)
@@ -91,6 +91,8 @@ def command_line(self, track, challenge, car, name, plugins, target_hosts):
         cmd += " --revision \"%s\"" % self.revision
         cmd += " --effective-start-date \"%s\"" % self.effective_start_date
         cmd += " --track=%s" % track
+        if track_params:
+            cmd += " --track-params=\"%s\"" % track_params
         cmd += " --challenge=%s" % challenge
         cmd += " --car=%s" % car
         cmd += " --user-tag=\"%s\"" % self.format_tag(additional_tags={"name": name})
@@ -144,7 +146,7 @@ def runnable(self, track, challenge, car, plugins, target_hosts):
             return False
         return True
 
-    def command_line(self, track, challenge, car, name, plugins, target_hosts):
+    def command_line(self, track, track_params, challenge, car, name, plugins, target_hosts):
         cmd = RALLY_BINARY
         cmd += " --configuration-name=%s" % self.configuration_name
         cmd += " --target-host=\"%s\"" % ",".join(target_hosts)
@@ -153,6 +155,8 @@ def command_line(self, track, challenge, car, name, plugins, target_hosts):
         cmd += " --distribution-version=%s" % self.distribution_version
         cmd += " --effective-start-date \"%s\"" % self.effective_start_date
         cmd += " --track=%s" % track
+        if track_params:
+            cmd += " --track-params=\"%s\"" % track_params
         cmd += " --challenge=%s" % challenge
         cmd += " --car=%s" % car
         cmd += " --user-tag=\"%s\"" % self.format_tag(additional_tags={"name": name})
@@ -190,7 +194,7 @@ def runnable(self, track, challenge, car, plugins, target_hosts):
             return "sorted" not in challenge
         return True
 
-    def command_line(self, track, challenge, car, name, plugins, target_hosts):
+    def command_line(self, track, track_params, challenge, car, name, plugins, target_hosts):
         cmd = RALLY_BINARY
         cmd += " --configuration-name=%s" % self.configuration_name
         cmd += " --target-host=\"%s\"" % ",".join(target_hosts)
@@ -199,6 +203,8 @@ def command_line(self, track, challenge, car, name, plugins, target_hosts):
         cmd += " --distribution-version=%s" % self.distribution_version
         cmd += " --effective-start-date \"%s\"" % self.effective_start_date
         cmd += " --track=%s" % track
+        if track_params:
+            cmd += " --track-params=\"%s\"" % track_params
         cmd += " --challenge=%s" % challenge
         cmd += " --car=%s" % car
         cmd += " --user-tag=\"%s\"" % self.format_tag(additional_tags={"name": name})
@@ -230,6 +236,7 @@ def run_rally(tracks, available_hosts, command, dry_run=False, skip_ansible=Fals
             node_count = combination.get("node-count", 1)
             target_hosts = choose_target_hosts(available_hosts, node_count)
             plugins = combination.get("plugins", "")
+            track_params = combination.get("track-params")
             info = race_info(track_name, challenge, car, plugins, node_count)
             if target_hosts:
                 if command.runnable(track_name, challenge, car, plugins, target_hosts):
@@ -240,7 +247,7 @@ def run_rally(tracks, available_hosts, command, dry_run=False, skip_ansible=Fals
                                "--tags=\"drop-caches,trim\" && cd -" % fixtures_dir)
                     logger.info("Running Rally on %s" % info)
                     start = time.perf_counter()
-                    if runner(command.command_line(track_name, challenge, car, name, plugins, target_hosts)):
+                    if runner(command.command_line(track_name, track_params, challenge, car, name, plugins, target_hosts)):
                         rally_failure = True
                         logger.error("Failed to run %s. Please check the logs." % info)
                     stop = time.perf_counter()

resources/tracks.json

@@ -6,7 +6,8 @@
         "#COMMENT": "We need to start with a configuration where we build all candidates on all machines as we skip the build step afterwards",
         "name": "geonames-append-4g-3nodes",
         "label": "add-4g-3nodes",
-        "challenge": "append-no-conflicts-index-only-1-replica",
+        "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:1",
         "car": "4gheap",
         "node-count": 3
       },
@@ -29,6 +30,7 @@
         "name": "geonames-append-4g-1node",
         "label": "add-4g",
         "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "4gheap"
       },
       {
@@ -47,6 +49,7 @@
         "#COMMENT": "Actually, we should not produce graphs for this one because indexing throughput is not that useful here.",
         "name": "geonames-append-verbose_iw-1node",
         "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "verbose_iw"
       }
     ]
@@ -96,12 +99,14 @@
         "name": "geopoint-append-4g-1node",
         "label": "add-4g",
         "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "4gheap"
       },
       {
         "name": "geopoint-append-4g-3nodes",
         "label": "add-4g-3nodes",
-        "challenge": "append-no-conflicts-index-only-1-replica",
+        "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:1",
         "car": "4gheap",
         "node-count": 3
       },
@@ -120,6 +125,7 @@
         "name": "pmc-append-defaults-1node",
         "label": "add-defaults",
         "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "defaults"
       },
       {
@@ -139,7 +145,8 @@
       {
         "name": "pmc-append-4g-3nodes",
         "label": "add-4g-3nodes",
-        "challenge": "append-no-conflicts-index-only-1-replica",
+        "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:1",
         "car": "4gheap",
         "node-count": 3
       },
@@ -178,12 +185,14 @@
         "name": "nyc_taxis-appendsorted-4g-1node",
         "label": "add-sorted-4g",
         "challenge": "append-sorted-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "4gheap"
       },
       {
         "name": "nyc_taxis-append-4g-3nodes",
         "label": "add-4g-3nodes",
-        "challenge": "append-no-conflicts-index-only-1-replica",
+        "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:1",
         "car": "4gheap",
         "node-count": 3
       }
@@ -196,6 +205,7 @@
         "name": "http_logs-append-defaults-1node",
         "label": "add-defaults",
         "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:0",
         "car": "defaults"
       },
       {
@@ -215,7 +225,8 @@
       {
         "name": "http_logs-append-4g-3nodes",
         "label": "add-4g-3nodes",
-        "challenge": "append-no-conflicts-index-only-1-replica",
+        "challenge": "append-no-conflicts-index-only",
+        "track-params": "number_of_replicas:1",
         "car": "4gheap",
         "node-count": 3
       },