Enable specifying test subsets

README.md

@@ -19,6 +19,7 @@ CLI arguments:
 - `--outDir` Location on the filesystem to download and unpack a zip file specified in `--testsRemote`. Has no meaning if `--testsRemote` is not specified.
 - `--target` URL of any appliance serving the Ethereum 2.0 API.
 - `--timeout` Time after which to abandon waiting tests. Defaults to 10 minutes. Uses [Go duration syntax](https://golang.org/pkg/time/#ParseDuration).
+- `--subset` The subset of paths to run tests for. For example, set this to "/v1/node" to only run tests for routes in that path. Defaults to "/" (all paths).
 
 ## Syntax of test cases
 

cmd/eth2-comply/main.go

@@ -20,6 +20,7 @@ func main() {
 	outDir := flag.String("outDir", "/tmp", "A directory where zip files will be downloaded and unzipped.")
 	target := flag.String("target", "NO TARGET PROVIDED", "A URL to run tests against, for example http://localhost:5051")
 	timeout := flag.String("timeout", "10m", "The time to wait for a case execution to complete. For example, 3600s, 60m, 1h")
+	subset := flag.String("subset", "/", "The subset of paths to run tests for. For example, set this to \"/v1/node\" to only run tests for routes in that path. Defaults to \"/\" (all paths).")
 	flag.Parse()
 
 	// Setup OAPI client.
@@ -51,7 +52,7 @@ func main() {
 
 	// Execute test cases.
 	for _, testCase := range testCases {
-		go testCase.Exec(ctx)
+		go testCase.Exec(ctx, *subset)
 	}
 
 	// Concurrent helper function cancels contexts (tests) that timeout.

pkg/testcases/import.go

@@ -63,7 +63,6 @@ func All(opts *TestsCasesOpts) ([]*Case, error) {
 	}
 
 	for _, config := range configs {
-		config.Route = opts.Target + config.Route
 		if err != nil {
 			return nil, err
 		}

pkg/testcases/testcases.go

@@ -24,6 +24,7 @@ type Case struct {
 	Config     CaseConfig
 	OapiClient *eth2spec.APIClient
 	Result     Result
+	Skipped    bool
 	Done       chan struct{}
 }
 
@@ -83,9 +84,16 @@ func (e ExpectationsError) Error() string {
 //    (is ill-formed), or a network condition prevented contacting the target.
 //
 // Otherwise, the Result is marked as a Success and the Error is left nil.
-func (c *Case) Exec(ctx context.Context) {
+func (c *Case) Exec(ctx context.Context, pathsRoot string) {
 	defer close(c.Done)
 
+	// If a test should be excluded because it is not beneath the paths root,
+	// skip it here.
+	if !strings.HasPrefix(c.Config.Route, pathsRoot) {
+		c.Skipped = true
+		return
+	}
+
 	// If a test specifies an await slot, wait for the node to sync that slot.
 	// Otherwise, just wait for the node to be healthy. awaitTargetHasSlot
 	// uses awaitTargetIsHealthy internally.
@@ -154,7 +162,9 @@ func (c Case) PrintResults() {
 
 	fmt.Printf("%s ", routeString)
 
-	if !c.Result.Success {
+	if c.Skipped {
+		fmt.Printf("Skipped\n")
+	} else if !c.Result.Success {
 		fmt.Printf("❌\n")
 		fmt.Println(c.Result.Error)
 	} else {