Auto-generate source code from json schema defintions and go generate instructions

.travis.yml

@@ -29,7 +29,7 @@ install:
   - "go get -d -t ./..."
 
 script:
-  - "if test $GIMME_OS.$GIMME_ARCH = linux.amd64; then make rebuild; ${GOPATH}/bin/gotestcover -v -coverprofile=coverage.report ./...; go tool cover -func=coverage.report; else go install ./...; fi"
+  - "if test $GIMME_OS.$GIMME_ARCH = linux.amd64; then make rebuild check; ${GOPATH}/bin/gotestcover -v -coverprofile=coverage.report ./...; go tool cover -func=coverage.report; else go install ./...; fi"
 
 after_script:
   - "if test $GIMME_OS.$GIMME_ARCH = linux.amd64; then $HOME/gopath/bin/goveralls -coverprofile=coverage.report -service=travis-ci; fi"

Makefile

@@ -14,6 +14,7 @@ build:
 generate:
 	go get github.com/progrium/go-extpoints
 	go get github.com/jonasfj/go-import-subtree
+	go get github.com/taskcluster/taskcluster-worker/codegen/...
 	go generate ./...
 	go fmt ./...
 

codegen/global-config.yml

@@ -0,0 +1,16 @@
+$schema:            http://json-schema.org/draft-04/schema#
+title:              Worker Config
+description: |-
+  Schema to represent the global config object used for configuring taskcluster-worker
+type:               object
+properties:
+  workerId:
+    title:          WorkerID
+    description: |-
+      blah blah
+    type:           string
+additionalProperties: false
+required:
+  - engines
+  - plugins
+  - workerId

codegen/go-composite-schema/main.go

@@ -0,0 +1,168 @@
+// go-composite-schema is responsible for auto-generating plugin and engine code based
+// on static yml json schema files
+package main
+
+import (
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docopt/docopt-go"
+	"github.com/ghodss/yaml"
+	"github.com/kr/text"
+	"github.com/taskcluster/jsonschema2go"
+	"github.com/taskcluster/taskcluster-base-go/jsontest"
+	"golang.org/x/tools/imports"
+)
+
+var (
+	version = "go-composite-schema 1.0.0"
+	usage   = `
+go-composite-schema
+go-composite-schema is a tool for generating go source code for a function to
+return a CompositeSchema based on a static json schema definition stored in a
+yaml/json file in a package directory of a go project, together with some
+parameters included in a "go:generate" command. See
+https://godoc.org/github.com/taskcluster/taskcluster-worker/runtime#CompositeSchema
+for more information.
+
+Although the go-composite-schema command line tool can be used wherever you
+require a CompositeSchema, it is currently most appliable to taskcluster-worker
+engines and plugins.
+
+Typically, you should include a code generation comment in your source code for
+each CompositeSchema returning function you wish to create:
+
+ //go:generate go-composite-schema [--required] PROPERTY INPUT-FILE OUTPUT-FILE
+
+We currently use CompositeSchemas for defining both config structures and
+payload structures, for both engines and plugins. Therefore typically there
+could be one or two "go:generate" lines required per plugin and probably two
+per engine. This is because some plugins might not have custom config nor even
+custom payload, but engines are likely to require both.
+
+Please note, it is recommended to set environment variable GOPATH in order for
+go-composite-schema to correctly determine the correct package name.
+
+
+  Usage:
+    go-composite-schema [--required] PROPERTY INPUT-FILE OUTPUT-FILE
+    go-composite-schema -h|--help
+    go-composite-schema --version
+
+  Options:
+    -h --help              Display this help text.
+    --version              Display the version (` + version + `).
+`
+)
+
+func main() {
+	// Clear all logging fields, such as timestamps etc...
+	log.SetFlags(0)
+	log.SetPrefix("go-composite-schema: ")
+
+	// Parse the docopt string and exit on any error or help message.
+	args, err := docopt.Parse(usage, nil, true, version, false, true)
+	if err != nil {
+		log.Fatalf("ERROR: Cannot parse arguments: %s\n", err)
+	}
+
+	// assuming non-nil, and always type bool
+	req := args["--required"].(bool)
+
+	// assuming non-nil, and always type string
+	schemaProperty := args["PROPERTY"].(string)
+	inputFile := args["INPUT-FILE"].(string)
+	outputFile := args["OUTPUT-FILE"].(string)
+
+	// Get working directory
+	currentFolder, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Unable to obtain current working directory: %s", err)
+	}
+
+	// Read current package
+	pkg, err := build.ImportDir(currentFolder, build.AllowBinary)
+	if err != nil {
+		log.Fatalf("ERROR: Failed to determine go package inside directory '%s' - is your GOPATH set correctly ('%s')? Error: %s", currentFolder, os.Getenv("GOPATH"), err)
+	}
+
+	// Generate go types...
+	ymlFile := filepath.Join(currentFolder, inputFile)
+	if _, err := os.Stat(ymlFile); err == nil {
+		log.Printf("Found yaml file '%v'", ymlFile)
+	} else {
+		log.Fatalf("ERROR: could not read file '%v'", ymlFile)
+	}
+	url := "file://" + ymlFile
+	goFile := filepath.Join(currentFolder, outputFile)
+	log.Printf("Generating '%v'...", goFile)
+	job := &jsonschema2go.Job{
+		Package:     pkg.Name,
+		URLs:        []string{url},
+		ExportTypes: false,
+	}
+	result, err := job.Execute()
+	if err != nil {
+		log.Fatalf("ERROR: Problem assembling content for file '%v': %s", goFile, err)
+	}
+	generatedCode := append(result.SourceCode, []byte("\n"+generateFunctions(ymlFile, result.SchemaSet.SubSchema(url).TypeName, schemaProperty, req))...)
+	sourceCode, err := imports.Process(
+		goFile,
+		[]byte(generatedCode),
+		&imports.Options{
+			AllErrors: true,
+			Comments:  true,
+			TabIndent: true,
+			TabWidth:  0,
+			Fragment:  false,
+		},
+	)
+	if err != nil {
+		log.Fatalf("ERROR: Could not format generated source code for file '%v': %s\nCode:\n%v", goFile, err, string(generatedCode))
+	}
+	ioutil.WriteFile(goFile, []byte(sourceCode), 0644)
+	if err != nil {
+		log.Fatalf("ERROR: Could not write generated source code to file '%v': %s", goFile, err)
+	}
+}
+
+func generateFunctions(ymlFile, goType, schemaProperty string, req bool) string {
+	data, err := ioutil.ReadFile(ymlFile)
+	if err != nil {
+		log.Fatalf("ERROR: Problem reading from file '%v' - %s", ymlFile, err)
+	}
+	// json is valid YAML, so we can safely convert, even if it is already json
+	rawJson, err := yaml.YAMLToJSON(data)
+	if err != nil {
+		log.Fatalf("ERROR: Problem converting file '%v' to json format - %s", ymlFile, err)
+	}
+	rawJson, err = jsontest.FormatJson(rawJson)
+	if err != nil {
+		log.Fatalf("ERROR: Problem pretty printing json in '%v' - %s", ymlFile, err)
+	}
+	result := "var " + goType + "Schema = func() runtime.CompositeSchema {\n"
+	result += "\tschema, err := runtime.NewCompositeSchema(\n"
+	result += "\t\t\"" + schemaProperty + "\",\n"
+	result += "\t\t`\n"
+	// the following strings.Replace function call safely escapes backticks (`) in rawJson
+	result += strings.Replace(text.Indent(fmt.Sprintf("%v", string(rawJson)), "\t\t")+"\n", "`", "` + \"`\" + `", -1)
+	result += "\t\t`,\n"
+	if req {
+		result += "\t\ttrue,\n"
+	}
+	result += "\t\tfunc() interface{} {\n"
+	result += "\t\t\treturn &" + goType + "{}\n"
+	result += "\t\t},\n"
+	result += "\t)\n"
+	result += "\tif err != nil {\n"
+	result += "\t\tpanic(err)\n"
+	result += "\t}\n"
+	result += "\treturn schema\n"
+	result += "}()\n"
+	return result
+}

engines/docker/config-schema.yml

@@ -0,0 +1,14 @@
+$schema:            http://json-schema.org/draft-04/schema#
+title:              Config
+description: |-
+  Config applicable to docker engine
+type:               object
+properties:
+  rootVolume:
+    title:          Root Volume
+    description: |-
+      Root Volume blah blah
+    type:           string
+additionalProperties: false
+required:
+  - rootVolume

engines/docker/docker.go

@@ -0,0 +1,4 @@
+//go:generate go-composite-schema --required config config-schema.yml generated_configschema.go
+
+// comments
+package docker

engines/docker/generated_configschema.go

@@ -0,0 +1,47 @@
+// This source code file is AUTO-GENERATED by github.com/taskcluster/jsonschema2go
+
+package docker
+
+import "github.com/taskcluster/taskcluster-worker/runtime"
+
+type (
+	// Config applicable to docker engine
+	config struct {
+
+		// Root Volume blah blah
+		RootVolume string `json:"rootVolume"`
+	}
+)
+
+var configSchema = func() runtime.CompositeSchema {
+	schema, err := runtime.NewCompositeSchema(
+		"config",
+		`
+		{
+		  "$schema": "http://json-schema.org/draft-04/schema#",
+		  "additionalProperties": false,
+		  "description": "Config applicable to docker engine",
+		  "properties": {
+		    "rootVolume": {
+		      "description": "Root Volume blah blah",
+		      "title": "Root Volume",
+		      "type": "string"
+		    }
+		  },
+		  "required": [
+		    "rootVolume"
+		  ],
+		  "title": "Config",
+		  "type": "object"
+		}
+		`,
+		true,
+		func() interface{} {
+			return &config{}
+		},
+	)
+	if err != nil {
+		panic(err)
+	}
+	return schema
+}()

engines/engine.go

@@ -11,10 +11,6 @@ type SandboxOptions struct {
 	TaskContext *runtime.TaskContext
 	// Result from PayloadSchema().Parse(). Implementors are safe to assert
 	// this back to their target type.
-	// Note: This is passed by-value for efficiency (and to prohibit nil), if
-	// adding any large fields please consider adding them as pointers.
-	// Note: This is intended to be a simple argument wrapper, do not add methods
-	// to this struct.
 	Payload interface{}
 }
 
@@ -126,6 +122,12 @@ func (EngineBase) PayloadSchema() runtime.CompositeSchema {
 	return runtime.NewEmptyCompositeSchema()
 }
 
+// ConfigSchema returns an empty jsonschema indicating that no custom config is
+// required.
+func (EngineBase) ConfigSchema() runtime.CompositeSchema {
+	return runtime.NewEmptyCompositeSchema()
+}
+
 // Capabilities returns an zero value Capabilities struct indicating that
 // most features aren't supported.
 func (EngineBase) Capabilities() Capabilities {

engines/enginetest/testutils.go

@@ -48,7 +48,7 @@ func (p *engineProvider) ensureEngine(engineName string) {
 		fmtPanic("Couldn't find EngineProvider: ", engineName)
 	}
 	// Create Engine instance
-	engine, err := engineProvider(extpoints.EngineOptions{
+	engine, err := engineProvider.NewEngine(extpoints.EngineOptions{
 		Environment: p.environment,
 		Log:         p.environment.Log.WithField("engine", engineName),
 	})

engines/extpoints/interfaces.go

@@ -34,4 +34,10 @@ type EngineOptions struct {
 // Any error here will be fatal and likely cause the worker to stop working.
 // If an implementor can determine that the platform isn't supported at
 // compile-time it is recommended to not register the implementation.
-type EngineProvider func(options EngineOptions) (engines.Engine, error)
+type EngineProvider interface {
+	NewEngine(options EngineOptions) (engines.Engine, error)
+
+	// ConfigSchema returns the CompositeSchema that represents the engine
+	// configuration
+	ConfigSchema() runtime.CompositeSchema
+}

engines/mock/generated_payloadschema.go

@@ -0,0 +1,70 @@
+// This source code file is AUTO-GENERATED by github.com/taskcluster/jsonschema2go
+
+package mockengine
+
+import "github.com/taskcluster/taskcluster-worker/runtime"
+
+type (
+	payload struct {
+		Argument string `json:"argument"`
+
+		Delay int `json:"delay"`
+
+		// Possible values:
+		//   * "true"
+		//   * "false"
+		//   * "set-volume"
+		//   * "get-volume"
+		//   * "ping-proxy"
+		//   * "write-log"
+		//   * "write-error-log"
+		Function string `json:"function"`
+	}
+)
+
+var payloadSchema = func() runtime.CompositeSchema {
+	schema, err := runtime.NewCompositeSchema(
+		"start",
+		`
+		{
+		  "$schema": "http://json-schema.org/draft-04/schema#",
+		  "additionalProperties": false,
+		  "properties": {
+		    "argument": {
+		      "type": "string"
+		    },
+		    "delay": {
+		      "type": "integer"
+		    },
+		    "function": {
+		      "enum": [
+		        "true",
+		        "false",
+		        "set-volume",
+		        "get-volume",
+		        "ping-proxy",
+		        "write-log",
+		        "write-error-log"
+		      ],
+		      "type": "string"
+		    }
+		  },
+		  "required": [
+		    "delay",
+		    "function",
+		    "argument"
+		  ],
+		  "title": "Payload",
+		  "type": "object"
+		}
+		`,
+		true,
+		func() interface{} {
+			return &payload{}
+		},
+	)
+	if err != nil {
+		panic(err)
+	}
+	return schema
+}()