ci: refactor CI setup

The current CI setup is complicated by the fact that we try to place
go:generate directives in the directory (package) where the result is
generated.

This cannot really be satisfied because one of the generation targets is
$modroot/.github/workflows. Per golang.org/issue/43985 this conflicts
with the fact that go generate ./... would not cover such a target, and
hence we need to wrap that go:generate directive in another that would
be covered by ./... - which is messy.

So instead:

* consolidate all CI-related CUE tools in internal/ci
* use go:generate directives in internal/ci for all CI-related
  generation, wrapping the CUE tools

This is in preparation for changes that ensure we have go test coverage
of the various GitHub workflow steps (import, validate, etc).

So now if you:

* change the workflow definitions in internal/ci, or
* pin to a later version of the GitHub workflow schema

just run:

    go generate ./internal/ci

Change-Id: Idbc548c195746f5e021b5a0c1f3ad8773c5fa05f
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/8481
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

internal/ci/ci_tool.cue

@@ -1,16 +1,58 @@
+// Copyright 2021 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package ci
 
 import (
-	"tool/file"
-	"tool/http"
-	"tool/exec"
 	"encoding/yaml"
+	"path"
+
+	"tool/exec"
+	"tool/file"
+	"tool/os"
 )
 
+// _#modroot is a common helper to get the module root
+//
+// TODO: use once we have a solution to cuelang.org/issue/704.
+// This will then allow us to remove the use of .. below.
+_#modroot: exec.Run & {
+	cmd:    "go list -m -f {{.Dir}}"
+	stdout: string
+}
+
+// Until we have the ability to inject contextual information
+// we need to pass in GOOS explicitly. Either by environment
+// variable (which we get for free when this is used via go generate)
+// or via a tag in the case you want to manually run the CUE
+// command.
+_#goos: os.Getenv & {
+	GOOS: *"unix" | string @tag(os)
+}
+
+// genworkflows regenerates the GitHub workflow Yaml definitions
+//
+// Until we have a resolution for cuelang.org/issue/704 this must be
+// run from the internal/ci package. At which point we can switch to using
+// _#modroot and move the associated go:generate directive into
+// internal/ci alongside this command definition.
 command: genworkflows: task: {
+	goos: _#goos
 	for w in workflows {
 		"\(w.file)": file.Create & {
-			filename: w.file
+			_dir:     path.FromSlash("../../.github/workflows", "unix")
+			filename: path.Join([_dir, w.file], goos.GOOS)
 			contents: """
 				# Generated by internal/ci/ci_tool.cue; do not edit
 
@@ -19,18 +61,3 @@ command: genworkflows: task: {
 		}
 	}
 }
-
-// vendorgithubschema is expected to be run within the cuelang.org/go
-// cue.mod directory
-command: vendorgithubschema: {
-	get: http.Get & {
-		request: body: ""
-
-		// Tip link: https://github.com/SchemaStore/schemastore/blob/master/src/schemas/json/github-workflow.json
-		url: "https://raw.githubusercontent.com/SchemaStore/schemastore/6fe4707b9d1c5d45cfc8d5b6d56968e65d2bdc38/src/schemas/json/github-workflow.json"
-	}
-	convert: exec.Run & {
-		stdin: get.response.body
-		cmd:   "go run cuelang.org/go/cmd/cue import -f -p json -l #Workflow: jsonschema: - --outfile pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue"
-	}
-}

.github/workflows/gen.go → internal/ci/gen.go

@@ -1,4 +1,4 @@
-// Copyright 2020 The CUE Authors
+// Copyright 2021 The CUE Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package workflows
+package ci
 
-//go:generate go run cuelang.org/go/cmd/cue cmd genworkflows cuelang.org/go/internal/ci
+//go:generate go run cuelang.org/go/cmd/cue cmd vendorgithubschema ./vendor
+//go:generate go run cuelang.org/go/cmd/cue cmd genworkflows

internal/ci/vendor/vendor_tool.cue

@@ -0,0 +1,73 @@
+// Copyright 2021 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vendor
+
+import (
+	"path"
+
+	"tool/exec"
+	"tool/file"
+	"tool/http"
+	"tool/os"
+)
+
+// _#modroot is a common helper to get the module root
+//
+// TODO: use once we have a solution to cuelang.org/issue/704.
+// This will then allow us to remove the use of .. below.
+_#modroot: exec.Run & {
+	cmd:    "go list -m -f {{.Dir}}"
+	stdout: string
+}
+
+// Until we have the ability to inject contextual information
+// we need to pass in GOOS explicitly. Either by environment
+// variable (which we get for free when this is used via go generate)
+// or via a tag in the case you want to manually run the CUE
+// command.
+_#goos: os.Getenv & {
+	GOOS: *"unix" | string @tag(os)
+}
+
+// vendorgithubschema vendors a "cue import"-ed version of the JSONSchema that
+// defines GitHub workflows into the main module's cue.mod/pkg.
+//
+// Until we have a resolution for cuelang.org/issue/704 this must be
+// run from the internal/ci package. At which point we can switch to using
+// _#modroot and move the associated go:generate directive into
+// internal/ci alongside this command definition.
+//
+// This also explains why the ../../ relative path specification below
+// appear wrong in the context of the containing directory internal/ci/vendor.
+command: vendorgithubschema: {
+	goos: _#goos
+	get:  http.Get & {
+		request: body: ""
+
+		// Tip link for humans:
+		// https://github.com/SchemaStore/schemastore/blob/master/src/schemas/json/github-workflow.json
+		url: "https://raw.githubusercontent.com/SchemaStore/schemastore/6fe4707b9d1c5d45cfc8d5b6d56968e65d2bdc38/src/schemas/json/github-workflow.json"
+	}
+	convert: exec.Run & {
+		stdin:  get.response.body
+		cmd:    "go run cuelang.org/go/cmd/cue import -f -p json -l #Workflow: jsonschema: - -o -"
+		stdout: string
+	}
+	write: file.Create & {
+		_path:    path.FromSlash("../../cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue", "unix")
+		filename: path.Join([_path], goos.GOOS)
+		contents: convert.stdout
+	}
+}

internal/ci/workflows.cue

@@ -1,3 +1,17 @@
+// Copyright 2021 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package ci
 
 import (