OPCT-199: added EFS removing bindata dependency (#73)

The goal of this PR is to replace bindata dependency to Go native embed
file system to handle static files used in OPCT (manifest plugins).

Changes in this PR:

- [blocker] remove bindata dependency, and workaround added to cover
missing fixes caught by Go static checks
- introduce embed filesystem to manage static data (plugin manifest
templates)
- improve the template handling, covering the entire manifest file,
providing flexibility to apply hooks in the manifest in runtime
(required for future goals like injecting dynamic scripts or manifests
to run dynamic/extended tests)
- increase unit tests coverage for static files and template processing
- deprecate assets command (non-documented and used - plugins may not
work independently, it don't need to export it to user)
- expand CI jobs to keep clearly what is running, run in parallel, and
keep clear the running jobs in the PRs

.github/workflows/go.yaml

@@ -1,4 +1,4 @@
-name: "Go"
+name: "OPCT"
 
 on:
   pull_request:
@@ -10,8 +10,9 @@ on:
       - '*'
 
 jobs:
-  static:
-    name: "Run Continuous Integration"
+  # TODO: add go-lint
+  go-test:
+    name: "go-test"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -20,23 +21,40 @@ jobs:
         with:
           go-version: 1.19
 
-      - name: Verify bindata was generated
-        run: |
-          go install github.com/go-bindata/go-bindata/go-bindata@latest
-          set -xe
-          ./hack/update-generated-bindata.sh
-          set +ex
-          # bypassing CI check for pkg/assets/bindata.go due go-bindata issues with ioutils
-          # https://issues.redhat.com/browse/OPCT-199
-          git checkout pkg/assets/bindata.go
-          git diff --exit-code
-
       - name: Run unit tests
         run: make test
 
       - name: Run go vet
         run: make vet
 
+      - name: Run static code analysis
+        uses: dominikh/staticcheck-action@v1.3.0
+        with:
+          version: "2022.1.3"
+
+  go-vet:
+    name: "go-vet"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Run go vet
+        run: make vet
+
+  go-static:
+    name: "go-staticcheck"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
       - name: Run static code analysis
         uses: dominikh/staticcheck-action@v1.3.0
         with:

.github/workflows/release.yaml

@@ -1,4 +1,4 @@
-name: Release
+name: OPCT Release
 
 on:
   push:

.github/workflows/static-website.yml

@@ -1,4 +1,4 @@
-name: Deploy static content to Pages
+name: Documentation
 
 on:
   # Static pages are build only targeting the main branch

Makefile

@@ -16,14 +16,15 @@ unexport GOFLAGS
 .PHONY: all
 all: linux-amd64-container cross-build-windows-amd64 cross-build-darwin-amd64 cross-build-darwin-arm64
 
+.PHONY: update-go
+update-go:
+	go get -u
+	go mod tidy
+
 .PHONY: build
 build:
 	go build -o openshift-provider-cert $(GO_BUILD_FLAGS)
 
-.PHONY: generate
-update:
-	./hack/update-generated-bindata.sh
-
 .PHONY: verify-codegen
 verify-codegen:
 	./hack/verify-codegen.sh

cmd/root.go

@@ -10,7 +10,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/vmware-tanzu/sonobuoy/cmd/sonobuoy/app"
 
-	"github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/assets"
 	"github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/destroy"
 	"github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/report"
 	"github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/retrieve"
@@ -60,7 +59,6 @@ func init() {
 	viper.BindPFlag("loglevel", rootCmd.PersistentFlags().Lookup("loglevel"))
 
 	// Link in child commands
-	rootCmd.AddCommand(assets.NewCmdAssets())
 	rootCmd.AddCommand(destroy.NewCmdDestroy())
 	rootCmd.AddCommand(retrieve.NewCmdRetrieve())
 	rootCmd.AddCommand(run.NewCmdRun())

data/data_test.go

@@ -0,0 +1,68 @@
+package data
+
+import (
+	"embed"
+	"testing"
+
+	efs "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/assets"
+	"github.com/stretchr/testify/assert"
+)
+
+//go:embed templates/plugins
+var testTemplatesPluginsAll embed.FS
+
+// TestDataTemplatesPluginsManifests asserts required plugin manifests are present in EFS.
+func TestDataTemplatesPluginsManifests(t *testing.T) {
+	type testCase struct {
+		name   string
+		assert func(tc *testCase)
+	}
+	cases := []testCase{
+		{
+			name: "plugins-manifest-required",
+			assert: func(tc *testCase) {
+				want := []string{
+					"templates/plugins/openshift-artifacts-collector.yaml",
+					"templates/plugins/openshift-cluster-upgrade.yaml",
+					"templates/plugins/openshift-conformance-validated.yaml",
+					"templates/plugins/openshift-kube-conformance.yaml",
+				}
+				got, err := efs.GetAllFilenames(efs.GetData(), "templates/plugins")
+				if err != nil {
+					t.Fatalf("failed to read efs: %v", err)
+				}
+				assert.Equal(t, want, got, "plugin manifest files are present")
+			},
+		},
+		{
+			name: "plugins-manifest-readable",
+			assert: func(tc *testCase) {
+				want := true
+				got := false
+				manifests, err := efs.GetAllFilenames(efs.GetData(), "templates/plugins")
+				if err != nil {
+					t.Fatalf("failed to read efs: %v", err)
+				}
+				for _, m := range manifests {
+					manifestFile, err := efs.GetData().ReadFile(m)
+					if err != nil {
+						t.Fatalf("unable to read manifest %s: %v", m, err)
+					}
+					if len(manifestFile) == 0 {
+						t.Fatalf("empty manifest %s", m)
+					}
+				}
+				got = true
+				assert.Equal(t, want, got, "plugin manifests are readable")
+			},
+		},
+	}
+
+	efs.UpdateData(&testTemplatesPluginsAll)
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			tc.assert(&tc)
+		})
+	}
+}

internal/assets/assets.go

@@ -0,0 +1,33 @@
+package assets
+
+import (
+	"embed"
+	"io/fs"
+)
+
+var efs *embed.FS
+
+func GetData() *embed.FS {
+	return efs
+}
+
+func UpdateData(d *embed.FS) {
+	efs = d
+}
+
+// GetAllFilenames return all file names from an path in embeded EFS.
+func GetAllFilenames(efs *embed.FS, path string) (files []string, err error) {
+	if err := fs.WalkDir(efs, path, func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+
+		files = append(files, path)
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}

main.go

@@ -1,7 +1,16 @@
 package main
 
-import "github.com/redhat-openshift-ecosystem/provider-certification-tool/cmd"
+import (
+	"embed"
+
+	cmd "github.com/redhat-openshift-ecosystem/provider-certification-tool/cmd"
+	"github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/assets"
+)
+
+//go:embed data/templates
+var vfs embed.FS
 
 func main() {
+	assets.UpdateData(&vfs)
 	cmd.Execute()
 }