Compile command (#321)

* adding flytepropeller dep

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* add compile cmd

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* printing summary

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compile cmd tests

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

This reverts commit 75665de7db90646a3e5d7ec65f64c61739c31955.

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* ci fix: go 1.18

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd: fix typos & err handling

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd: more tests

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* exporting UnMarshalContents

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compile uses UnMarshalContents

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compile cmd usage examples

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd: popping errors up

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd: add flag config

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd better docstring

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* compilecmd uses flags config

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

* clean up

Signed-off-by: David Przybilla <dav.alejandro@gmail.com>

flytectl/.github/workflows/checks.yml

@@ -20,21 +20,21 @@ jobs:
     name: Lint
     uses: flyteorg/flytetools/.github/workflows/lint.yml@master
     with:
-      go-version: 1.17
+      go-version: 1.18
 
   tests:
     name: Unit Tests
     uses: flyteorg/flytetools/.github/workflows/tests.yml@master
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
     with:
-      go-version: 1.17
+      go-version: 1.18
 
   generate:
     name: Check Go Gennerate
     uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master
     with:
-      go-version: 1.17
+      go-version: 1.18
 
   dry_run_goreleaser:
     name: Dry Run Goreleaser
@@ -52,7 +52,7 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.17'
+          go-version: '1.18'
       - name: Run GoReleaser dry run
         uses: goreleaser/goreleaser-action@v2
         with:
@@ -74,7 +74,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Build Flytectl binary
         run: make compile
       - name: Create a sandbox cluster
@@ -104,7 +104,7 @@ jobs:
           lfs: true
       - uses: actions/setup-go@v1
         with:
-          go-version: '1.17'
+          go-version: '1.18'
       - uses: actions/setup-python@v1
         with:
           python-version: 3.8

flytectl/cmd/compile/compile.go

@@ -0,0 +1,146 @@
+package compile
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	config "github.com/flyteorg/flytectl/cmd/config/subcommand/compile"
+	cmdCore "github.com/flyteorg/flytectl/cmd/core"
+	"github.com/flyteorg/flytectl/cmd/register"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+	"github.com/flyteorg/flytepropeller/pkg/compiler"
+	"github.com/flyteorg/flytepropeller/pkg/compiler/common"
+)
+
+// Utility function for compiling a list of Tasks
+func compileTasks(tasks []*core.TaskTemplate) ([]*core.CompiledTask, error) {
+	res := make([]*core.CompiledTask, 0, len(tasks))
+	for _, task := range tasks {
+		compiledTask, err := compiler.CompileTask(task)
+		if err != nil {
+			return nil, err
+		}
+		res = append(res, compiledTask)
+	}
+	return res, nil
+}
+
+/*
+ Utility to compile a packaged workflow locally.
+ compilation is done locally so no flyte cluster is required.
+*/
+func compileFromPackage(packagePath string) error {
+	args := []string{packagePath}
+	fileList, tmpDir, err := register.GetSerializeOutputFiles(context.Background(), args, true)
+	defer os.RemoveAll(tmpDir)
+	if err != nil {
+		fmt.Println("Error found while extracting package..")
+		return err
+	}
+	fmt.Println("Successfully extracted package...")
+	fmt.Println("Processing Protobuf files...")
+	workflows := make(map[string]*admin.WorkflowSpec)
+	plans := make(map[string]*admin.LaunchPlan)
+	tasks := []*admin.TaskSpec{}
+
+	for _, pbFilePath := range fileList {
+		rawTsk, err := ioutil.ReadFile(pbFilePath)
+		if err != nil {
+			fmt.Printf("error unmarshalling task..")
+			return err
+		}
+		spec, err := register.UnMarshalContents(context.Background(), rawTsk, pbFilePath)
+		if err != nil {
+			return err
+		}
+
+		switch v := spec.(type) {
+		case *admin.TaskSpec:
+			tasks = append(tasks, v)
+		case *admin.WorkflowSpec:
+			workflows[v.Template.Id.Name] = v
+		case *admin.LaunchPlan:
+			plans[v.Id.Name] = v
+		}
+	}
+
+	// compile tasks
+	taskTemplates := []*core.TaskTemplate{}
+	for _, task := range tasks {
+		taskTemplates = append(taskTemplates, task.Template)
+	}
+
+	fmt.Println("\nCompiling tasks...")
+	compiledTasks, err := compileTasks(taskTemplates)
+	if err != nil {
+		fmt.Println("Error while compiling tasks...")
+		return err
+	}
+
+	// compile workflows
+	for wfName, workflow := range workflows {
+
+		fmt.Println("\nCompiling workflow:", wfName)
+		plan := plans[wfName]
+
+		_, err := compiler.CompileWorkflow(workflow.Template,
+			workflow.SubWorkflows,
+			compiledTasks,
+			[]common.InterfaceProvider{compiler.NewLaunchPlanInterfaceProvider(*plan)})
+		if err != nil {
+			fmt.Println(":( Error Compiling workflow:", wfName)
+			return err
+		}
+
+	}
+
+	fmt.Println("All Workflows compiled successfully!")
+	fmt.Println("\nSummary:")
+	fmt.Println(len(workflows), " workflows found in package")
+	fmt.Println(len(tasks), " Tasks found in package")
+	fmt.Println(len(plans), " Launch plans found in package")
+	return nil
+}
+
+const (
+	compileShort = `Validate flyte packages without registration needed.`
+	compileLong  = `
+Validate workflows by compiling flyte's serialized protobuf files  (task, workflows and launch plans). This is useful for testing workflows and tasks without neededing to talk with a flyte cluster.
+
+::
+
+ flytectl compile --file my-flyte-package.tgz
+
+::
+
+ flytectl compile --file /home/user/dags/my-flyte-package.tgz
+
+.. note::
+   Input file is a path to a tgz. This file is generated by either pyflyte or jflyte. tgz file contains protobuf files describing workflows, tasks and launch plans.
+
+`
+)
+
+func compile(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error {
+	packageFilePath := config.DefaultCompileConfig.File
+	if packageFilePath == "" {
+		return fmt.Errorf("path to package tgz's file is a required flag")
+	}
+	return compileFromPackage(packageFilePath)
+}
+
+func CreateCompileCommand() map[string]cmdCore.CommandEntry {
+	compileResourcesFuncs := map[string]cmdCore.CommandEntry{
+		"compile": {
+			Short:                    compileShort,
+			Long:                     compileLong,
+			CmdFunc:                  compile,
+			PFlagProvider:            config.DefaultCompileConfig,
+			ProjectDomainNotRequired: true,
+		},
+	}
+	return compileResourcesFuncs
+}

flytectl/cmd/compile/compile_test.go

@@ -0,0 +1,71 @@
+package compile
+
+import (
+	"context"
+	"testing"
+
+	config "github.com/flyteorg/flytectl/cmd/config/subcommand/compile"
+	cmdCore "github.com/flyteorg/flytectl/cmd/core"
+	u "github.com/flyteorg/flytectl/cmd/testutils"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCompileCommand(t *testing.T) {
+	rootCmd := &cobra.Command{
+		Long:              "Flytectl is a CLI tool written in Go to interact with the FlyteAdmin service.",
+		Short:             "Flytectl CLI tool",
+		Use:               "flytectl",
+		DisableAutoGenTag: true,
+	}
+	compileCommand := CreateCompileCommand()
+	cmdCore.AddCommands(rootCmd, compileCommand)
+	cmdNouns := rootCmd.Commands()
+	assert.Equal(t, cmdNouns[0].Use, "compile")
+	assert.Equal(t, cmdNouns[0].Flags().Lookup("file").Name, "file")
+	// check shorthand
+	assert.Equal(t, cmdNouns[0].Short, compileShort)
+
+	// compiling via cobra command
+	compileCfg := config.DefaultCompileConfig
+	compileCfg.File = "testdata/valid-package.tgz"
+	var setup = u.Setup
+	s := setup()
+	compileCmd := CreateCompileCommand()["compile"]
+	err := compileCmd.CmdFunc(context.Background(), []string{}, s.CmdCtx)
+	assert.Nil(t, err, "compiling via cmd returns err")
+
+	// calling command with empty file flag
+	compileCfg = config.DefaultCompileConfig
+	compileCfg.File = ""
+	err = compileCmd.CmdFunc(context.Background(), []string{}, s.CmdCtx)
+	assert.NotNil(t, err, "calling compile with Empty file flag does not error")
+}
+
+func TestCompilePackage(t *testing.T) {
+	// valid package contains two workflows
+	// with three tasks
+	err := compileFromPackage("testdata/valid-package.tgz")
+	assert.Nil(t, err, "unable to compile a valid package")
+
+	// invalid gzip header
+	err = compileFromPackage("testdata/invalid.tgz")
+	assert.NotNil(t, err, "compiling an invalid package returns no error")
+
+	// invalid workflow, types do not match
+	err = compileFromPackage("testdata/bad-workflow-package.tgz")
+	assert.NotNil(t, err, "compilin an invalid workflow returns no error")
+
+	// testing badly serialized task
+	err = compileFromPackage("testdata/invalidtask.tgz")
+	assert.NotNil(t, err, "unable to handle invalid task")
+
+	// testing badly serialized launchplan
+	err = compileFromPackage("testdata/invalidlaunchplan.tgz")
+	assert.NotNil(t, err, "unable to handle invalid launchplan")
+
+	// testing badly serialized workflow
+	err = compileFromPackage("testdata/invalidworkflow.tgz")
+	assert.NotNil(t, err, "unable to handle invalid workflow")
+
+}

flytectl/cmd/compile/testdata/invalid.tgz

@@ -0,0 +1 @@
+invalid tgz file

flytectl/cmd/config/subcommand/compile/compile_config.go

@@ -0,0 +1,11 @@
+package compile
+
+//go:generate pflags Config --default-var DefaultCompileConfig --bind-default-var
+var (
+	DefaultCompileConfig = &Config{}
+)
+
+// Config stores the flags required by compile command
+type Config struct {
+	File string `json:"file" pflag:",Path to a flyte package file. Flyte packages are tgz files generated by pyflyte or jflyte."`
+}