Skip to content

Commit

Permalink
Add global debug option to flow command and correct flags (#851)
Browse files Browse the repository at this point in the history 
* Add global debug option and correct flags

- add --verbose flag to validate and generate tests

* Add slices package

* Update go.mod

* Add missing connection flag

* Fix TestContainerRemoveFailure

Co-authored-by: Neel Dalsania <neel.dalsania@astronomer.io>
  • Loading branch information
@feluelle @neel-astro
feluelle and neel-astro committed Nov 16, 2022
1 parent bc1e43b commit 9d2377f
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 47 deletions.
65 changes: 44 additions & 21 deletions cmd/sql/flow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/astronomer/astro-cli/sql"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
)

var (
Expand All @@ -17,6 +18,7 @@ var (
projectDir string
generateTasks bool
verbose bool
debug bool
)

func getAbsolutePath(path string) (string, error) {
Expand Down Expand Up @@ -46,17 +48,6 @@ func createProjectDir(projectDir string) (mountDir string, err error) {
return projectDir, nil
}

func buildCommonFlags() map[string]string {
flags := make(map[string]string)
if environment != "" {
flags["env"] = environment
}
if connection != "" {
flags["connection"] = connection
}
return flags
}

func getBaseMountDirs(projectDir string) ([]string, error) {
mountDir, err := createProjectDir(projectDir)
if err != nil {
Expand All @@ -67,7 +58,7 @@ func getBaseMountDirs(projectDir string) ([]string, error) {
}

func buildFlagsAndMountDirs(projectDir string, setProjectDir, setAirflowHome, setAirflowDagsFolder bool) (flags map[string]string, mountDirs []string, err error) {
flags = buildCommonFlags()
flags = make(map[string]string)
mountDirs, err = getBaseMountDirs(projectDir)
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -104,6 +95,9 @@ func buildFlagsAndMountDirs(projectDir string, setProjectDir, setAirflowHome, se

func executeCmd(cmd *cobra.Command, args []string, flags map[string]string, mountDirs []string) error {
cmdString := []string{cmd.Parent().Name(), cmd.Name()}
if debug {
cmdString = slices.Insert(cmdString, 1, "--debug")
}
err := sql.CommonDockerUtil(cmdString, args, flags, mountDirs)
if err != nil {
return fmt.Errorf("error running %v: %w", cmdString, err)
Expand Down Expand Up @@ -149,6 +143,18 @@ func executeValidate(cmd *cobra.Command, args []string) error {
projectDirAbsolute := mountDirs[0]
args = []string{projectDirAbsolute}

if environment != "" {
flags["env"] = environment
}

if connection != "" {
flags["connection"] = connection
}

if verbose {
args = append(args, "--verbose")
}

return executeCmd(cmd, args, flags, mountDirs)
}

Expand All @@ -166,6 +172,14 @@ func executeGenerate(cmd *cobra.Command, args []string) error {
args = append(args, "--generate-tasks")
}

if environment != "" {
flags["env"] = environment
}

if verbose {
args = append(args, "--verbose")
}

return executeCmd(cmd, args, flags, mountDirs)
}

Expand All @@ -179,6 +193,10 @@ func executeRun(cmd *cobra.Command, args []string) error {
return err
}

if environment != "" {
flags["env"] = environment
}

if verbose {
args = append(args, "--verbose")
}
Expand Down Expand Up @@ -227,8 +245,8 @@ func initCommand() *cobra.Command {
SilenceUsage: true,
}
cmd.SetHelpFunc(executeHelp)
cmd.Flags().StringVarP(&airflowHome, "airflow-home", "a", "", "")
cmd.Flags().StringVarP(&airflowDagsFolder, "airflow-dags-folder", "d", "", "")
cmd.Flags().StringVar(&airflowHome, "airflow-home", "", "")
cmd.Flags().StringVar(&airflowDagsFolder, "airflow-dags-folder", "", "")
return cmd
}

Expand All @@ -240,7 +258,9 @@ func validateCommand() *cobra.Command {
SilenceUsage: true,
}
cmd.SetHelpFunc(executeHelp)
cmd.Flags().StringVarP(&connection, "connection", "c", "", "")
cmd.Flags().StringVar(&environment, "env", "default", "")
cmd.Flags().StringVar(&connection, "connection", "", "")
cmd.Flags().BoolVar(&verbose, "verbose", false, "")
return cmd
}

Expand All @@ -252,8 +272,10 @@ func generateCommand() *cobra.Command {
SilenceUsage: true,
}
cmd.SetHelpFunc(executeHelp)
cmd.Flags().StringVarP(&projectDir, "project-dir", "p", ".", "")
cmd.Flags().BoolVarP(&generateTasks, "generate-tasks", "g", false, "")
cmd.Flags().BoolVar(&generateTasks, "generate-tasks", false, "")
cmd.Flags().StringVar(&environment, "env", "default", "")
cmd.Flags().StringVar(&projectDir, "project-dir", ".", "")
cmd.Flags().BoolVar(&verbose, "verbose", false, "")
return cmd
}

Expand All @@ -265,9 +287,10 @@ func runCommand() *cobra.Command {
SilenceUsage: true,
}
cmd.SetHelpFunc(executeHelp)
cmd.Flags().StringVarP(&projectDir, "project-dir", "p", ".", "")
cmd.Flags().BoolVarP(&generateTasks, "generate-tasks", "g", false, "")
cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "")
cmd.Flags().BoolVar(&generateTasks, "generate-tasks", false, "")
cmd.Flags().StringVar(&environment, "env", "default", "")
cmd.Flags().StringVar(&projectDir, "project-dir", ".", "")
cmd.Flags().BoolVar(&verbose, "verbose", false, "")
return cmd
}

Expand All @@ -287,7 +310,7 @@ func NewFlowCommand() *cobra.Command {
SilenceUsage: true,
}
cmd.SetHelpFunc(executeHelp)
cmd.PersistentFlags().StringVarP(&environment, "env", "e", "", "")
cmd.PersistentFlags().BoolVar(&debug, "debug", false, "")
cmd.AddCommand(versionCommand())
cmd.AddCommand(aboutCommand())
cmd.AddCommand(initCommand())
Expand Down
21 changes: 15 additions & 6 deletions cmd/sql/flow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func TestFlowValidateCmd(t *testing.T) {
err := execFlowCmd("init", projectDir)
assert.NoError(t, err)

err = execFlowCmd("validate", projectDir, "--connection", "sqlite_conn")
err = execFlowCmd("validate", projectDir, "--connection", "sqlite_conn", "--verbose")
assert.NoError(t, err)
}

Expand All @@ -137,25 +137,25 @@ func TestFlowGenerateCmd(t *testing.T) {
err := execFlowCmd("init", projectDir)
assert.NoError(t, err)

err = execFlowCmd("generate", "example_basic_transform", "--project-dir", projectDir)
err = execFlowCmd("generate", "example_basic_transform", "--project-dir", projectDir, "--verbose")
assert.NoError(t, err)
}

func TestFlowGenerateGenerateTasksCmd(t *testing.T) {
projectDir := t.TempDir()
err := execFlowCmd([]string{"init", projectDir}...)
err := execFlowCmd("init", projectDir)
assert.NoError(t, err)

err = execFlowCmd([]string{"generate", "example_basic_transform", "--project-dir", projectDir, "--generate-tasks"}...)
err = execFlowCmd("generate", "example_basic_transform", "--project-dir", projectDir, "--generate-tasks")
assert.NoError(t, err)
}

func TestFlowRunGenerateTasksCmd(t *testing.T) {
projectDir := t.TempDir()
err := execFlowCmd([]string{"init", projectDir}...)
err := execFlowCmd("init", projectDir)
assert.NoError(t, err)

err = execFlowCmd([]string{"run", "example_basic_transform", "--project-dir", projectDir, "--generate-tasks"}...)
err = execFlowCmd("run", "example_basic_transform", "--project-dir", projectDir, "--generate-tasks")
assert.NoError(t, err)
}

Expand All @@ -177,6 +177,15 @@ func TestFlowRunCmd(t *testing.T) {
assert.NoError(t, err)
}

func TestDebugFlowRunCmd(t *testing.T) {
projectDir := t.TempDir()
err := execFlowCmd("init", projectDir)
assert.NoError(t, err)

err = execFlowCmd("--debug", "run", "example_templating", "--env", "dev", "--project-dir", projectDir)
assert.NoError(t, err)
}

func TestFlowRunCmdWorkflowNameNotSet(t *testing.T) {
projectDir := t.TempDir()
err := execFlowCmd("init", projectDir)
Expand Down
13 changes: 7 additions & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ require (
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.9.0
github.com/stretchr/testify v1.7.1
golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
golang.org/x/text v0.3.7 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.1.0
golang.org/x/sys v0.1.0
golang.org/x/text v0.4.0 // indirect
)

require (
Expand All @@ -37,8 +37,9 @@ require (
github.com/mitchellh/mapstructure v1.4.2
github.com/opencontainers/image-spec v1.0.2
github.com/whilp/git-urls v1.0.0
golang.org/x/mod v0.4.2
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
golang.org/x/exp v0.0.0-20221114172223-0cf76af32a3a
golang.org/x/mod v0.6.0
golang.org/x/term v0.1.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
Expand Down
25 changes: 14 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1101,8 +1101,8 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 h1:Tgea0cVUD0ivh5ADBX4WwuI12DUd2to3nCYe2eayMIw=
golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand All @@ -1113,6 +1113,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221114172223-0cf76af32a3a h1:21oggPEmhaCcoqxXRalB1CnlgrRfRZBYDXDGOiAjF+Y=
golang.org/x/exp v0.0.0-20221114172223-0cf76af32a3a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand All @@ -1137,8 +1139,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1192,8 +1195,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -1323,13 +1326,13 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -1338,8 +1341,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
10 changes: 7 additions & 3 deletions sql/flow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ var (
mockDisplayMessagesErr = func(r io.Reader) error {
return errMock
}
mockIoCopyNil = func(dst io.Writer, src io.Reader) (written int64, err error) {
return 0, nil
}
mockIoCopyErr = func(dst io.Writer, src io.Reader) (written int64, err error) {
return 0, errMock
}
Expand Down Expand Up @@ -247,11 +250,12 @@ func TestContainerRemoveFailure(t *testing.T) {
mockDockerBinder.On("ContainerRemove", mock.Anything, mock.Anything, mock.Anything).Return(errMock)
return mockDockerBinder, nil
}
ioCopy = func(dst io.Writer, src io.Reader) (written int64, err error) {
return 0, nil
}
IoCopy = mockIoCopyNil
DisplayMessages = mockDisplayMessagesNil
err := CommonDockerUtil(testCommand, nil, nil, nil)
expectedErr := fmt.Errorf("docker remove failed %w", errMock)
assert.Equal(t, expectedErr, err)
mockDockerBinder.AssertExpectations(t)
DisplayMessages = displayMessages
IoCopy = io.Copy
}
1 change: 1 addition & 0 deletions src/astro-sql-cli
Submodule astro-sql-cli added at 0173a0

0 comments on commit 9d2377f

Please sign in to comment.