From 63a3998c3f9ae487fee8353989998b22564a3529 Mon Sep 17 00:00:00 2001 From: Andrii Soldatenko Date: Mon, 15 Jul 2019 21:24:02 +0300 Subject: [PATCH] Add astro dev * aliased to astro airflow * (#253) * add astro dev * aliased to astro airflow * * move astro logs to astro deployment logs * update README.md * make astro airflow is deprecated command * replace astro airflow to astro dev * remove dev from airflow aliases --- README.md | 11 ++++++----- cmd/airflow.go | 28 ++++++++++++++++++++++++---- cmd/airflow_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ cmd/deployment.go | 3 +++ cmd/deployment_test.go | 16 ++++++++++++++++ cmd/logs.go | 28 ++++++++++++++++++++-------- cmd/logs_test.go | 16 ++++++++++++++++ 7 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 cmd/airflow_test.go create mode 100644 cmd/deployment_test.go create mode 100644 cmd/logs_test.go diff --git a/README.md b/README.md index fe84094a9..802038133 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,12 @@ Usage: astro [command] Available Commands: - airflow Manage airflow projects and deployments auth Manage astronomer identity cluster Manage Astronomer EE clusters config Manage astro project configurations + deploy Deploy an airflow project deployment Manage airflow deployments + dev Manage airflow projects and deployments help Help about any command service-account Manage astronomer service accounts upgrade Check for newer version of Astronomer CLI @@ -93,7 +94,7 @@ $ astro ``` $ mkdir hello-astro && cd hello-astro -$ astro airflow init +$ astro dev init ``` This will generate a skeleton project directory: @@ -113,7 +114,7 @@ Dags can go in the `dags` folder, custom airflow plugins in `plugins`, python pa 1. Start airflow -Run `astro airflow start` to start a local version of airflow on your machine. This will spin up a few locally running docker containers - one for the airflow scheduler, one for the webserver, and one for postgres. +Run `astro dev start` to start a local version of airflow on your machine. This will spin up a few locally running docker containers - one for the airflow scheduler, one for the webserver, and one for postgres. (Run `docker ps` to verify) ## Help @@ -129,11 +130,11 @@ $ astro help Or for subcommands: ``` -$ astro airflow --help +$ astro dev --help ``` ``` -$ astro airflow deploy --help +$ astro deploy --help ``` ## Development diff --git a/cmd/airflow.go b/cmd/airflow.go index eb979691a..e0f909386 100644 --- a/cmd/airflow.go +++ b/cmd/airflow.go @@ -33,13 +33,22 @@ var ( RunExample = ` # Create default admin user. -astro airflow run create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin +astro dev run create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin ` airflowRootCmd = &cobra.Command{ - Use: "airflow", - Short: "Manage airflow projects and deployments", - Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment", + Use: "airflow", + Aliases: []string{"a"}, + Short: "Manage airflow projects", + Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment", + Deprecated: "could please use new command instead `astro dev [subcommands] [flags]`", + } + + devRootCmd = &cobra.Command{ + Use: "dev", + Aliases: []string{"d"}, + Short: "Manage airflow projects", + Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment", } airflowInitCmd = &cobra.Command{ @@ -116,15 +125,20 @@ astro airflow run create_user -r Admin -u admin -e admin@example.com -f admin -l func init() { // Airflow root + // Make sure after 1.0 we have only devRootCmd + RootCmd.AddCommand(devRootCmd) RootCmd.AddCommand(airflowRootCmd) // Airflow init airflowInitCmd.Flags().StringVarP(&projectName, "name", "n", "", "Name of airflow project") airflowInitCmd.Flags().StringVarP(&airflowVersion, "airflow-version", "v", "", "Version of airflow you want to deploy") + airflowRootCmd.AddCommand(airflowInitCmd) + devRootCmd.AddCommand(airflowInitCmd) // Airflow deploy airflowRootCmd.AddCommand(airflowDeployCmd) + devRootCmd.AddCommand(airflowDeployCmd) airflowDeployCmd.Flags().BoolVarP(&forceDeploy, "force", "f", false, "Force deploy if uncommitted changes") airflowDeployCmd.Flags().BoolVarP(&forcePrompt, "prompt", "p", false, "Force prompt to choose target deployment") airflowDeployCmd.Flags().BoolVarP(&saveDeployConfig, "save", "s", false, "Save deployment in config for future deploys") @@ -132,25 +146,31 @@ func init() { // Airflow start airflowRootCmd.AddCommand(airflowStartCmd) + devRootCmd.AddCommand(airflowStartCmd) airflowStartCmd.Flags().StringVarP(&envFile, "env", "e", ".env", "Location of file containing environment variables") // Airflow kill airflowRootCmd.AddCommand(airflowKillCmd) + devRootCmd.AddCommand(airflowKillCmd) // Airflow logs airflowRootCmd.AddCommand(airflowLogsCmd) + devRootCmd.AddCommand(airflowLogsCmd) airflowLogsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Follow log output") airflowLogsCmd.Flags().BoolVarP(&schedulerLogs, "scheduler", "s", false, "Output scheduler logs") airflowLogsCmd.Flags().BoolVarP(&webserverLogs, "webserver", "w", false, "Output webserver logs") // Airflow stop airflowRootCmd.AddCommand(airflowStopCmd) + devRootCmd.AddCommand(airflowStopCmd) // Airflow PS airflowRootCmd.AddCommand(airflowPSCmd) + devRootCmd.AddCommand(airflowPSCmd) // Airflow Run airflowRootCmd.AddCommand(airflowRunCmd) + devRootCmd.AddCommand(airflowRunCmd) } func ensureProjectDir(cmd *cobra.Command, args []string) error { diff --git a/cmd/airflow_test.go b/cmd/airflow_test.go new file mode 100644 index 000000000..93fe11e18 --- /dev/null +++ b/cmd/airflow_test.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "bytes" + "testing" + + "github.com/spf13/cobra" +) + +func executeCommandC(root *cobra.Command, args ...string) (c *cobra.Command, output string, err error) { + buf := new(bytes.Buffer) + root.SetArgs(args) + c, err = root.ExecuteC() + return c, buf.String(), err +} + +func executeCommand(root *cobra.Command, args ...string) (output string, err error) { + _, output, err = executeCommandC(root, args...) + return output, err +} + +func TestDevCommand(t *testing.T) { + output, err := executeCommand(RootCmd, "dev") + if output != "" { + t.Errorf("Unexpected output: %v", output) + } + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} + +func TestAirflowCommand(t *testing.T) { + output, err := executeCommand(RootCmd, "airflow") + if output != "" { + t.Errorf("Unexpected output: %v", output) + } + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} diff --git a/cmd/deployment.go b/cmd/deployment.go index cff15e4f9..8a000c10a 100644 --- a/cmd/deployment.go +++ b/cmd/deployment.go @@ -78,6 +78,9 @@ func init() { // deployment update deploymentRootCmd.AddCommand(deploymentUpdateCmd) + + // deployment logs + deploymentRootCmd.AddCommand(logsCmd) } func deploymentCreate(cmd *cobra.Command, args []string) error { diff --git a/cmd/deployment_test.go b/cmd/deployment_test.go new file mode 100644 index 000000000..f5b79acbd --- /dev/null +++ b/cmd/deployment_test.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "testing" +) + + +func TestDeploymentLogsCommand(t *testing.T) { + output, err := executeCommand(deploymentRootCmd, "logs") + if output != "" { + t.Errorf("Unexpected output: %v", output) + } + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} diff --git a/cmd/logs.go b/cmd/logs.go index 4c9f0d305..61abe4da5 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -12,16 +12,16 @@ var ( since time.Duration logsExample = ` # Return logs for last 5 minutes of webserver logs and output them. - astro logs webserver example-deployment-uuid + astro deployment logs webserver example-deployment-uuid # Subscribe logs from airflow workers for last 5 min and specify search term, and subscribe to more. - astro logs workers example-deployment-uuid --follow --search "some search terms" + astro deployment logs workers example-deployment-uuid --follow --search "some search terms" # Return logs from airflow webserver for last 25 min. - astro logs webserver example-deployment-uuid --since 25m + astro deployment logs webserver example-deployment-uuid --since 25m # Subscribe logs from airflow scheduler. - astro logs scheduler example-deployment-uuid -f + astro deployment logs scheduler example-deployment-uuid -f ` logsCmd = &cobra.Command{ @@ -32,13 +32,22 @@ var ( Example: logsExample, } + logsDeprecatedCmd = &cobra.Command{ + Use: "logs", + Aliases: []string{"log", "l"}, + Short: "Stream logs from an Airflow deployment", + Long: "Stream logs from an Airflow deployment", + Example: logsExample, + Deprecated: "could please use new command instead `astro deployment logs [subcommands] [flags]`", + } + webserverLogsCmd = &cobra.Command{ Use: "webserver", Aliases: []string{"web", "w"}, Short: "Stream logs from an Airflow webserver", Long: `Stream logs from an Airflow webserver. For example: -astro logs webserver YOU_DEPLOYMENT_ID -s string-to-find +astro deployment logs webserver YOU_DEPLOYMENT_ID -s string-to-find `, Args: cobra.ExactArgs(1), RunE: webserverRemoteLogs, @@ -50,7 +59,7 @@ astro logs webserver YOU_DEPLOYMENT_ID -s string-to-find Short: "Stream logs from an Airflow scheduler", Long: `Stream logs from an Airflow scheduler. For example: -astro logs scheduler YOU_DEPLOYMENT_ID -s string-to-find +astro deployment logs scheduler YOU_DEPLOYMENT_ID -s string-to-find `, Args: cobra.ExactArgs(1), RunE: schedulerRemoteLogs, @@ -62,7 +71,7 @@ astro logs scheduler YOU_DEPLOYMENT_ID -s string-to-find Short: "Stream logs from Airflow workers", Long: `Stream logs from Airflow workers. For example: -astro logs workers YOU_DEPLOYMENT_ID -s string-to-find +astro deployment logs workers YOU_DEPLOYMENT_ID -s string-to-find `, Args: cobra.ExactArgs(1), RunE: workersRemoteLogs, @@ -70,7 +79,7 @@ astro logs workers YOU_DEPLOYMENT_ID -s string-to-find ) func init() { - RootCmd.AddCommand(logsCmd) + RootCmd.AddCommand(logsDeprecatedCmd) webserverLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs") webserverLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs") webserverLogsCmd.Flags().DurationVarP(&since, "since", "t", 0, "Only return logs newer than a relative duration like 5m, 1h, or 24h") @@ -78,6 +87,7 @@ func init() { // get airflow webserver logs logsCmd.AddCommand(webserverLogsCmd) + logsDeprecatedCmd.AddCommand(webserverLogsCmd) workersLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs") workersLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs") @@ -85,6 +95,7 @@ func init() { workersLogsCmd.Flags().BoolP("help", "h", false, "Help for " + workersLogsCmd.Name()) // get airflow workers logs logsCmd.AddCommand(workersLogsCmd) + logsDeprecatedCmd.AddCommand(workersLogsCmd) schedulerLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs") schedulerLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs") @@ -92,6 +103,7 @@ func init() { schedulerLogsCmd.Flags().BoolP("help", "h", false, "Help for " + schedulerLogsCmd.Name()) // get airflow scheduler logs logsCmd.AddCommand(schedulerLogsCmd) + logsDeprecatedCmd.AddCommand(schedulerLogsCmd) } func webserverRemoteLogs(cmd *cobra.Command, args []string) error { diff --git a/cmd/logs_test.go b/cmd/logs_test.go new file mode 100644 index 000000000..7dfe14b02 --- /dev/null +++ b/cmd/logs_test.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "testing" +) + + +func TestLogsCommand(t *testing.T) { + output, err := executeCommand(RootCmd, "logs") + if output != "" { + t.Errorf("Unexpected output: %v", output) + } + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +}