Skip to content

Commit

Permalink
Add astro dev * aliased to astro airflow * (#253)
Browse files Browse the repository at this point in the history
* add astro dev * aliased to astro airflow *

* move astro logs to astro deployment logs

* update README.md

* make astro airflow is deprecated command

* replace astro airflow to astro dev

* remove dev from airflow aliases
  • Loading branch information
andriisoldatenko authored and schnie committed Jul 15, 2019
1 parent fa2f167 commit 63a3998
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 17 deletions.
11 changes: 6 additions & 5 deletions README.md
Expand Up @@ -9,11 +9,12 @@ Usage:
astro [command]
Available Commands:
airflow Manage airflow projects and deployments
auth Manage astronomer identity
cluster Manage Astronomer EE clusters
config Manage astro project configurations
deploy Deploy an airflow project
deployment Manage airflow deployments
dev Manage airflow projects and deployments
help Help about any command
service-account Manage astronomer service accounts
upgrade Check for newer version of Astronomer CLI
Expand Down Expand Up @@ -93,7 +94,7 @@ $ astro

```
$ mkdir hello-astro && cd hello-astro
$ astro airflow init
$ astro dev init
```

This will generate a skeleton project directory:
Expand All @@ -113,7 +114,7 @@ Dags can go in the `dags` folder, custom airflow plugins in `plugins`, python pa

1. Start airflow

Run `astro airflow start` to start a local version of airflow on your machine. This will spin up a few locally running docker containers - one for the airflow scheduler, one for the webserver, and one for postgres.
Run `astro dev start` to start a local version of airflow on your machine. This will spin up a few locally running docker containers - one for the airflow scheduler, one for the webserver, and one for postgres.
(Run `docker ps` to verify)

## Help
Expand All @@ -129,11 +130,11 @@ $ astro help
Or for subcommands:

```
$ astro airflow --help
$ astro dev --help
```

```
$ astro airflow deploy --help
$ astro deploy --help
```

## Development
Expand Down
28 changes: 24 additions & 4 deletions cmd/airflow.go
Expand Up @@ -33,13 +33,22 @@ var (

RunExample = `
# Create default admin user.
astro airflow run create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin
astro dev run create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin
`

airflowRootCmd = &cobra.Command{
Use: "airflow",
Short: "Manage airflow projects and deployments",
Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment",
Use: "airflow",
Aliases: []string{"a"},
Short: "Manage airflow projects",
Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment",
Deprecated: "could please use new command instead `astro dev [subcommands] [flags]`",
}

devRootCmd = &cobra.Command{
Use: "dev",
Aliases: []string{"d"},
Short: "Manage airflow projects",
Long: "Airflow projects are a single top-level directory which represents a single production Airflow deployment",
}

airflowInitCmd = &cobra.Command{
Expand Down Expand Up @@ -116,41 +125,52 @@ astro airflow run create_user -r Admin -u admin -e admin@example.com -f admin -l

func init() {
// Airflow root
// Make sure after 1.0 we have only devRootCmd
RootCmd.AddCommand(devRootCmd)
RootCmd.AddCommand(airflowRootCmd)

// Airflow init
airflowInitCmd.Flags().StringVarP(&projectName, "name", "n", "", "Name of airflow project")
airflowInitCmd.Flags().StringVarP(&airflowVersion, "airflow-version", "v", "", "Version of airflow you want to deploy")

airflowRootCmd.AddCommand(airflowInitCmd)
devRootCmd.AddCommand(airflowInitCmd)

// Airflow deploy
airflowRootCmd.AddCommand(airflowDeployCmd)
devRootCmd.AddCommand(airflowDeployCmd)
airflowDeployCmd.Flags().BoolVarP(&forceDeploy, "force", "f", false, "Force deploy if uncommitted changes")
airflowDeployCmd.Flags().BoolVarP(&forcePrompt, "prompt", "p", false, "Force prompt to choose target deployment")
airflowDeployCmd.Flags().BoolVarP(&saveDeployConfig, "save", "s", false, "Save deployment in config for future deploys")
airflowDeployCmd.Flags().StringVar(&workspaceId, "workspace-id", "", "workspace assigned to deployment")

// Airflow start
airflowRootCmd.AddCommand(airflowStartCmd)
devRootCmd.AddCommand(airflowStartCmd)
airflowStartCmd.Flags().StringVarP(&envFile, "env", "e", ".env", "Location of file containing environment variables")

// Airflow kill
airflowRootCmd.AddCommand(airflowKillCmd)
devRootCmd.AddCommand(airflowKillCmd)

// Airflow logs
airflowRootCmd.AddCommand(airflowLogsCmd)
devRootCmd.AddCommand(airflowLogsCmd)
airflowLogsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Follow log output")
airflowLogsCmd.Flags().BoolVarP(&schedulerLogs, "scheduler", "s", false, "Output scheduler logs")
airflowLogsCmd.Flags().BoolVarP(&webserverLogs, "webserver", "w", false, "Output webserver logs")

// Airflow stop
airflowRootCmd.AddCommand(airflowStopCmd)
devRootCmd.AddCommand(airflowStopCmd)

// Airflow PS
airflowRootCmd.AddCommand(airflowPSCmd)
devRootCmd.AddCommand(airflowPSCmd)

// Airflow Run
airflowRootCmd.AddCommand(airflowRunCmd)
devRootCmd.AddCommand(airflowRunCmd)
}

func ensureProjectDir(cmd *cobra.Command, args []string) error {
Expand Down
40 changes: 40 additions & 0 deletions cmd/airflow_test.go
@@ -0,0 +1,40 @@
package cmd

import (
"bytes"
"testing"

"github.com/spf13/cobra"
)

func executeCommandC(root *cobra.Command, args ...string) (c *cobra.Command, output string, err error) {
buf := new(bytes.Buffer)
root.SetArgs(args)
c, err = root.ExecuteC()
return c, buf.String(), err
}

func executeCommand(root *cobra.Command, args ...string) (output string, err error) {
_, output, err = executeCommandC(root, args...)
return output, err
}

func TestDevCommand(t *testing.T) {
output, err := executeCommand(RootCmd, "dev")
if output != "" {
t.Errorf("Unexpected output: %v", output)
}
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}

func TestAirflowCommand(t *testing.T) {
output, err := executeCommand(RootCmd, "airflow")
if output != "" {
t.Errorf("Unexpected output: %v", output)
}
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
3 changes: 3 additions & 0 deletions cmd/deployment.go
Expand Up @@ -78,6 +78,9 @@ func init() {

// deployment update
deploymentRootCmd.AddCommand(deploymentUpdateCmd)

// deployment logs
deploymentRootCmd.AddCommand(logsCmd)
}

func deploymentCreate(cmd *cobra.Command, args []string) error {
Expand Down
16 changes: 16 additions & 0 deletions cmd/deployment_test.go
@@ -0,0 +1,16 @@
package cmd

import (
"testing"
)


func TestDeploymentLogsCommand(t *testing.T) {
output, err := executeCommand(deploymentRootCmd, "logs")
if output != "" {
t.Errorf("Unexpected output: %v", output)
}
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
28 changes: 20 additions & 8 deletions cmd/logs.go
Expand Up @@ -12,16 +12,16 @@ var (
since time.Duration
logsExample = `
# Return logs for last 5 minutes of webserver logs and output them.
astro logs webserver example-deployment-uuid
astro deployment logs webserver example-deployment-uuid
# Subscribe logs from airflow workers for last 5 min and specify search term, and subscribe to more.
astro logs workers example-deployment-uuid --follow --search "some search terms"
astro deployment logs workers example-deployment-uuid --follow --search "some search terms"
# Return logs from airflow webserver for last 25 min.
astro logs webserver example-deployment-uuid --since 25m
astro deployment logs webserver example-deployment-uuid --since 25m
# Subscribe logs from airflow scheduler.
astro logs scheduler example-deployment-uuid -f
astro deployment logs scheduler example-deployment-uuid -f
`

logsCmd = &cobra.Command{
Expand All @@ -32,13 +32,22 @@ var (
Example: logsExample,
}

logsDeprecatedCmd = &cobra.Command{
Use: "logs",
Aliases: []string{"log", "l"},
Short: "Stream logs from an Airflow deployment",
Long: "Stream logs from an Airflow deployment",
Example: logsExample,
Deprecated: "could please use new command instead `astro deployment logs [subcommands] [flags]`",
}

webserverLogsCmd = &cobra.Command{
Use: "webserver",
Aliases: []string{"web", "w"},
Short: "Stream logs from an Airflow webserver",
Long: `Stream logs from an Airflow webserver. For example:
astro logs webserver YOU_DEPLOYMENT_ID -s string-to-find
astro deployment logs webserver YOU_DEPLOYMENT_ID -s string-to-find
`,
Args: cobra.ExactArgs(1),
RunE: webserverRemoteLogs,
Expand All @@ -50,7 +59,7 @@ astro logs webserver YOU_DEPLOYMENT_ID -s string-to-find
Short: "Stream logs from an Airflow scheduler",
Long: `Stream logs from an Airflow scheduler. For example:
astro logs scheduler YOU_DEPLOYMENT_ID -s string-to-find
astro deployment logs scheduler YOU_DEPLOYMENT_ID -s string-to-find
`,
Args: cobra.ExactArgs(1),
RunE: schedulerRemoteLogs,
Expand All @@ -62,36 +71,39 @@ astro logs scheduler YOU_DEPLOYMENT_ID -s string-to-find
Short: "Stream logs from Airflow workers",
Long: `Stream logs from Airflow workers. For example:
astro logs workers YOU_DEPLOYMENT_ID -s string-to-find
astro deployment logs workers YOU_DEPLOYMENT_ID -s string-to-find
`,
Args: cobra.ExactArgs(1),
RunE: workersRemoteLogs,
}
)

func init() {
RootCmd.AddCommand(logsCmd)
RootCmd.AddCommand(logsDeprecatedCmd)
webserverLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs")
webserverLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs")
webserverLogsCmd.Flags().DurationVarP(&since, "since", "t", 0, "Only return logs newer than a relative duration like 5m, 1h, or 24h")
webserverLogsCmd.Flags().BoolP("help", "h", false, "Help for " + webserverLogsCmd.Name())

// get airflow webserver logs
logsCmd.AddCommand(webserverLogsCmd)
logsDeprecatedCmd.AddCommand(webserverLogsCmd)

workersLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs")
workersLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs")
workersLogsCmd.Flags().DurationVarP(&since, "since", "t", 0, "Only return logs newer than a relative duration like 5m, 1h, or 24h")
workersLogsCmd.Flags().BoolP("help", "h", false, "Help for " + workersLogsCmd.Name())
// get airflow workers logs
logsCmd.AddCommand(workersLogsCmd)
logsDeprecatedCmd.AddCommand(workersLogsCmd)

schedulerLogsCmd.Flags().StringVarP(&search, "search", "s", "", "Search term inside logs")
schedulerLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Subscribe to watch more logs")
schedulerLogsCmd.Flags().DurationVarP(&since, "since", "t", 0, "Only return logs newer than a relative duration like 5m, 1h, or 24h")
schedulerLogsCmd.Flags().BoolP("help", "h", false, "Help for " + schedulerLogsCmd.Name())
// get airflow scheduler logs
logsCmd.AddCommand(schedulerLogsCmd)
logsDeprecatedCmd.AddCommand(schedulerLogsCmd)
}

func webserverRemoteLogs(cmd *cobra.Command, args []string) error {
Expand Down
16 changes: 16 additions & 0 deletions cmd/logs_test.go
@@ -0,0 +1,16 @@
package cmd

import (
"testing"
)


func TestLogsCommand(t *testing.T) {
output, err := executeCommand(RootCmd, "logs")
if output != "" {
t.Errorf("Unexpected output: %v", output)
}
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
}

0 comments on commit 63a3998

Please sign in to comment.