Add basic start-job-run functionality.

cmd/main.go

@@ -13,7 +13,7 @@ import (
 func main() {
 	adhesiveCli, err := command.NewAdhesiveCli("")
 	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
 		os.Exit(1)
 	}
 
@@ -24,7 +24,7 @@ func main() {
 			os.Exit(exitErr.ExitCode())
 		}
 
-		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err)
 		os.Exit(1)
 	}
 }

internal/cli/cobra.go

@@ -3,9 +3,11 @@ package cli
 import (
 	"github.com/mana-sys/adhesive/internal/cli/command"
 	"github.com/mana-sys/adhesive/internal/cli/command/deploy"
+	"github.com/mana-sys/adhesive/internal/cli/command/historyserver"
 	"github.com/mana-sys/adhesive/internal/cli/command/local"
 	package1 "github.com/mana-sys/adhesive/internal/cli/command/package"
 	"github.com/mana-sys/adhesive/internal/cli/command/remove"
+	"github.com/mana-sys/adhesive/internal/cli/command/startjobrun"
 	"github.com/mana-sys/adhesive/internal/cli/version"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -43,6 +45,8 @@ func NewRootCommand(adhesiveCli *command.AdhesiveCli) *cobra.Command {
 		local.NewLocalCommand(),
 		package1.NewPackageCommand(adhesiveCli),
 		remove.NewRemoveCommand(adhesiveCli),
+		historyserver.NewHistoryServerCommand(adhesiveCli),
+		startjobrun.NewStartJobRunCommand(adhesiveCli),
 	)
 
 	return cmd

internal/cli/command/cli.go

@@ -3,6 +3,8 @@ package command
 import (
 	"os"
 
+	"github.com/aws/aws-sdk-go/service/glue"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
@@ -23,8 +25,9 @@ type AdhesiveCli struct {
 
 	FoundConfigFile bool
 
-	cfn *cloudformation.CloudFormation
-	s3  *s3.S3
+	cfn  *cloudformation.CloudFormation
+	s3   *s3.S3
+	glue *glue.Glue
 }
 
 func NewAdhesiveCli(path string) (*AdhesiveCli, error) {
@@ -65,6 +68,7 @@ func (cli *AdhesiveCli) InitializeClients() error {
 
 	cli.cfn = cloudformation.New(sess)
 	cli.s3 = s3.New(sess)
+	cli.glue = glue.New(sess)
 	return nil
 }
 
@@ -75,3 +79,7 @@ func (cli *AdhesiveCli) S3() *s3.S3 {
 func (cli *AdhesiveCli) CloudFormation() *cloudformation.CloudFormation {
 	return cli.cfn
 }
+
+func (cli *AdhesiveCli) Glue() *glue.Glue {
+	return cli.glue
+}

internal/cli/command/historyserver/cmd.go

@@ -1,16 +1,11 @@
 package historyserver
 
 import (
-	"errors"
-	"os"
-	"os/exec"
-	"path/filepath"
-
 	"github.com/mana-sys/adhesive/internal/cli/command"
 	"github.com/spf13/cobra"
 )
 
-func NewHistoryServer(adhesiveCli *command.AdhesiveCli) *cobra.Command {
+func NewHistoryServerCommand(adhesiveCli *command.AdhesiveCli) *cobra.Command {
 	opts := &adhesiveCli.Config.HistoryServer
 
 	cmd := &cobra.Command{
@@ -27,43 +22,43 @@ func NewHistoryServer(adhesiveCli *command.AdhesiveCli) *cobra.Command {
 	return cmd
 }
 
-// buildDockerCommand builds an exec.Cmd to run the Docker container with the provided options.
-func buildDockerCommand(entrypoint string, args []string) (*exec.Cmd, error) {
-	var (
-		err  error
-		envs []string
-		vols []string
-	)
-
-	for _, env := range options.env {
-		envs = append(envs, "-e "+env)
-	}
-
-	for _, vol := range options.volumes {
-		vols = append(vols, "-v "+vol)
-	}
-
-	credsDir := options.credentials
-	if credsDir == "" {
-		if credsDir, err = os.UserHomeDir(); err != nil {
-			return nil, errors.New("unable to determine home directory")
-		}
-		credsDir = filepath.Join(credsDir, ".aws")
-	}
-
-	wd, err := os.Getwd()
-
-	dockerArgs := []string{"run", "--rm", "-t"}
-	dockerArgs = append(dockerArgs, envs...)
-	dockerArgs = append(dockerArgs, vols...)
-	dockerArgs = append(dockerArgs, "-v", credsDir+":/root/.aws",
-		"-v", DistPackagesVolume+":"+DistPackagesDirectory,
-		"-v", wd+":/project",
-		"--entrypoint", entrypoint, DockerImageName+":0.9")
-	dockerArgs = append(dockerArgs, args...)
-
-	return exec.Command("docker", dockerArgs...), nil
-}
+//// buildDockerCommand builds an exec.Cmd to run the Docker container with the provided options.
+//func buildDockerCommand(entrypoint string, args []string) (*exec.Cmd, error) {
+//	var (
+//		err  error
+//		envs []string
+//		vols []string
+//	)
+//
+//	for _, env := range options.env {
+//		envs = append(envs, "-e "+env)
+//	}
+//
+//	for _, vol := range options.volumes {
+//		vols = append(vols, "-v "+vol)
+//	}
+//
+//	credsDir := options.credentials
+//	if credsDir == "" {
+//		if credsDir, err = os.UserHomeDir(); err != nil {
+//			return nil, errors.New("unable to determine home directory")
+//		}
+//		credsDir = filepath.Join(credsDir, ".aws")
+//	}
+//
+//	wd, err := os.Getwd()
+//
+//	dockerArgs := []string{"run", "--rm", "-t"}
+//	dockerArgs = append(dockerArgs, envs...)
+//	dockerArgs = append(dockerArgs, vols...)
+//	dockerArgs = append(dockerArgs, "-v", credsDir+":/root/.aws",
+//		"-v", DistPackagesVolume+":"+DistPackagesDirectory,
+//		"-v", wd+":/project",
+//		"--entrypoint", entrypoint, DockerImageName+":0.9")
+//	dockerArgs = append(dockerArgs, args...)
+//
+//	return exec.Command("docker", dockerArgs...), nil
+//}
 
 func historyServer(adhesiveCli *command.AdhesiveCli) error {
 	return nil

internal/cli/command/local/docker.go

@@ -2,13 +2,9 @@ package local
 
 import (
 	"errors"
-	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"sync"
-
-	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -62,45 +58,15 @@ func buildAndRunDockerCommand(entrypoint string, options *dockerOptions, args []
 		return err
 	}
 
-	stdout, err := dockerCmd.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	stderr, err := dockerCmd.StderrPipe()
-	if err != nil {
-		return err
-	}
 	dockerCmd.Stdin = os.Stdin
-
-	var wg sync.WaitGroup
-
-	go func() {
-		defer stdout.Close()
-		io.Copy(os.Stdout, stdout)
-		log.Debugf("End of stdout")
-		wg.Done()
-	}()
-	go func() {
-		defer stdout.Close()
-		io.Copy(os.Stderr, stderr)
-		log.Debugf("End of stderr")
-		wg.Done()
-	}()
-	//go func() {
-	//	defer stdin.Close()
-	//	io.Copy(stdin, os.Stdin)
-	//	log.Debugf("End of stdin")
-	//	wg.Done()
-	//}()
-
-	wg.Add(2)
+	dockerCmd.Stdout = os.Stdout
+	dockerCmd.Stderr = os.Stderr
 
 	if err = dockerCmd.Start(); err != nil {
 		return err
 	}
 
 	err = dockerCmd.Wait()
-	wg.Wait()
 
 	return err
 }

internal/cli/command/local/pip.go

@@ -10,7 +10,7 @@ func NewPipCommand(opts *dockerOptions) *cobra.Command {
 		Short:              "Install Python dependencies for local job runs",
 		DisableFlagParsing: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return buildAndRunDockerCommand("pytest", opts, args)
+			return buildAndRunDockerCommand("pip", opts, args)
 		},
 	}
 

internal/cli/command/startjobrun/cmd.go

@@ -0,0 +1,74 @@
+package startjobrun
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudformation"
+	"github.com/aws/aws-sdk-go/service/glue"
+	"github.com/mana-sys/adhesive/internal/cli/command"
+	"github.com/mana-sys/adhesive/internal/cli/config"
+	"github.com/spf13/cobra"
+)
+
+func NewStartJobRunCommand(adhesiveCli *command.AdhesiveCli) *cobra.Command {
+	opts := &adhesiveCli.Config.StartJobRun
+
+	cmd := &cobra.Command{
+		Use:   "start-job-run",
+		Short: "Remove the current deployment of your CloudFormation template.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return startJobRun(adhesiveCli, opts)
+		},
+	}
+
+	flags := cmd.Flags()
+	flags.StringVar(&opts.StackName, "stack-name", opts.StackName, "The name of the CloudFormation stack to remove")
+	flags.StringVar(&opts.JobName, "job-name", opts.JobName, "The name of the Glue job to run. This can also be the logical resource ID.")
+
+	return cmd
+}
+
+func startJobRun(adhesiveCli *command.AdhesiveCli, opts *config.StartJobRunOptions) error {
+
+	if err := adhesiveCli.InitializeClients(); err != nil {
+		return err
+	}
+
+	glu := adhesiveCli.Glue()
+	cfn := adhesiveCli.CloudFormation()
+
+	name := opts.JobName
+
+	if name == "" {
+		return errors.New("option --job-name is required")
+	}
+
+	// If stack-name is specified, then interpret job-name as the logical resource ID
+	// of the job to be run.
+	if opts.StackName != "" {
+		out, err := cfn.DescribeStackResource(&cloudformation.DescribeStackResourceInput{
+			StackName:         aws.String(opts.StackName),
+			LogicalResourceId: aws.String(opts.JobName),
+		})
+		if err != nil {
+			return err
+		}
+
+		if *out.StackResourceDetail.ResourceType != "AWS::Glue::Job" {
+			return errors.New(name + " is not of type AWS::Glue::Job")
+		}
+
+		name = *out.StackResourceDetail.PhysicalResourceId
+	}
+
+	_, err := glu.StartJobRun(&glue.StartJobRunInput{
+		JobName: aws.String(name),
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/cli/config/config.go

@@ -31,8 +31,16 @@ type RemoveOptions struct {
 	StackName string `toml:"stack-name"`
 }
 
+type StartJobRunOptions struct {
+	JobName   string `toml:"job-name"`
+	JobRunID  string `toml:"job-run-id"`
+	StackName string `toml:"stack-name"`
+	TailLogs  bool   `toml:"tail-logs"`
+}
+
 type Config struct {
 	Deploy        DeployOptions
+	StartJobRun   StartJobRunOptions
 	Local         LocalOptions
 	Package       PackageOptions
 	Remove        RemoveOptions