feat: have one binary that does it all (#8)

- opctl and optimus binaries are combined into one that is optimus
- configuration file used so far with the name optimus.yaml is renamed to .optimus.yaml
- added documentation for configuring optimus server and client
- server environment variables have changed, please go through documentation
- fix plugins being left open even after main application terminates

.gitignore

@@ -13,15 +13,13 @@ TODO
 version
 temp
 coverage.out
-/opctl
 jobs
 /resources/resource_fs.go
 /auth.json
 /development.env
 /resources/resource_fs.go
 __pycache__
-
-optimus.yaml
+.optimus.yaml
 
 # git ignore generate files related to gRPC and proto
 /proton/

.goreleaser.yml

@@ -7,25 +7,7 @@ before:
     - make install
     - make generate
 builds:
-  - main: ./cmd/opctl/main.go
-    id: "opctl"
-    binary: opctl
-    flags:
-      - -a
-    ldflags:
-      - -s -w -X github.com/odpf/optimus/config.Version={{.Tag}} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}}
-    goos:
-      - linux
-      - darwin
-      - windows
-    goarch:
-      - amd64
-      - 386
-      - arm
-      - arm64
-    env:
-      - CGO_ENABLED=0
-  - main: ./cmd/optimus/main.go
+  - main: ./main.go
     id: "optimus"
     binary: optimus
     flags:
@@ -89,7 +71,7 @@ brews:
     dependencies:
       - name: git
     install: |-
-      bin.install "opctl"
+      bin.install "optimus"
     commit_author:
       name: Kush Sharma
       email: 3647166+kushsharma@users.noreply.github.com

Makefile

@@ -2,22 +2,17 @@
 .DELETE_ON_ERROR:
 MAKEFLAGS += --no-builtin-rules
 NAME = "github.com/odpf/optimus"
-CTL_VERSION := "$(shell git rev-parse --short HEAD)"
-OPMS_VERSION := "$(shell git rev-parse --short HEAD)"
+LAST_COMMIT := $(shell git rev-parse --short HEAD)
+LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
+OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
 
 all: build
 
-.PHONY: build build-optimus smoke-test unit-test test clean generate dist init vet
+.PHONY: build smoke-test unit-test test clean generate dist init vet
 
-build-ctl: generate ## generate opctl
-	@echo " > building opctl version ${CTL_VERSION}"
-	@go build -ldflags "-X config.Version=${CTL_VERSION}" ${NAME}/cmd/opctl
-
-build-optimus: generate ## generate optimus server
+build: generate # build optimus binary
 	@echo " > building optimus version ${OPMS_VERSION}"
-	@go build -ldflags "-X 'config.Version=${OPMS_VERSION}'" ${NAME}/cmd/optimus
-
-build: build-optimus build-ctl
+	@go build -ldflags "-X ${NAME}/config.Version=${OPMS_VERSION} -X ${NAME}/config.BuildCommit=${LAST_COMMIT}" -o optimus .
 	@echo " - build complete"
 
 test: smoke-test unit-test vet ## run tests
@@ -39,7 +34,7 @@ generate-proto: ## regenerate protos
 unit-test:
 	go list ./... | grep -v -e third_party -e api/proto | xargs go test -count 1 -cover -race -timeout 1m -tags=unit_test
 
-smoke-test: build-ctl
+smoke-test: build
 	@bash ./scripts/smoke-test.sh
 
 integration-test: build
@@ -55,7 +50,7 @@ dist: generate
 	@bash ./scripts/build-distributables.sh
 
 clean:
-	rm -rf ./optimus ./opctl ./dist ./proton ./api/proto/*
+	rm -rf ./optimus ./dist ./proton ./api/proto/*
 
 install: ## install required dependencies
 	@echo "> installing dependencies"

README.md

@@ -30,7 +30,7 @@ Optimus helps your organization to build & manage data pipelines with ease.
 ## Getting Started
 
 Optimus has two components, Optimus service that is the core orchestrator installed
-on server side, and a CLI binary(`opctl`) used to interact with this service.
+on server side, and a CLI binary used to interact with this service.
 
 ### Compiling from source
 
@@ -40,52 +40,33 @@ Optimus requires the following dependencies:
 * Golang (version 1.16 or above)
 * Git
 
-#### Opctl
-
-Run the following commands to compile `opctl` from source
-```shell
-git clone git@github.com:odpf/optimus.git
-cd optimus
-make build-ctl
-cp ./opctl /usr/bin # optional - copy the executables to a location in $PATH
-```
-The last step isn't necessarily required. Feel free to put the compiled executable anywhere you want.
-
-Test if `opctl` is working as expected
-```shell
-./opctl version
-```
-
-#### Optimus
-
 Run the following commands to compile `optimus` from source
 ```shell
 git clone git@github.com:odpf/optimus.git
 cd optimus
-make build-optimus
+make build
 ```
 Use the following command to run
 ```shell
-./optimus
+./optimus version
 ```
-Note: without required arguments, optimus won't be able to start.
 
+Optimus service can be started with
+```shell
+./optimus serve
+```
 
-**Optimus Service configuration**
+`serve` command has few required configurations that needs to be set for it to start. Configuration can either be stored
+in `.optimus.yaml` file or set as environment variable. Read more about it in [references](./docs/reference/configuration.md).
 
-Configuration inputs can either be passed as command arguments or set as environment variable
+### Installing via brew
 
-| command                | env name               | required | description                                                       |
-| ---------------------- | ---------------------- | -------- | ----------------------------------------------------------------- |
-| server-port            | SERVER_PORT            | N        | port on which service will listen for http calls, default. `8080` |
-| log-level              | LOG_LEVEL              | N        | log level - DEBUG, INFO, WARNING, ERROR, FATAL                    |
-| ingress-host           | INGRESS_HOST           | Y        | e.g. optimus.example.io:80                                        |
-| db-host                | DB_HOST                | Y        |                                                                   |
-| db-name                | DB_NAME                | Y        |                                                                   |
-| db-user                | DB_USER                | Y        |                                                                   |
-| db-password            | DB_PASSWORD            | Y        |                                                                   |
-| app-key                | APP_KEY                | Y        | 32 character random hash used to encrypt secrets                  |
+You can install Optimus using homebrew on macOS:
 
+```shell
+brew install odpf/taps/optimus
+optimus version
+```
 
 ## Credits
 

cmd/opctl/commands/admin.go → cmd/admin.go

@@ -1,4 +1,4 @@
-package commands
+package cmd
 
 import (
 	"github.com/odpf/optimus/models"
@@ -23,7 +23,6 @@ func adminBuildCommand(l logger) *cli.Command {
 		Short: "Register a job run and get required assets",
 	}
 	cmd.AddCommand(adminBuildInstanceCommand(l))
-	cmd.AddCommand(adminBuildPluginCommand(l))
 	return cmd
 }
 

cmd/opctl/commands/admin_build_instance.go → cmd/admin_build_instance.go

@@ -1,4 +1,4 @@
-package commands
+package cmd
 
 import (
 	"context"
@@ -35,7 +35,7 @@ func adminBuildInstanceCommand(l logger) *cli.Command {
 	cmd := &cli.Command{
 		Use:     "instance",
 		Short:   "Builds a Job instance including the assets for a scheduled execution",
-		Example: "opctl admin build instance sample_replace --project \"project-id\" --output-dir /tmp",
+		Example: "optimus admin build instance sample_replace --project \"project-id\" --output-dir /tmp",
 		Args:    cli.MinimumNArgs(1),
 	}
 
@@ -53,18 +53,17 @@ func adminBuildInstanceCommand(l logger) *cli.Command {
 	cmd.Flags().StringVar(&optimusHost, "host", "", "optimus service endpoint url")
 	cmd.MarkFlagRequired("host")
 
-	cmd.Run = func(c *cli.Command, args []string) {
+	cmd.RunE = func(c *cli.Command, args []string) error {
 		jobName := args[0]
 		l.Printf("requesting resources for project %s, job %s at %s\nplease wait...\n", projectName, jobName, optimusHost)
 		l.Printf("run name %s, run type %s, scheduled at %s\n", runName, runType, scheduledAt)
 		// append base path to input file directory
 		inputDirectory := filepath.Join(assetOutputDir, taskInputDirectory)
 
 		if err := getInstanceBuildRequest(l, jobName, inputDirectory, optimusHost, projectName, scheduledAt, runType, runName); err != nil {
-			l.Print(err)
-			l.Print(errRequestFail)
-			os.Exit(1)
+			return err
 		}
+		return nil
 	}
 	return cmd
 }

cmd/opctl/commands/admin_get_plugins.go → cmd/admin_get_plugins.go

@@ -1,4 +1,4 @@
-package commands
+package cmd
 
 import (
 	"context"
@@ -11,16 +11,16 @@ func adminGetPluginsCommand(l logger, taskRepo models.TaskPluginRepository, hook
 	cmd := &cli.Command{
 		Use:     "plugins",
 		Short:   "Get discovered plugins",
-		Example: `opctl admin get plugins`,
+		Example: `optimus admin get plugins`,
 	}
 
 	//TODO: add an option to list all server supported plugins
-	cmd.Run = func(c *cli.Command, args []string) {
+	cmd.RunE = func(c *cli.Command, args []string) error {
 		l.Println("Discovered tasks:")
 		for taskIdx, tasks := range taskRepo.GetAll() {
 			schema, err := tasks.GetTaskSchema(context.Background(), models.GetTaskSchemaRequest{})
 			if err != nil {
-				errExit(l, err)
+				return err
 			}
 			l.Printf("%d. %s\n", taskIdx+1, schema.Name)
 			l.Printf("Description: %s\n", schema.Description)
@@ -32,13 +32,14 @@ func adminGetPluginsCommand(l logger, taskRepo models.TaskPluginRepository, hook
 		for hookIdx, hooks := range hookRepo.GetAll() {
 			schema, err := hooks.GetHookSchema(context.Background(), models.GetHookSchemaRequest{})
 			if err != nil {
-				errExit(l, err)
+				return err
 			}
 			l.Printf("%d. %s\n", hookIdx+1, schema.Name)
 			l.Printf("Description: %s\n", schema.Description)
 			l.Printf("Image: %s\n", schema.Image)
 			l.Println("")
 		}
+		return nil
 	}
 	return cmd
 }

cmd/opctl/commands/admin_get_status.go → cmd/admin_get_status.go

@@ -1,8 +1,7 @@
-package commands
+package cmd
 
 import (
 	"context"
-	"os"
 	"sort"
 	"time"
 
@@ -24,24 +23,24 @@ func adminGetStatusCommand(l logger) *cli.Command {
 	cmd := &cli.Command{
 		Use:     "status",
 		Short:   "Get current job status",
-		Example: `opctl admin get status sample_replace --project \"project-id\"`,
+		Example: `optimus admin get status sample_replace --project \"project-id\"`,
 		Args:    cli.MinimumNArgs(1),
 	}
 	cmd.Flags().StringVar(&projectName, "project", "", "name of the tenant")
 	cmd.MarkFlagRequired("project")
 	cmd.Flags().StringVar(&optimusHost, "host", "", "optimus service endpoint url")
 	cmd.MarkFlagRequired("host")
 
-	cmd.Run = func(c *cli.Command, args []string) {
+	cmd.RunE = func(c *cli.Command, args []string) error {
 		jobName := args[0]
 		l.Printf("requesting status for project %s, job %s[%s] at %s\nplease wait...\n",
 			projectName, jobName, optimusHost)
 
 		if err := getJobStatusRequest(l, jobName, optimusHost, projectName); err != nil {
-			l.Print(err)
-			l.Print(errRequestFail)
-			os.Exit(1)
+			return err
 		}
+
+		return nil
 	}
 	return cmd
 }