Skip to content

Commit

Permalink
Merge master
Browse files Browse the repository at this point in the history
  • Loading branch information
Chris Gilmer committed Aug 1, 2019
2 parents e242353 + 283e8ae commit 224bd00
Show file tree
Hide file tree
Showing 90 changed files with 3,834 additions and 537 deletions.
3 changes: 3 additions & 0 deletions .circleci/config.yml
Expand Up @@ -176,6 +176,8 @@ commands:
bash -c "$(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)"
docker tag << parameters.tag >> ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/<< parameters.repo >>:git-${CIRCLE_SHA1}
docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/<< parameters.repo >>:git-${CIRCLE_SHA1}
docker tag << parameters.tag >> ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/<< parameters.repo >>:git-branch-${CIRCLE_BRANCH}
docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/<< parameters.repo >>:git-branch-${CIRCLE_BRANCH}
e2e_tests:
steps:
- run:
Expand Down Expand Up @@ -274,6 +276,7 @@ jobs:
keys:
- pre-commit-dot-cache-{{ checksum ".pre-commit-config.yaml" }}
- run: echo 'export PATH=${PATH}:~/go/bin:~/transcom/mymove/bin' >> $BASH_ENV
- run: echo 'export GOLANGCI_LINT_CONCURRENCY=1' >> $BASH_ENV
- run: make bin/callgraph
- run: make server_generate mocks_generate
# this is so we can avoid go mod downloading and resulting in an error on a false positive
Expand Down
3 changes: 3 additions & 0 deletions .envrc
Expand Up @@ -210,6 +210,9 @@ export HTTP_ORDERS_SERVER_NAME=orderslocal
# Set feature flags
export FEATURE_FLAG_ACCESS_CODE=false

# Set override of golangci-lint concurrency env variable
export GOLANGCI_LINT_CONCURRENCY=6

##############################################
# Load Local Overrides and Check Environment #
##############################################
Expand Down
3 changes: 2 additions & 1 deletion .pre-commit-config.yaml
Expand Up @@ -31,7 +31,8 @@ repos:
rev: v1.17.1
hooks:
- id: golangci-lint
entry: golangci-lint run
entry: bash -c 'exec golangci-lint run -j=${GOLANGCI_LINT_CONCURRENCY:-1}' # custom bash so we can override concurrency for faster dev runs


- repo: git://github.com/igorshubovych/markdownlint-cli
rev: v0.17.0
Expand Down
4 changes: 4 additions & 0 deletions .spelling
Expand Up @@ -415,6 +415,10 @@ uncomment
ecs-service-logs
Wildcards
envrc.local
golangci-lint
goland
SVY_SCORE
RATE_SCORE
# Put all custom terms BEFORE this comment, lest 'pre-commit' and 'make spellcheck' yield different errors.
- /usr/local
- docs/data/tspp-data-creation.md
Expand Down
29 changes: 27 additions & 2 deletions Makefile
Expand Up @@ -871,7 +871,7 @@ gofmt: ## Run go fmt over all Go files
go fmt $$(go list ./...) >> /dev/null

.PHONY: pre_commit_tests
pre_commit_tests: .server_generate.stamp .client_deps.stamp ## Run pre-commit tests
pre_commit_tests: .server_generate.stamp .mocks_generate.stamp .client_deps.stamp ## Run pre-commit tests
pre-commit run --all-files

.PHONY: pretty
Expand Down Expand Up @@ -912,7 +912,7 @@ clean: # Clean all generated files
find ./pkg -type d -name "mocks" -exec rm -rf {} +

.PHONY: spellcheck
spellcheck: .client_deps.stamp # Run interactive spellchecker
spellcheck: .client_deps.stamp ## Run interactive spellchecker
node_modules/.bin/mdspell --ignore-numbers --ignore-acronyms --en-us \
`find . -type f -name "*.md" \
-not -path "./node_modules/*" \
Expand All @@ -923,4 +923,29 @@ spellcheck: .client_deps.stamp # Run interactive spellchecker
# ----- END RANDOM TARGETS -----
#

#
# ----- START DOCKER COMPOSE TARGETS -----
#

.PHONY: docker_compose_setup
docker_compose_setup: .check_hosts.stamp ## Install requirements to use docker-compose
brew install -f bash git docker docker-compose direnv || true
brew cask install -f aws-vault || true

.PHONY: docker_compose_up
docker_compose_up: ## Bring up docker-compose containers
aws ecr get-login --no-include-email --region us-west-2 --no-include-email | sh
scripts/update-docker-compose
docker-compose up

.PHONY: docker_compose_down
docker_compose_down: ## Destroy docker-compose containers
docker-compose down
# Instead of using `--rmi all` which might destroy postgres we just remove the AWS containers
docker rmi $(shell docker images --filter=reference='*amazonaws*/*:*' --format "{{.ID}}")

#
# ----- END DOCKER COMPOSE TARGETS -----
#

default: help
6 changes: 6 additions & 0 deletions README.md
Expand Up @@ -26,6 +26,7 @@ This prototype was built by a [Defense Digital Service](https://www.dds.mil/) te
* [Setup: Quick Initial Setup](#setup-quick-initial-setup)
* [Setup: Prerequisites](#setup-prerequisites)
* [Setup: Direnv](#setup-direnv)
* [Helpful variables for `.envrc.local`](#helpful-variables-for-envrclocal)
* [Setup: Pre-Commit](#setup-pre-commit)
* [Setup: Hosts](#setup-hosts)
* [Setup: Dependencies](#setup-dependencies)
Expand Down Expand Up @@ -214,6 +215,11 @@ You can add a `.envrc.local` file. One way to do this is using chamber. You mus

If you wish to not maintain a `.envrc.local` you can alternatively run `cp .envrc.chamber.template .envrc.chamber` to enable getting secret values from `chamber`. **Note** that this method does not work for users of the `fish` shell unless you replace `direnv allow` with `direnv export fish | source`.

#### Helpful variables for `.envrc.local`

* `export GOLANGCI_LINT_CONCURRENCY=8` - variable to increase concurrency of golangci-lint; defaults to 6 on dev machines and to 1 in CircleCI.
* `export GOLAND=1` - variable to enable go code debugging in goland

### Setup: Pre-Commit

Run `pre-commit install` to install a pre-commit hook into `./git/hooks/pre-commit`. This is different than `brew install pre-commit` and must be done so that the hook will check files you are about to commit to the repository. Next install the pre-commit hook libraries with `pre-commit install-hooks`.
Expand Down
73 changes: 0 additions & 73 deletions cmd/load_duty_stations/main.go

This file was deleted.

157 changes: 157 additions & 0 deletions cmd/milmove/gen_duty_stations_migration.go
@@ -0,0 +1,157 @@
package main

import (
"fmt"
"log"
"os"
"path/filepath"
"text/template"

"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"go.uber.org/zap"

"github.com/transcom/mymove/pkg/cli"
"github.com/transcom/mymove/pkg/logging"
dutyStations "github.com/transcom/mymove/pkg/services/duty_stations"
)

const (
// filename containing the details for new duty stations
DutyStationsFilenameFlag string = "duty-stations-filename"
)

type MigrationInfo struct {
Filename string
}

const (
DutyStationMigration string = `
-- Migration generated using: cmd/milmove/gen_duty_stations_migration.go
-- Duty stations file: {{.Filename}}`
)

// DutyStationsFilenameFlag initializes add_duty_stations command line flags
func InitAddDutyStationsFlags(flag *pflag.FlagSet) {
flag.StringP(DutyStationsFilenameFlag, "f", "", "File name of csv file containing the new duty stations users")
}

// CheckAddDutyStations validates add_duty_stations command line flags
func CheckAddDutyStations(v *viper.Viper, logger logger) error {
if err := cli.CheckDatabase(v, logger); err != nil {
return err
}

if err := cli.CheckMigration(v); err != nil {
return err
}

if err := cli.CheckMigrationFile(v); err != nil {
return err
}

DutyStationsFilename := v.GetString(DutyStationsFilenameFlag)
if DutyStationsFilename == "" {
return fmt.Errorf("--duty-stations-filename is required")
}
return nil
}

func initGenDutyStationsMigrationFlags(flag *pflag.FlagSet) {
// DB Config
cli.InitDatabaseFlags(flag)

// Migration Config
cli.InitMigrationFlags(flag)

// Migration File Config
cli.InitMigrationFileFlags(flag)

// Add Duty Stations
InitAddDutyStationsFlags(flag)

// Sort command line flags
flag.SortFlags = true
}

func createDutyStationMigration(path string, filename string, ds []dutyStations.DutyStationMigration) error {
migrationPath := filepath.Join(path, filename)
migrationFile, err := os.Create(migrationPath)
defer closeFile(migrationFile)
if err != nil {
return errors.Wrapf(err, "error creating %s", migrationPath)
}

t1 := template.Must(template.New("temp1").Parse(DutyStationMigration))
err = t1.Execute(migrationFile, MigrationInfo{filename})
t2 := template.Must(template.New("temp2").Parse(dutyStations.InsertTemplate))
err = t2.Execute(migrationFile, ds)
if err != nil {
log.Println("error executing template: ", err)
}
log.Printf("new migration file created at: %q\n", migrationPath)
return nil
}

func genDutyStationsMigration(cmd *cobra.Command, args []string) error {
err := cmd.ParseFlags(args)
flag := cmd.Flags()
err = flag.Parse(os.Args[1:])
if err != nil {
return errors.Wrap(err, "could not parse flags")
}
v := viper.New()
err = v.BindPFlags(flag)
if err != nil {
return errors.Wrap(err, "could not bind flags")
}

logger, err := logging.Config(v.GetString(cli.LoggingEnvFlag), v.GetBool(cli.VerboseFlag))
if err != nil {
return err
}

err = CheckAddDutyStations(v, logger)
if err != nil {
return err
}

migrationPath := v.GetString(cli.MigrationPathFlag)
migrationManifest := v.GetString(cli.MigrationManifestFlag)
migrationName := v.GetString(cli.MigrationNameFlag)
migrationVersion := v.GetString(cli.MigrationVersionFlag)
dutyStationsFilename := v.GetString(DutyStationsFilenameFlag)

// Create a connection to the DB
dbConnection, err := cli.InitDatabase(v, logger)
if err != nil {
if dbConnection == nil {
// No connection object means that the configuraton failed to validate and we should kill server startup
logger.Fatal("Invalid DB Configuration", zap.Error(err))
} else {
// A valid connection object that still has an error indicates that the DB is not up and
// thus is not ready for migrations
logger.Fatal("DB is not ready for connections", zap.Error(err))
}
}

builder := dutyStations.NewMigrationBuilder(dbConnection, logger)
insertions, err := builder.Build(dutyStationsFilename)
if err != nil {
logger.Panic("Error while building migration", zap.Error(err))
}

migrationFilename := fmt.Sprintf("%s_%s.up.sql", migrationVersion, migrationName)
err = createDutyStationMigration(migrationPath, migrationFilename, insertions)
if err != nil {
return err
}

err = addMigrationToManifest(migrationManifest, migrationFilename)
if err != nil {
return err
}
return nil
}
11 changes: 11 additions & 0 deletions cmd/milmove/main.go
Expand Up @@ -85,6 +85,17 @@ func main() {
initGenOfficeUserMigrationFlags(genOfficeUserMigrationCommand.Flags())
genCommand.AddCommand(genOfficeUserMigrationCommand)

genDutyStationsMigrationCommand := &cobra.Command{
Use: "duty-stations-migration -f CSV_FILENAME -n MIGRATION_NAME",
Short: "Generate migrations required for adding duty stations",
Long: "Generate migrations required for adding duty stations",
RunE: genDutyStationsMigration,
DisableFlagsInUseLine: true,
SilenceErrors: true, // not needed
}
initGenDutyStationsMigrationFlags(genDutyStationsMigrationCommand.Flags())
genCommand.AddCommand(genDutyStationsMigrationCommand)

genDisableUserMigrationCommand := &cobra.Command{
Use: "disable-user-migration -e EMAIL",
Short: "Generate migrations required for disabling a user",
Expand Down

0 comments on commit 224bd00

Please sign in to comment.