RDS IAM support

.circleci/config.yml

@@ -78,7 +78,7 @@ commands:
           at: bin
       - deploy:
           name: Deploy task service
-          command: scripts/do-exclusively --job-name ${CIRCLE_JOB} bin/ecs-deploy-task-container --aws-account-id ${AWS_ACCOUNT_ID} --aws-region ${AWS_DEFAULT_REGION} --service app --environment ${APP_ENVIRONMENT} --repository-name app-tasks --image-tag git-${CIRCLE_SHA1} --command save-fuel-price-data
+          command: scripts/do-exclusively --job-name ${CIRCLE_JOB} bin/ecs-deploy-task-container --aws-account-id ${AWS_ACCOUNT_ID} --aws-region ${AWS_DEFAULT_REGION} --service app --environment ${APP_ENVIRONMENT} --repository-name app-tasks --image-tag git-${CIRCLE_SHA1} --command save-fuel-price-data --command-args '--db-iam'
           environment:
             DB_PORT: 5432
             DB_USER: master
@@ -184,15 +184,14 @@ commands:
             source $BASH_ENV
             make e2e_test_docker
           environment:
-            # Env vars needed by the `scripts/apply-secure-migration.sh` script
             DB_PASSWORD: mysecretpassword
             DB_USER: postgres
             DB_HOST: localhost
             DB_PORT: 5432
             DB_NAME: test_db
             # Env vars needed for the webserver to run inside docker
-            SECURE_MIGRATION_DIR: /home/circleci/transcom/mymove/local_migrations
-            SECURE_MIGRATION_SOURCE: local
+            MIGRATION_PATH: "/migrate/local_migrations;/migrate/migrations"
+            MIGRATION_MANIFEST: "/migrate/migrations_manifest.txt"
             LOGIN_GOV_CALLBACK_PROTOCOL: http
             LOGIN_GOV_MY_CLIENT_ID: urn:gov:gsa:openidconnect.profiles:sp:sso:dod:mymovemillocal
             LOGIN_GOV_OFFICE_CLIENT_ID: urn:gov:gsa:openidconnect.profiles:sp:sso:dod:officemovemillocal
@@ -452,7 +451,6 @@ jobs:
             source $BASH_ENV
             make server_test
           environment:
-            # Env vars needed by the `scripts/apply-secure-migration.sh` script
             DB_PASSWORD: mysecretpassword
             DB_USER: postgres
             DB_HOST: localhost
@@ -461,9 +459,8 @@ jobs:
             EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
             ENV: test
             ENVIRONMENT: test
-            MIGRATION_PATH: /home/circleci/transcom/mymove/migrations
-            SECURE_MIGRATION_DIR: /home/circleci/transcom/mymove/local_migrations
-            SECURE_MIGRATION_SOURCE: local
+            MIGRATION_PATH: "/home/circleci/transcom/mymove/local_migrations;/home/circleci/transcom/mymove/migrations"
+            MIGRATION_MANIFEST: "/home/circleci/transcom/mymove/migrations_manifest.txt"
 
       - announce_failure
 
@@ -500,7 +497,6 @@ jobs:
             source $BASH_ENV
             make server_test_coverage_generate
           environment:
-            # Env vars needed by the `scripts/apply-secure-migration.sh` script
             DB_PASSWORD: mysecretpassword
             DB_USER: postgres
             DB_HOST: localhost
@@ -509,9 +505,8 @@ jobs:
             EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
             ENV: test
             ENVIRONMENT: test
-            MIGRATION_PATH: /home/circleci/transcom/mymove/migrations
-            SECURE_MIGRATION_DIR: /home/circleci/transcom/mymove/local_migrations
-            SECURE_MIGRATION_SOURCE: local
+            MIGRATION_PATH: "/home/circleci/transcom/mymove/local_migrations;/home/circleci/transcom/mymove/migrations"
+            MIGRATION_MANIFEST: "/home/circleci/transcom/mymove/migrations_manifest.txt"
       - run:
           name: upload code coverage to codecov
           command: |
@@ -572,7 +567,8 @@ jobs:
             - v2-mymove-node-modules-{{ checksum "yarn.lock" }}
       - run: make bin/chamber
       - run: make bin/rds-combined-ca-bundle.pem
-      - run: make build
+      - run: make client_build
+      - run: make server_build
       - build_tag_push:
           dockerfile: Dockerfile
           tag: ppp:web-dev
@@ -588,7 +584,7 @@ jobs:
           docker_layer_caching: true
       - run: make bin/chamber
       - run: make bin/rds-combined-ca-bundle.pem
-      - run: make bin/milmove
+      - run: make server_build
       - build_tag_push:
           dockerfile: Dockerfile.migrations
           tag: ppp-migrations:dev
@@ -988,8 +984,8 @@ workflows:
             - server_test
             - build_app
             - build_tools
-            - build_migrations
             - build_tasks
+            - build_migrations
             - acceptance_tests_local
             - acceptance_tests_staging
             - integration_tests_mymove

.envrc

@@ -79,10 +79,8 @@ export CHAMBER_RETRIES=20
 export ENVIRONMENT=development
 
 # Migration Path
-export MIGRATION_PATH="${MYMOVE_DIR}/migrations"
-
-# Configuration needed for secure migrations.
-export SECURE_MIGRATION_DIR="${MYMOVE_DIR}/local_migrations"
+export MIGRATION_PATH="file://${MYMOVE_DIR}/local_migrations;file://${MYMOVE_DIR}/migrations"
+export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations_manifest.txt"
 
 # Default DB configuration
 export DB_PASSWORD=mysecretpassword

.pre-commit-config.yaml

@@ -32,7 +32,6 @@ repos:
     hooks:
       - id: golangci-lint
         entry: golangci-lint run --verbose
-        verbose: true
 
   - repo: git://github.com/igorshubovych/markdownlint-cli
     rev: v0.16.0

Dockerfile

@@ -1,15 +1,17 @@
 FROM gcr.io/distroless/base:latest
 
 COPY bin/rds-combined-ca-bundle.pem /bin/rds-combined-ca-bundle.pem
+
 COPY bin/chamber /bin/chamber
+
 COPY bin/milmove /bin/milmove
 
 COPY config /config
+
 COPY swagger/* /swagger/
+
 COPY build /build
 
 ENTRYPOINT ["/bin/milmove"]
 
-CMD ["serve", "--debug-logging"]
-
 EXPOSE 8080

Dockerfile.migrations

@@ -1,22 +1,15 @@
 FROM alpine:3.7
 
 COPY bin/rds-combined-ca-bundle.pem /bin/rds-combined-ca-bundle.pem
+
 COPY bin/chamber /bin/chamber
+
 COPY bin/milmove /bin/milmove
-RUN mkdir -p /scripts
-COPY scripts/apply-secure-migration.sh /scripts
 
 COPY migrations /migrate/migrations
 
-RUN mkdir -p /migrate/secure
-
-ENV GO_ENV=container
+COPY migrations_manifest.txt /migrate/migrations_manifest.txt
 
-# Install tools needed in container
-RUN apk update
-RUN apk add postgresql-client python3 ca-certificates bash --no-cache
-RUN pip3 install awscli
+WORKDIR /
 
-WORKDIR /scripts
 ENTRYPOINT ["/bin/milmove"]
-CMD ["migrate", "-p", "/migrate/migrations"]

Dockerfile.migrations_local

@@ -1,21 +1,22 @@
 FROM alpine:3.7
 
 COPY bin_linux/milmove /bin/milmove
-RUN mkdir -p /scripts
-COPY scripts/apply-secure-migration.sh /scripts
+
 COPY bin_linux/generate-test-data /bin/generate-test-data
 
 COPY migrations /migrate/migrations
-COPY local_migrations /migrate/local
+
+COPY local_migrations /migrate/local_migrations
+
+COPY migrations_manifest.txt /migrate/migrations_manifest.txt
+
 RUN mkdir -p /bin/pkg/testdatagen/testdata/
 COPY pkg/testdatagen/testdata/test.pdf /bin/pkg/testdatagen/testdata/test.pdf
 
-ENV GO_ENV=test
-ENV SECURE_MIGRATION_SOURCE=local
-ENV SECURE_MIGRATION_DIR=/migrate/local
-
-# Install tools needed in container
+# psql (aka postgresql-client) is required to run `TRUNCATE users CASCADE;` to reset local database
 RUN apk update
 RUN apk add postgresql-client bash --no-cache
 
-WORKDIR /scripts
+WORKDIR /
+
+ENTRYPOINT ["/bin/milmove"]

Dockerfile.tasks

@@ -1,7 +1,9 @@
 FROM gcr.io/distroless/base:latest
 
 COPY bin/rds-combined-ca-bundle.pem /bin/rds-combined-ca-bundle.pem
+
 COPY bin/chamber /bin/chamber
+
 COPY bin/save-fuel-price-data /bin/save-fuel-price-data
 
 WORKDIR /bin