feat!: move deployment pipeline to CDK (#664)

.eslintignore

@@ -1 +1,2 @@
-auditLogMover/
+auditLogMover/
+cdk.out/

.github/workflows/deploy-smart.yaml

@@ -48,6 +48,7 @@ jobs:
   deploy:
     needs: pre-deployment-check
     name: Deploy to Dev - enableMultiTenancy=${{ matrix.enableMultiTenancy }}
+    environment: FWoA Integ Test Env
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -57,11 +58,17 @@ jobs:
             issuerEndpointSecretName: SMART_ISSUER_ENDPOINT
             oAuth2ApiEndpointSecretName: SMART_OAUTH2_API_ENDPOINT
             patientPickerEndpointSecretName: SMART_PATIENT_PICKER_ENDPOINT
+            cdk_issuerEndpointSecretName: CDK_SMART_ISSUER_ENDPOINT
+            cdk_oAuth2ApiEndpointSecretName: CDK_SMART_OAUTH2_API_ENDPOINT
+            cdk_patientPickerEndpointSecretName: CDK_SMART_PATIENT_PICKER_ENDPOINT
           - enableMultiTenancy: true
             region: us-west-1
             issuerEndpointSecretName: MULTITENANCY_SMART_ISSUER_ENDPOINT
             oAuth2ApiEndpointSecretName: MULTITENANCY_SMART_OAUTH2_API_ENDPOINT
             patientPickerEndpointSecretName: MULTITENANCY_SMART_PATIENT_PICKER_ENDPOINT
+            cdk_issuerEndpointSecretName: CDK_MT_SMART_ISSUER_ENDPOINT
+            cdk_oAuth2ApiEndpointSecretName: CDK_MT_SMART_OAUTH2_API_ENDPOINT
+            cdk_patientPickerEndpointSecretName: CDK_MT_SMART_PATIENT_PICKER_ENDPOINT
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -101,6 +108,13 @@ jobs:
         run: |
           yarn install
           serverless deploy --stage dev --region ${{ matrix.region }} --issuerEndpoint ${{ secrets[matrix.issuerEndpointSecretName] }} --oAuth2ApiEndpoint ${{ secrets[matrix.oAuth2ApiEndpointSecretName] }} --patientPickerEndpoint ${{ secrets[matrix.patientPickerEndpointSecretName] }} --useHapiValidator true --enableSubscriptions true --enableMultiTenancy ${{ matrix.enableMultiTenancy }} --conceal
+      - name: Deploy FWoA with CDK
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CDK_SMART_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CDK_SMART_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          yarn install
+          yarn deploy -c issuerEndpoint=${{ secrets[matrix.cdk_issuerEndpointSecretName] }} -c oAuth2ApiEndpoint=${{ secrets[matrix.cdk_oAuth2ApiEndpointSecretName] }} -c patientPickerEndpoint=${{ secrets[matrix.cdk_patientPickerEndpointSecretName] }} -c enableSubscriptions=true -c useHapiValidator=true -c region=${{ matrix.region }} -c enableMultiTenancy=${{ matrix.enableMultiTenancy }} --all --require-approval=never
       - name: Deploy auditLogMover
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.SMART_AWS_ACCESS_KEY_ID}}
@@ -121,10 +135,12 @@ jobs:
             region: us-east-2
             serviceUrlSuffix: ''
             smartServiceURLSecretName: SMART_SERVICE_URL
+            cdk_smartServiceURLSecretName: CDK_SMART_SERVICE_URL
           - enableMultiTenancy: true
             region: us-west-1
             serviceUrlSuffix: /tenant/tenant1
             smartServiceURLSecretName: MULTITENANCY_SMART_SERVICE_URL
+            cdk_smartServiceURLSecretName: CDK_MULTITENANCY_SMART_SERVICE_URL
     steps:
       - uses: actions/checkout@v2
         with:
@@ -157,9 +173,30 @@ jobs:
           sed -i -e "s#AUTH_PASSWORD#$AUTH_PASSWORD#g" fhir-works.json
           bundle exec rake db:create db:schema:load
           bundle exec rake inferno:execute_batch[fhir-works.json]
+      - name: Execute tests on CDK
+        env:
+          SERVICE_URL: ${{ secrets[matrix.cdk_smartServiceURLSecretName]}}${{ matrix.serviceUrlSuffix }}
+          CLIENT_ID: ${{ secrets.CDK_SMART_AUTH_CLIENT_ID}}
+          CLIENT_SECRET: ${{ secrets.CDK_SMART_AUTH_CLIENT_SECRET }}
+          AUTH_ENDPOINT: ${{ secrets.CDK_SMART_AUTH_ENDPOINT }}
+          TOKEN_ENDPOINT: ${{ secrets.CDK_SMART_TOKEN_ENDPOINT}}
+          AUTH_USERNAME: ${{ secrets.CDK_SMART_AUTH_USERNAME}}
+          AUTH_PASSWORD: ${{ secrets.CDK_SMART_AUTH_PASSWORD}}
+        run: |
+          cp fhir-works-example.json fhir-works.json
+          sed -i -e "s#SERVER_ENDPOINT#$SERVICE_URL#g" fhir-works.json
+          sed -i -e "s#CLIENT_ID#$CLIENT_ID#g" fhir-works.json
+          sed -i -e "s#CLIENT_SECRET#$CLIENT_SECRET#g" fhir-works.json
+          sed -i -e "s#AUTH_ENDPOINT#$AUTH_ENDPOINT#g" fhir-works.json
+          sed -i -e "s#TOKEN_ENDPOINT#$TOKEN_ENDPOINT#g" fhir-works.json
+          sed -i -e "s#AUTH_USERNAME#$AUTH_USERNAME#g" fhir-works.json
+          sed -i -e "s#AUTH_PASSWORD#$AUTH_PASSWORD#g" fhir-works.json
+          bundle exec rake db:create db:schema:load
+          bundle exec rake inferno:execute_batch[fhir-works.json]
   custom-integration-tests:
     needs: inferno-test
     name: Run custom integration tests - enableMultiTenancy=${{ matrix.enableMultiTenancy }}
+    environment: FWoA Integ Test Env
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -174,6 +211,14 @@ jobs:
             subscriptionsNotificationsTableSecretName: SMART_SUBSCRIPTIONS_NOTIFICATIONS_TABLE
             subscriptionsEndpointSecretName: SMART_SUBSCRIPTIONS_ENDPOINT
             subscriptionsApiKeySecretName: SMART_SUBSCRIPTIONS_API_KEY
+            cdk_smartOauth2ApiEndpointSecretName: CDK_SMART_OAUTH2_API_ENDPOINT
+            cdk_smartAuthUsernameSecretName: CDK_SMART_AUTH_USERNAME
+            cdk_smartAuthAdminUsernameSecretName: CDK_SMART_AUTH_ADMIN_USERNAME
+            cdk_smartServiceURLSecretName: CDK_SMART_SERVICE_URL
+            cdk_smartApiKeySecretName: CDK_SMART_API_KEY
+            cdk_subscriptionsNotificationsTableSecretName: CDK_SMART_SUBSCRIPTIONS_NOTIFICATIONS_TABLE
+            cdk_subscriptionsEndpointSecretName: CDK_SMART_SUBSCRIPTIONS_ENDPOINT
+            cdk_subscriptionsApiKeySecretName: CDK_SMART_SUBSCRIPTIONS_API_KEY
           - enableMultiTenancy: true
             region: us-west-1
             smartOauth2ApiEndpointSecretName: MULTITENANCY_SMART_OAUTH2_API_ENDPOINT
@@ -184,6 +229,14 @@ jobs:
             subscriptionsNotificationsTableSecretName: MULTITENANCY_SMART_SUBSCRIPTIONS_NOTIFICATIONS_TABLE
             subscriptionsEndpointSecretName: MULTITENANCY_SMART_SUBSCRIPTIONS_ENDPOINT
             subscriptionsApiKeySecretName: MULTITENANCY_SMART_SUBSCRIPTIONS_API_KEY
+            cdk_smartOauth2ApiEndpointSecretName: CDK_MT_SMART_OAUTH2_API_ENDPOINT
+            cdk_smartAuthUsernameSecretName: CDK_MT_SMART_AUTH_USERNAME
+            cdk_smartAuthAdminUsernameSecretName: CDK_MT_SMART_AUTH_ADMIN_USERNAME
+            cdk_smartServiceURLSecretName: CDK_MT_SMART_SERVICE_URL
+            cdk_smartApiKeySecretName: CDK_MT_SMART_API_KEY
+            cdk_subscriptionsNotificationsTableSecretName: CDK_MT_SMART_SUBSCRIPTIONS_NOTIFICATIONS_TABLE
+            cdk_subscriptionsEndpointSecretName: CDK_MT_SMART_SUBSCRIPTIONS_ENDPOINT
+            cdk_subscriptionsApiKeySecretName: CDK_MT_SMART_SUBSCRIPTIONS_API_KEY
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -215,6 +268,27 @@ jobs:
           AWS_REGION: ${{ matrix.region }}
           API_AWS_REGION: ${{ matrix.region }}
         run: yarn int-test
+      - name: Execute tests on CDK
+        env:
+          SMART_OAUTH2_API_ENDPOINT: ${{ secrets[matrix.cdk_smartOauth2ApiEndpointSecretName] }}
+          SMART_INTEGRATION_TEST_CLIENT_ID: ${{ secrets.CDK_SMART_INTEGRATION_TEST_CLIENT_ID}}
+          SMART_INTEGRATION_TEST_CLIENT_PW: ${{ secrets.CDK_SMART_INTEGRATION_TEST_CLIENT_PW}}
+          SMART_AUTH_USERNAME: ${{ secrets[matrix.cdk_smartAuthUsernameSecretName] }}
+          SMART_AUTH_ADMIN_USERNAME: ${{ secrets[matrix.cdk_smartAuthAdminUsernameSecretName] }}
+          SMART_AUTH_ADMIN_ANOTHER_TENANT_USERNAME: ${{ secrets.CDK_SMART_AUTH_ADMIN_ANOTHER_TENANT_USERNAME}}
+          SMART_AUTH_PASSWORD: ${{ secrets.CDK_SMART_AUTH_PASSWORD}}
+          SMART_SERVICE_URL: ${{ secrets[matrix.cdk_smartServiceURLSecretName] }}
+          SMART_API_KEY: ${{ secrets[matrix.cdk_smartApiKeySecretName] }}
+          MULTI_TENANCY_ENABLED: ${{ matrix.enableMultiTenancy }}
+          SUBSCRIPTIONS_ENABLED: 'true'
+          SUBSCRIPTIONS_NOTIFICATIONS_TABLE: ${{ secrets[matrix.cdk_subscriptionsNotificationsTableSecretName] }}
+          SUBSCRIPTIONS_ENDPOINT: ${{ secrets[matrix.cdk_subscriptionsEndpointSecretName] }}
+          SUBSCRIPTIONS_API_KEY: ${{ secrets[matrix.cdk_subscriptionsApiKeySecretName] }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CDK_SMART_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CDK_SMART_AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ matrix.region }}
+          API_AWS_REGION: ${{ matrix.region }}
+        run: yarn int-test
   merge-develop-to-mainline:
     needs: custom-integration-tests
     name: Merge smart-develop to smart-mainline

.gitignore

@@ -30,3 +30,9 @@ yalc.lock
 /compiledImplementationGuides/*
 !/compiledImplementationGuides/gitkeep
 /.webpack/
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out
+build
+dist

DEVELOPMENT.md

@@ -28,10 +28,12 @@ Copy these files to your parent directory
 
 Run `./fhir-works-local-develop.sh` from the parent directory. This script will use `yarn` to link your `persistence`, `search`, `authz`, and `routing` package to your `interface` package. It will then link all five of those packages to your `deployment` package.
 
-Once the script has finished running, you can run `yarn watch` in the directory of each package that you're developing in. This will pick up live changes from the packages. Then you can run this command in the `deployment` package directory to spin up your local environment:
+Once the script has finished running, you can run `yarn watch` in the directory of each package that you're developing in. This will pick up live changes from the packages. Then you can run this command in the `deployment` package directory to spin up your local environment if you are using serverless:
 
 `AWS_ACCESS_KEY_ID=<Access-Key> AWS_SECRET_ACCESS_KEY=<Secret-Key> OFFLINE_BINARY_BUCKET=<FHIRBinaryBucket> OFFLINE_ELASTICSEARCH_DOMAIN_ENDPOINT=<ElasticSearchDomainEndpoint> serverless offline start`
 
+With CDK, you can follow the guide [here](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-cdk-testing.html) and use `sam local invoke` to locally test your environment.
+
 ### Local Development with Implementation Guides
 
 If you're using [Implementation Guides](./USING_IMPLEMENTATION_GUIDES.md), then follow these steps to run FHIR Works with IG locally. You'll need to provide the `OFFLINE_LAMBDA_VALIDATOR_ALIAS`.

INSTALL.md

@@ -188,7 +188,13 @@ Yarn is a node package management tool similar to npm. Instructions for installi
 brew install yarn
 ```
 
-### serverless CLI
+### CDK CLI
+
+AWS CDK (Cloud Development Kit) is a framework for defining cloud infrastructure such as Lambda functions and associated resources in code and provisioning it in the target AWS Account through AWS CloudFormation. Instructions for installing CDK are provided for different platforms here:
+
+> https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html
+
+### serverless CLI (LEGACY)
 
 Serverless is a tool used to deploy Lambda functions and associated resources to the target AWS account.
 Instructions for installing Serverless are provided for different platforms here:
@@ -234,7 +240,8 @@ yarn install
 yarn run release
 ```
 
-### IAM User ARN
+### IAM User ARN (LEGACY)
+Note: this customization is only needed if deploying with serverless; it is not needed with CDK.
 
 Create a new file in the package's root folder named
 
@@ -247,8 +254,21 @@ In the _serverless_config.json_ file, add the following, using the previously no
   "devAwsUserAccountArn": "<IAM USER ARN>"
 }
 ```
+### AWS service deployment with CDK
+Using the previously noted AWS Profile, deploy the required AWS services to your AWS account using the default setting of stage: dev and region: us-west-2. To change the default stage/region, simply modify the values near the bottom of the `cdk.json` file.
+
+```sh
+yarn deploy --profile YOUR_AWS_PROFILE -c issuerEndpoint=YOUR_ISSUER_ENDPOINT -c oAuth2ApiEndpoint=YOUR_OAUTH2_API_ENDPOINT -c patientPickerEndpoint=YOUR_PATIENT_PICKER_ENDPOINT
+```
+
+Or you can deploy with a custom stage/region:
+```sh
+yarn deploy --profile YOUR_AWS_PROFILE -c issuerEndpoint=YOUR_ISSUER_ENDPOINT -c oAuth2ApiEndpoint=YOUR_OAUTH2_API_ENDPOINT -c patientPickerEndpoint=YOUR_PATIENT_PICKER_ENDPOINT -c stage=YOUR_STAGE -c region=YOUR_REGION
+```
+After deployment is successful, the stack outputs will be printed in the console. These are saved into the `Info_Output.log` file for future reference.
+
 
-### AWS service deployment
+### AWS service deployment with Serverless (LEGACY)
 
 Using the previously noted AWS Profile, deploy the required AWS services to your AWS account using the default setting of stage: dev and region: us-west-2. To change the default stage/region look for the stage/region variable in the [serverless.yaml](./serverless.yaml) file under the provider: object.
 
@@ -329,8 +349,9 @@ The Kibana server allows you to explore data inside your Elasticsearch instance
 
 In order to be able to access the Kibana server for your Elasticsearch Service Instance, you need to create and confirm a Cognito user. Run the below command or create a user from the Cognito console.
 
+With CDK, you can find the needed variables in the Info_Output.log file after deployment.
 ```sh
-# Find ELASTIC_SEARCH_KIBANA_USER_POOL_APP_CLIENT_ID in the printout
+# Find ELASTIC_SEARCH_KIBANA_USER_POOL_APP_CLIENT_ID in the printout (LEGACY)
 serverless info --verbose
 
 # Create new user
@@ -341,7 +362,7 @@ aws cognito-idp sign-up \
   --password <TEMP_PASSWORD> \
   --user-attributes Name="email",Value="<youremail@address.com>"
 
-# Find ELASTIC_SEARCH_KIBANA_USER_POOL_ID in the printout
+# Find ELASTIC_SEARCH_KIBANA_USER_POOL_ID in the printout (LEGACY)
 # Notice this is a different ID from the one used in the last step
 serverless info --verbose
 
@@ -367,7 +388,7 @@ aws cognito-idp admin-confirm-sign-up \
 
 ###### Get Kibana url
 
-After the Cognito user is created and confirmed you can now log in with the username and password, at the ELASTIC_SEARCH_DOMAIN_KIBANA_ENDPOINT (found with the `serverless info --verbose` command). **Note** Kibana will be empty at first and have no indices, they will be created once the FHIR server writes resources to the DynamoDB
+After the Cognito user is created and confirmed you can now log in with the username and password, at the ELASTIC_SEARCH_DOMAIN_KIBANA_ENDPOINT (found in `Info_Output.log`, or with the `serverless info --verbose` command (LEGACY)). **Note** Kibana will be empty at first and have no indices, they will be created once the FHIR server writes resources to the DynamoDB
 
 #### DynamoDB table backups
 

README.md

@@ -89,7 +89,9 @@ This project is licensed under the Apache-2.0 license.
 
 ### Retrieving user variables
 
-After installation, all user-specific variables (such as `SERVICE_ENDPOINT`) can be found in the `Info_Output.log` file. You can also retrieve these values by running `serverless info --verbose --region <REGION> --stage <STAGE>`. **NOTE:** default stage is `dev` and region is `us-west-2`.
+After installation, all user-specific variables (such as `SERVICE_ENDPOINT`) can be found in the `Info_Output.log` file. 
+
+If you have deployed using serverless, you can also retrieve these values by running `serverless info --verbose --region <REGION> --stage <STAGE>`. **NOTE:** default stage is `dev` and region is `us-west-2`.
 
 If you are receiving `Error: EACCES: permission denied` when executing a command, try re-running the command with `sudo`.
 
@@ -119,7 +121,10 @@ After you import the collection, you need to set up your environment. You can se
 
 Instructions for importing the environment JSON is located [here](https://thinkster.io/tutorials/testing-backend-apis-with-postman/managing-environments-in-postman). The environment file is [FHIR_SMART.postman_environment.json](./postman/FHIR_SMART.postman_environment.json)
 
-The API_URL & API_KEY variables required in the POSTMAN collection can be found in `Info_Output.log` or by running `serverless info --verbose`. The remaining variables should be found within your authorization server.
+The API_URL & API_KEY variables required in the POSTMAN collection can be found in `Info_Output.log`.
+If you have deployed with serverless, you can also run `serverless info --verbose`.
+
+The remaining variables should be found within your authorization server.
 
 - API_URL: from Service Information:endpoints: ANY
 - API_KEY: from Service Information: api keys: developer-key

USING_IMPLEMENTATION_GUIDES.md

@@ -33,6 +33,14 @@ The prerequisites for FHIR IGs are same as in the FHIR [installation documentati
    ```
    **Note:** This command needs to be invoked in the top level directory of the cloned `fhir-works-on-aws-deployment` repository
 1. Deploy the Hapi Validator using the following commands:
+
+  When deploying, simply append the useHapiValidator flag (and the optional fhirVersion flag, which defaults to 4.0.1):
+  ```sh
+  yarn deploy -c useHapiValidator=true -c fhirVersion=4.0.1
+  ```
+
+  Or, with Serverless, continue with these steps: (LEGACY)
+
    ```bash
    #fhir-works-on-aws-deployment/javaHapiValidatorLambda
    cd javaHapiValidatorLambda

USING_MULTI_TENANCY.md

@@ -10,6 +10,11 @@ is logically partitioned to ensure that tenants are prevented from accessing ano
 
 Use the `enableMultiTenancy` option when deploying the stack: 
 
+```sh
+yarn deploy -c enableMultiTenancy=true
+```
+
+or with Serverless, (LEGACY)
 ```bash
 serverless deploy --enableMultiTenancy true
 ```

USING_SUBSCRIPTIONS.md

@@ -16,6 +16,12 @@ Update [src/subscriptions/allowList.ts](src/subscriptions/allowList.ts) to confi
 
 2. Use the `enableSubscriptions` option when deploying the stack:
 
+  ```sh
+  yarn deploy -c enableSubscriptions=true
+  ```
+
+  Or with Serverless (LEGACY),
+
    ```bash
    serverless deploy --enableSubscriptions true
    ```