feat: add python monorepo support (#162)

src/commands/docker/build_push_image.yml

@@ -46,6 +46,10 @@ parameters:
     description: KMS Key to sign the containers
     type: string
     default: "awskms:///2e64fa98-d1b0-491a-acf6-1f5fc6f94ecf"
+  inject_aws_credentials:
+    description: Inject AWS credentials into the build
+    type: boolean
+    default: false
 steps:
   - when:
       condition: << parameters.checkout >>
@@ -84,4 +88,5 @@ steps:
         IMAGE_TAG_OVERRIDE: '<< parameters.image_tag >>'
         IMAGE_REPO: '<< parameters.image_repo >>'
         KMS_KEY: '<< parameters.kms_key >>'
+        INJECT_AWS_CREDENTIALS: "<< parameters.inject_aws_credentials >>"
       command: <<include(scripts/docker/push.sh)>>

src/commands/monorepo/monorepo_save_cache.yml

@@ -46,3 +46,11 @@ steps:
                   root: "."
                   paths:
                     - node_modules/.cache
+        - when:
+            condition:
+              equal: ["nx", << parameters.monorepo_engine >>]
+            steps:
+              - persist_to_workspace:
+                  root: "."
+                  paths:
+                    - .cache

src/commands/poetry/authenticate_poetry.yml

@@ -0,0 +1,15 @@
+parameters:
+  working_directory:
+    description: Directory containing pyproject.toml
+    type: string
+    default: './'
+  run_in_background:
+    description: run the command in background
+    type: boolean
+    default: false
+steps:
+  - run:
+      working_directory: << parameters.working_directory >>
+      background: << parameters.run_in_background >>
+      name: authenticate poetry
+      command: <<include(scripts/python/authenticate-poetry.sh)>>

src/commands/poetry/install_python_modules.yml

@@ -0,0 +1,54 @@
+parameters:
+  install_args:
+    description: Additional poetry install command options
+    type: string
+    default: ""
+  working_directory:
+    description: Directory containing package.json
+    type: string
+    default: './'
+  step_name:
+    description: Name of the step
+    type: string
+    default: Install node modules
+  cache_prefix:
+    description: Cache prefix
+    type: string
+    default: ''
+  run_in_background:
+    description: Run poetry install in background mode
+    type: boolean
+    default: false
+  wait:
+    description: wait until all the commands are finished
+    type: boolean
+    default: false
+  run_in_container:
+    description: Run build in a container
+    type: boolean
+    default: false
+  request_remote_docker:
+    description: Request remote Docker
+    type: boolean
+    default: false
+  container_folder_to_copy:
+    description: Container folder to copy after the execution
+    type: string
+    default: ""
+steps:
+  - authenticate_poetry
+  - vf_python_restore_cache:
+      working_directory: << parameters.working_directory >>
+      cache_prefix: << parameters.cache_prefix >>
+  - yarn_command:
+      working_directory: << parameters.working_directory >>
+      run_in_background: << parameters.run_in_background >>
+      request_remote_docker: << parameters.request_remote_docker >>
+      container_folder_to_copy: << parameters.container_folder_to_copy >>
+      run_in_container: << parameters.run_in_container >>
+      step_name: << parameters.step_name >>
+      wait: << parameters.wait >>
+      yarn_command: poetry install --directory << parameters.working_directory >>/.poetry << parameters.install_args >>
+  - vf_python_save_cache: # special step to save the dependency cache
+      working_directory: << parameters.working_directory >>
+      cache_prefix: << parameters.cache_prefix >>

src/commands/poetry/persist_python_build.yml

@@ -0,0 +1,43 @@
+parameters:
+  package:
+    description: Package name
+    type: string
+    default: ""
+  package_folder:
+    description: Package folder
+    type: string
+    default: "packages"
+steps:
+  # Persist to workspace
+  - when:
+      condition:
+        equal:
+          - all
+          - "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - ./*/*/dist
+  - when:
+      condition:
+        and:
+          - "<< parameters.package >>"
+          - not:
+              equal:
+                - all
+                - "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - << parameters.package_folder >>/<< parameters.package >>/dist
+              - << parameters.package_folder >>/<< parameters.package >>/poetry.lock
+
+  - unless:
+      condition: "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - dist

src/commands/poetry/vf_python_restore_cache.yml

@@ -0,0 +1,13 @@
+parameters:
+  working_directory:
+    description: Cache directory for poetry.lock file
+    type: string
+    default: './'
+  cache_prefix:
+    description: Cache prefix
+    type: string
+    default: ''
+steps:
+  - restore_cache:
+      keys:
+        - python-package-cache-<< parameters.cache_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ checksum "<< parameters.working_directory >>/poetry.lock" }}

src/commands/poetry/vf_python_save_cache.yml

@@ -0,0 +1,14 @@
+parameters:
+  working_directory:
+    description: Directory containing package.json
+    type: string
+    default: './'
+  cache_prefix:
+    description: Cache prefix
+    type: string
+    default: ''
+steps:
+  - save_cache: # special step to save the dependency cache
+      key: python-package-cache-<< parameters.cache_prefix >>-{{ .Environment.CACHE_VERSION }}-{{ checksum "<< parameters.working_directory >>/poetry.lock" }}
+      paths:
+        - << parameters.working_directory >>/.venv

src/commands/track/update_track.yml

@@ -58,6 +58,10 @@ parameters:
     description: KMS Key to sign the containers
     type: string
     default: "awskms:///2e64fa98-d1b0-491a-acf6-1f5fc6f94ecf"
+  inject_aws_credentials:
+    description: Inject AWS credentials into the build
+    type: boolean
+    default: false
 steps:
   - when:
       condition: << parameters.checkout >>
@@ -99,4 +103,5 @@ steps:
         LOCAL_REGISTRY: "<< parameters.local_registry >>"
         BUILD_ARGS: "<< parameters.build_args >>"
         DOCKERFILE: "<< parameters.dockerfile >>"
+        INJECT_AWS_CREDENTIALS: "<< parameters.inject_aws_credentials >>"
       command: <<include(scripts/track/update_track.sh)>>

src/commands/yarn/install_node_modules.yml

@@ -43,8 +43,29 @@ parameters:
     description: Container folder to copy after the execution
     type: string
     default: ""
+  language:
+    description: language to execute
+    type: string
+    default: "node"
 steps:
   - authenticate_npm
+  - when:
+      condition:
+        equal:
+          - python
+          - "<< parameters.language >>"
+      steps:
+        - authenticate_poetry:
+            working_directory: << parameters.working_directory >>
+        - vf_python_restore_cache:
+            cache_prefix: << parameters.cache_prefix >>
+        - run:
+            name: Install Python dependencies
+            command: |
+              poetry install
+              source .venv/bin/activate
+              # Activates the virtual environment for the current job
+              echo "source .venv/bin/activate" >> $BASH_ENV
   - vf_restore_cache:
       yarn_lock_restore_cache_directory: << parameters.yarn_lock_restore_cache_directory >>
       cache_prefix: << parameters.cache_prefix >>
@@ -77,6 +98,15 @@ steps:
       name: Check yarn lock integrity
       command: <<include(scripts/yarn/check_lock_integrity.sh)>>
 
+  - when:
+      condition:
+        equal:
+          - python
+          - "<< parameters.language >>"
+      steps:
+        - vf_python_save_cache:
+            cache_prefix: << parameters.cache_prefix >>
+
   - vf_save_cache: # special step to save the dependency cache
       working_directory: << parameters.working_directory >>
       cache_prefix: << parameters.cache_prefix >>

src/commands/yarn/persist_node_build.yml

@@ -0,0 +1,43 @@
+parameters:
+  package:
+    description: Package name
+    type: string
+    default: ""
+  package_folder:
+    description: Package folder
+    type: string
+    default: "packages"
+steps:
+  # Persist to workspace
+  - when:
+      condition:
+        equal:
+          - all
+          - "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - ./*/*/build
+  - when:
+      condition:
+        and:
+          - "<< parameters.package >>"
+          - not:
+              equal:
+                - all
+                - "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - << parameters.package_folder >>/<< parameters.package >>/build
+              - << parameters.package_folder >>/<< parameters.package >>/yarn.lock
+
+  - unless:
+      condition: "<< parameters.package >>"
+      steps:
+        - persist_to_workspace:
+            root: "."
+            paths:
+              - build

src/executors/python/python-executor.yml

@@ -0,0 +1,12 @@
+parameters:
+  default_resource_class:
+    description: Default resource class for the executor
+    type: string
+    default: medium+
+docker:
+  - image: 168387678261.dkr.ecr.us-east-1.amazonaws.com/ci-image-python:v1
+    aws_auth:
+      aws_access_key_id: $AWS_ACCESS_KEY_ID
+      aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
+working_directory: ~/voiceflow
+resource_class: << parameters.default_resource_class >>

src/jobs/docker/build_push_image.yml

@@ -39,6 +39,10 @@ parameters:
     description: Semantic release version of the package
     type: string
     default: ""
+  inject_aws_credentials:
+    description: Inject AWS credentials into the build
+    type: boolean
+    default: false
 steps:
   - build_push_image:
       image_repo: "<< parameters.image_repo >>"
@@ -51,3 +55,4 @@ steps:
       package_folder: "<< parameters.package_folder >>"
       kms_key: "<< parameters.kms_key >>"
       sem_ver_override: "<< parameters.sem_ver_override >>"
+      inject_aws_credentials: "<< parameters.inject_aws_credentials >>"

src/jobs/monorepo/monorepo_release.yml

@@ -1,4 +1,4 @@
-executor: node-executor
+executor: << parameters.executor >>
 parameters:
   install_args:
     description: Additional yarn install command options
@@ -31,6 +31,10 @@ parameters:
     description: Release engine to use
     type: string
     default: "chore(release): publish"
+  language:
+    description: language to execute
+    type: string
+    default: "node"
   pre_publish_steps:
     description: Steps to run before executing the publish command
     type: steps
@@ -39,6 +43,10 @@ parameters:
     description: Steps to run after executing the publish command
     type: steps
     default: []
+  executor:
+    description: The executor used to run this job
+    type: executor
+    default: node-executor
 steps:
   - add_ssh_keys:
       fingerprints:
@@ -47,6 +55,7 @@ steps:
   - install_node_modules:
       install_args: "<< parameters.install_args >>"
       avoid_post_install_scripts: "<< parameters.avoid_post_install_scripts >>"
+      language: "<< parameters.language >>"
   - attach_workspace:
       at: ~/voiceflow
   - run:
@@ -79,6 +88,14 @@ steps:
                 --message "<< parameters.commit_message >>" --yes \
                 --conventional-commits --no-verify-access << parameters.publish_args >>
 
+  - when:
+      condition:
+        equal: ["nx", << parameters.release_engine >>]
+      steps:
+        - run:
+            name: Release Monorepo
+            command: yarn run publish
+
   - steps: << parameters.post_publish_steps >>
 
   - run:

src/jobs/track/update_track.yml

@@ -58,6 +58,10 @@ parameters:
     description: KMS Key to sign the containers
     type: string
     default: "awskms:///2e64fa98-d1b0-491a-acf6-1f5fc6f94ecf"
+  inject_aws_credentials:
+    description: Inject AWS credentials into the build
+    type: boolean
+    default: false
 steps:
   - update_track:
       image_repo: "<< parameters.image_repo >>"
@@ -75,3 +79,4 @@ steps:
       package_folder: "<< parameters.package_folder >>"
       image_tag: "<< parameters.image_tag >>"
       kms_key: "<< parameters.kms_key >>"
+      inject_aws_credentials: "<< parameters.inject_aws_credentials >>"