Merge pull request #1445 from firesim/dedup-ci-reqs

Dedup. CI Requirements

.github/actions/change-workflow-instance-states/action.yml

@@ -21,4 +21,4 @@ runs:
           aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - run: .github/scripts/change-workflow-instance-states.py ${{ inputs.platform }} ${{ github.run_id }} ${{ inputs.new-state }} ${{ inputs.github-token }}
-        shell: bash
+        shell: bash -el {0}

.github/actions/repo-setup-aws/action.yml

@@ -14,4 +14,3 @@ runs:
         with:
           key: ${{ env.FIRESIM_PEM }}
           known_hosts: unnecessary
-      - uses: ./.github/actions/install-ci-python-reqs

.github/workflows/firesim-cleanup.yml

@@ -28,8 +28,15 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TERM: xterm-256-color
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          environment-file: conda-reqs/ci-shared.yaml
+          miniforge-version: latest
       - uses: ./.github/actions/repo-setup-aws
       - run: .github/scripts/cull-old-ci-instances.py
       - if: ${{ failure() }}
@@ -46,7 +53,14 @@ jobs:
     runs-on: ubuntu-latest
     env:
       TERM: xterm-256-color
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          environment-file: conda-reqs/ci-shared.yaml
+          miniforge-version: latest
       - uses: ./.github/actions/repo-setup-aws
       - run: .github/scripts/cull-old-ci-runners.py

.github/workflows/firesim-run-tests.yml

@@ -74,11 +74,12 @@ jobs:
 
             # If conda requirements was modified ensure the lock file is re-generated
             conda-reqs:
-                - 'conda-reqs.yaml'
+                - 'conda-reqs/firesim.yaml'
+                - 'conda-reqs/ci-shared.yaml'
 
             # If conda requirements was modified ensure the lock file is re-generated
             conda-lock:
-                - 'conda-reqs.conda-lock.yml'
+                - 'conda-reqs/conda-reqs.conda-lock.yml'
 
   # Note: This doesn't check if the lock file is synced/faithful to the requirements file.
   # This just ensures that both were modified in the same PR (ideally the lock file was regenerated
@@ -92,18 +93,24 @@ jobs:
     steps:
       - name: Check conda lock file was regenerated with conda requirements file
         run: |
-          echo "ERROR: Either the conda-reqs.yaml or conda-reqs.conda-lock.yml was not updated properly. See the developer docs for more information"
+          echo "ERROR: Either the conda-reqs/{firesim,ci-shared}.yaml or conda-reqs/conda-reqs.conda-lock.yml was not updated properly. See the developer docs for more information"
           false
 
   setup-self-hosted-manager:
     name: setup-self-hosted-manager
     needs: change-filters
     if: needs.change-filters.outputs.needs-manager == 'true'
     runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v3
-      - name: Install Python CI requirements
-        uses: ./.github/actions/repo-setup-aws
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          environment-file: conda-reqs/ci-shared.yaml
+          miniforge-version: latest
+      - uses: ./.github/actions/repo-setup-aws
       - name: Launch AWS instance used for the FireSim manager (instance info found here)
         run: ./.github/scripts/launch-manager-instance.py
       - name: Setup N Github Actions Runners on AWS instance

build-setup-nolog.sh

@@ -136,11 +136,10 @@ if [ "$IS_LIBRARY" = true ]; then
     fi
 else
     # note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154
-    LOCKFILE="$RDIR/conda-reqs.conda-lock.yml"
-    YAMLFILE="$RDIR/conda-reqs.yaml"
+    LOCKFILE="$RDIR/conda-reqs/conda-reqs.conda-lock.yml"
     if [ "$USE_PINNED_DEPS" = false ]; then
         # auto-gen the lockfile
-        conda-lock -f "$YAMLFILE" --lockfile "$LOCKFILE"
+        conda-lock -f "$RDIR/conda-reqs/firesim.yaml" -f "$RDIR/conda-reqs/ci-shared.yaml" --lockfile "$LOCKFILE"
     fi
     conda-lock install -p $RDIR/.conda-env $LOCKFILE
     source $RDIR/.conda-env/etc/profile.d/conda.sh

conda-reqs/ci-shared.yaml

@@ -0,0 +1,40 @@
+channels:
+    - conda-forge
+    - ucb-bar
+    - nodefaults
+
+# remove non-standard conda-lock 'platforms' key to use this file with conda-incubator/setup-miniconda GH action
+
+dependencies:
+    # https://conda-forge.org/feedstock-outputs/
+    #   filterable list of all conda-forge packages
+    # https://conda-forge.org/#contribute
+    #   instructions on adding a recipe
+    # https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/pkg-specs.html#package-match-specifications
+    #   documentation on package_spec syntax for constraining versions
+    - boto3
+    - boto3-stubs
+    - mypy-boto3-s3
+    - mypy_boto3_ec2
+    - botocore-stubs
+
+    - pytz
+    - types-pytz
+
+    - pyyaml
+    - types-pyyaml
+
+    - requests
+    - types-requests
+
+    - azure-identity
+    - azure-mgmt-compute # ci-only
+    - azure-mgmt-network # ci-only
+    - azure-mgmt-resource>=18 # ci-only
+
+    - fsspec
+
+    - pip
+    - pip:
+        - fab-classic>=1.19.2
+        - azure-mgmt-resourcegraph # ci-only