feat: Faster ci test setup (#2489)

### Description

This PR speeds up the CI tests.



| Before: | After |
| --- | ----| 
|
![image](https://github.com/snakemake/snakemake/assets/1038978/4e3cf3a3-1a76-472b-8c15-7a3af68fd680)
|
![image](https://github.com/snakemake/snakemake/assets/1038978/418400da-1778-4a25-86df-ab3b25c2fe75)
|

This was done by using
[setup-micromamba](https://github.com/mamba-org/setup-micromamba) to
install mamba and the Python environment. This is especially fast
because the environment is cached.

Here are the speed improvements for micromamba install + environment
creation:

| Platform |  Before: | After: |
| --- | ----| ---- | 
| Linux | 3 min | 11 sec |
| Win | 12 min | 2.5 min | 

(maybe snakemake should use micromamba for its conda-jobs)...

The black formatting job is also much faster: Total time: 60 sec -> 11
sec.

A small speed up also comes from replacing the `cancel-previous` job
with a github-action built-feature (`concurrency`) which does the same.

Finally, a lot of speedup comes from adding a `.test_duration` file so
`pytest-split` can minimize test time by spreading the tests across jobs
in an optimal way. This cuts 30% of the actual test time.


### QC
<!-- Make sure that you can tick the boxes below. -->

* [x] The PR contains a test case for the changes or the changes are
already covered by an existing test case.
* [x] The documentation (`docs/`) is updated to reflect the changes or
this is not necessary (e.g. if the change does neither modify the
language nor the behavior or functionalities of Snakemake).

---------

Co-authored-by: Johannes Köster <johannes.koester@tu-dortmund.de>
Co-authored-by: Johannes Koester <johannes.koester@uni-due.de>

.github/workflows/main.yml

@@ -6,33 +6,33 @@ on:
       - main
   pull_request:
     branches_ignore: []
+
+
+concurrency: 
+  # Cancel concurrent flows on PRs
+  group: ci-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+
 jobs:
-  cancel-previous:
-    runs-on: ubuntu-latest
-    if: github.ref != 'refs/heads/main'
-    steps:
-      - uses: khan/pull-request-workflow-cancel@1.0.1
-        with:
-          workflows: main.yml
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
   formatting:
     permissions:
       contents: read # for actions/checkout to fetch code
       pull-requests: write # for marocchino/sticky-pull-request-comment to create or update PR comment
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Setup black environment
-        run: |
-          conda create -c conda-forge -y -q --name black black
-      - name: Check formatting
-        run: >
-          export PATH="/usr/share/miniconda/bin:$PATH"
+      - uses: actions/checkout@v4
 
-          source activate black
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: black
+          create-args: black
+          cache-environment: true
+
+      - name: Check formatting
+        shell: bash -el {0}
+        run: black --check --diff .
 
-          black --check --diff .
       - name: Comment PR
         if: github.event_name == 'pull_request' && failure()
         uses: marocchino/sticky-pull-request-comment@v2.8.0
@@ -46,58 +46,41 @@ jobs:
         test_group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     runs-on: ubuntu-latest
     needs: formatting
-    services:
-      mysql:
-        image: mysql:8.0
-        env:
-          MYSQL_ROOT_PASSWORD: root
-        ports:
-          - "8888:3306"
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     env:
       AWS_AVAILABLE: "${{ secrets.AWS_ACCESS_KEY_ID }}"
       GCP_AVAILABLE: "${{ secrets.GCP_SA_KEY }}"
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
-      - name: Setup mamba
-        uses: conda-incubator/setup-miniconda@v2
+            
+      - name: Setup snakemke environment
+        uses: mamba-org/setup-micromamba@v1
         with:
-          activate-environment: snakemake
-          channels: "conda-forge, bioconda"
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
-
-      - name: Setup Snakemake environment
+          environment-file: test-environment.yml
+          environment-name: snakemake
+          cache-environment: true          
+
+      - name: Install snakemake from source
         shell: bash -el {0}
         run: |
           conda config --set channel_priority strict
 
-          mamba env update -q -n snakemake --file test-environment.yml
-
           # TODO remove and add as regular dependency once released
           pip install git+https://github.com/snakemake/snakemake-interface-common.git
           pip install git+https://github.com/snakemake/snakemake-interface-executor-plugins.git
           pip install git+https://github.com/snakemake/snakemake-executor-plugin-cluster-generic.git
           pip install git+https://github.com/snakemake/snakemake-interface-storage-plugins.git
           pip install git+https://github.com/snakemake/snakemake-storage-plugin-http.git
           pip install git+https://github.com/snakemake/snakemake-storage-plugin-s3.git
-          pip install -e .
-
-          # additionally add singularity (not necessary anymore, included in the test env now)
-          # TODO remove version constraint: needed because 3.8.7 fails with missing libz:
-          # bin/unsquashfs: error while loading shared libraries: libz.so.1:
-          # cannot open shared object file: No such file or directory
-          # mamba install -n snakemake "singularity<=3.8.6"
+          pip install .
 
       - name: Setup apt dependencies
         run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update
-          sudo apt install -y stress git wget openmpi-bin libopenmpi-dev mariadb-server
+          sudo apt install -y stress git wget openmpi-bin libopenmpi-dev
 
       - name: Test local
         env:
@@ -106,18 +89,22 @@ jobs:
           #PYTHONTRACEMALLOC: 10
         shell: bash -el {0}
         run: |
-          # long tests
-          pytest --show-capture=stderr --splits 10 --group ${{ matrix.test_group }} -v -x tests/tests.py
-          # other tests
-          pytest --show-capture=stderr -v -x tests/test_expand.py tests/test_io.py tests/test_schema.py tests/test_linting.py tests/test_executor_test_suite.py tests/test_api.py
+          pytest -v -x --show-capture=stderr \
+            --splits 10 --group ${{ matrix.test_group }} --splitting-algorithm=least_duration \
+            tests/tests.py \
+            tests/test_expand.py \
+            tests/test_io.py \
+            tests/test_schema.py \
+            tests/test_linting.py \
+            tests/test_executor_test_suite.py \
+            tests/test_api.py
 
   build-container-image:
     runs-on: ubuntu-latest
     needs: testing
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v4
+
       - name: Build container image
         run: docker build .
 
@@ -139,18 +126,17 @@ jobs:
           for line in fileinput.input("test-environment.yml", inplace=True):
               if all(pkg not in line for pkg in excluded_on_win):
                   print(line)
-      - name: Setup miniconda
-        uses: conda-incubator/setup-miniconda@v2
+      
+      - name: Setup snakemke environment
+        uses: mamba-org/setup-micromamba@v1
         with:
-          activate-environment: snakemake
-          channels: "conda-forge, bioconda"
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
-      - name: Setup Snakemake environment
+          environment-file: test-environment.yml
+          environment-name: snakemake
+          init-shell: powershell
+          cache-environment: true
+
+      - name: Install snakemake from source
         run: |
-          conda config --set channel_priority strict
-          mamba env update -q --file test-environment.yml
-
           # TODO remove and add as regular dependency once released
           pip install git+https://github.com/snakemake/snakemake-interface-common.git
           pip install git+https://github.com/snakemake/snakemake-interface-executor-plugins.git
@@ -159,10 +145,11 @@ jobs:
           pip install git+https://github.com/snakemake/snakemake-storage-plugin-http.git
           pip install git+https://github.com/snakemake/snakemake-storage-plugin-s3.git
           
-          pip install -e .
+          pip install .
+
       - name: Run tests
         env:
           CI: true
           ZENODO_SANDBOX_PAT: "${{ secrets.ZENODO_SANDBOX_PAT }}"
         run: |
-          python -m pytest --show-capture=stderr -v -x --splits 10 --group ${{ matrix.test_group }} tests/tests.py
+          python -m pytest --show-capture=stderr -v -x --splits 10 --group ${{ matrix.test_group }} --splitting-algorithm=least_duration tests/tests.py