Add windows to CI (#319)

* Add windows to CI

* Try different shebang per actions/runner-images#7253 (comment)

* Revert "Try different shebang per actions/runner-images#7253 (comment)"

This reverts commit 4a8f093.

* Try setting PATH

* Reduce matrix for testing

* Set default shell to bash

* Fix wrong syntax

* Explicitly set bash path in test

* Fix syntax error

* Fix mistake

* Try installing GNU make instead of using available mingw32-make

* Undo explicit bash executable

* Set conda executable

* Fix black

* Add conda directory to system paths

* Clearer phony declaration

* Add diagnostic printing

* More printing

* Check which make

* Try adding Git bin directory to path

* Explicit bash shell

* Try adding explicit PATH

* Try making path style consistent

* Explicit conda executable

* try using which

* Try Scripts directory for adding to PATH

* Try removing unnecessary conda stuff

* Separate check step

* Install from conda-forge

* Missing colon

* Try setup miniconda to see if it's faster

* Change default shell

* Use conda environment

* Skip which conda

* Add to path anyways

* Try micromamba

* Use mamba as conda executable

* Fix typo

* micromamba shell hook

* Missing quote

* Full micromamba path; add caching

* Try path instead

* Remove backslash

* Use mambaforge instead of micromamba

* Add conda to PATH

* Use conda activate

* Try permissions hack

* Remove unneeded packages

* Use different repo name per config

* Cleanup

* Fallbacks

* Fix backwards check

* Print test durations

* Try conda and defaults again

* Don't change shell

* Normal shell

* Clean up

* More cleanup

* More cleanup

---------

Co-authored-by: Jay Qi <jayqi@users.noreply.github.com>

.github/workflows/tests.yml

@@ -42,8 +42,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+    defaults:
+      run:
+        shell: bash
 
     steps:
       - uses: actions/checkout@v3
@@ -58,11 +61,49 @@ jobs:
             pyproject.toml
             dev-requirements.txt
 
+      - name: Cache conda packages
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache explicitly
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('ccds/hook_utils/dependencies.py') }}
+
+      - name: Setup for Windows
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          # Install GNU make
+          choco install --verbose make
+          # Add conda binaries directory to PATH so that conda is accessible
+          echo "${CONDA}\Scripts" >> $GITHUB_PATH
+          # Set bash executable explicitly since Make may pick wrong shell
+          echo "BASH_EXECUTABLE=$(which bash)" >> "$GITHUB_ENV"
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           make requirements
 
+      - name: Check dependencies
+        run: |
+          if [[ -z "${BASH_EXECUTABLE}" ]]; then
+            bash --version
+          else
+            echo $BASH_EXECUTABLE
+            $BASH_EXECUTABLE --version
+          fi
+          which make
+          make --version
+          which conda
+          conda --version
+          which pipenv
+          pipenv --version
+          which virtualenv
+          virtualenv --version
+
       - name: Run tests
         run: |
           make test

Makefile

@@ -46,7 +46,7 @@ docs-serve:
 ###     TESTS
 
 test: _prep
-	pytest -vvv
+	pytest -vvv --durations=0
 
 test-fastest: _prep
 	pytest -vvv -FFF

ccds/hook_utils/dependencies.py

@@ -12,18 +12,18 @@ def write_dependencies(
 
     elif dependencies == "environment.yml":
         with open(dependencies, "w") as f:
-            lines = [f"name: {repo_name}", "dependencies:"]
+            lines = [
+                f"name: {repo_name}",
+                "dependencies:",
+            ]
 
+            lines += [f"  - python={python_version}"]
             lines += [f"  - {p}" for p in packages if p not in pip_only_packages]
 
-            lines += ["  - pip:"] + [
-                f"    - {p}" for p in packages if p in pip_only_packages
-            ]
-
+            lines += ["  - pip:"]
+            lines += [f"    - {p}" for p in packages if p in pip_only_packages]
             lines += ["    - -e ."]
 
-            lines += [f"  - python={python_version}"]
-
             f.write("\n".join(lines))
 
     elif dependencies == "Pipfile":

hooks/post_gen_project.py

@@ -14,8 +14,6 @@
     "isort",
     "pip",
     "python-dotenv",
-    "setuptools",
-    "wheel",
 ]
 
 # {% if cookiecutter.dataset_storage.s3 %}

tests/conftest.py

@@ -59,9 +59,10 @@ def _is_valid(config):
     # remove invalid configs
     configs = [c for c in configs if _is_valid(c)]
 
-    for c in configs:
+    for ind, c in enumerate(configs):
         config = dict(c)
         config.update(default_args)
+        config["repo_name"] += f"-{ind}"
         yield config
 
         # just do a single config if fast passed once or three times

tests/test_creation.py

@@ -1,10 +1,15 @@
+import json
+import os
 import sys
 from pathlib import Path
 from subprocess import PIPE, run
 
 from conftest import bake_project
 
 
+BASH_EXECUTABLE = os.getenv("BASH_EXECUTABLE", "bash")
+
+
 def no_curlies(filepath):
     """Utility to make sure no curly braces appear in a file.
     That is, was Jinja able to render everything?
@@ -21,7 +26,7 @@ def test_baking_configs(config, fast):
     """For every generated config in the config_generator, run all
     of the tests.
     """
-    print("using config", config)
+    print("using config", json.dumps(config, indent=2))
     with bake_project(config) as project_directory:
         verify_folders(project_directory, config)
         verify_files(project_directory, config)
@@ -139,7 +144,9 @@ def verify_makefile_commands(root, config):
         )
 
     result = run(
-        ["bash", str(harness_path), str(root.resolve())], stderr=PIPE, stdout=PIPE
+        [BASH_EXECUTABLE, str(harness_path), str(root.resolve())],
+        stderr=PIPE,
+        stdout=PIPE,
     )
     result_returncode = result.returncode
 
@@ -149,6 +156,7 @@ def verify_makefile_commands(root, config):
         encoding = "utf-8"
 
     # normally hidden by pytest except in failure we want this displayed
+    print("PATH=", os.getenv("PATH"))
     print("\n======================= STDOUT ======================")
     print(result.stdout.decode(encoding))
 

{{ cookiecutter.repo_name }}/Makefile

@@ -1,5 +1,3 @@
-.PHONY: clean data lint format requirements sync_data_down sync_data_up
-
 #################################################################################
 # GLOBALS                                                                       #
 #################################################################################
@@ -9,13 +7,13 @@ PROJECT_NAME = {{ cookiecutter.repo_name }}
 PYTHON_VERSION = {{ cookiecutter.python_version_number }}
 PYTHON_INTERPRETER = python
 
-
 #################################################################################
 # COMMANDS                                                                      #
 #################################################################################
 
 {% if cookiecutter.dependency_file != 'none' %}
 ## Install Python Dependencies
+.PHONY: requirements
 requirements:
 	{% if "requirements.txt" == cookiecutter.dependency_file -%}
 	$(PYTHON_INTERPRETER) -m pip install -U pip setuptools wheel
@@ -29,22 +27,26 @@ requirements:
 
 
 ## Delete all compiled Python files
+.PHONY: clean
 clean:
 	find . -type f -name "*.py[co]" -delete
 	find . -type d -name "__pycache__" -delete
 
 ## Lint using flake8 and black (use `make format` to do formatting)
+.PHONY: lint
 lint:
 	flake8 {{ cookiecutter.module_name }}
 	black --check --config pyproject.toml {{ cookiecutter.module_name }}
 
 
 ## Format source code with black
+.PHONY: format
 format:
 	black --config pyproject.toml {{ cookiecutter.module_name }}
 
 {% if not cookiecutter.dataset_storage.none %}
 ## Download Data from storage system
+.PHONY: sync_data_down
 sync_data_down:
 	{% if cookiecutter.dataset_storage.s3 -%}
 	aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/\
@@ -57,6 +59,7 @@ sync_data_down:
 	{% endif %}
 
 ## Upload Data to storage system
+.PHONY: sync_data_up
 sync_data_up:
 	{% if cookiecutter.dataset_storage.s3 -%}
 	aws s3 sync s3://{{ cookiecutter.dataset_storage.s3.bucket }}/data/ data/\
@@ -71,6 +74,7 @@ sync_data_up:
 
 {% if cookiecutter.environment_manager != 'none' %}
 ## Set up python interpreter environment
+.PHONY: create_environment
 create_environment:
 	{% if cookiecutter.environment_manager == 'conda' -%}
 	{% if cookiecutter.dependency_file != 'environment.yml' %}
@@ -94,6 +98,7 @@ create_environment:
 #################################################################################
 
 ## Make Dataset
+.PHONY: data
 data: requirements
 	$(PYTHON_INTERPRETER) {{ cookiecutter.module_name }}/data/make_dataset.py