From c71057dd5d54f345c7704b2d88c2b43a459d3bde Mon Sep 17 00:00:00 2001 From: Scott Gigante Date: Fri, 13 May 2022 15:27:03 -0400 Subject: [PATCH] fix docker hack --- .github/workflows/run_tests.yml | 11 ++++--- workflow/Snakefile | 23 +++++++------- workflow/snakemake_tools.py | 56 ++++++++++++++------------------- 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 8676211fc2..c30847e4cf 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -53,18 +53,19 @@ jobs: python -c "import openproblems" - name: Build Docker images + if: "!startsWith(github.ref, 'refs/heads/main')" run: | cd workflow - snakemake -j $(grep -c processor /proc/cpuinfo) docker_build + snakemake -j $(nproc) docker_build cd .. - - name: Push Docker images - if: startsWith(github.ref, 'refs/heads/main') + - name: Build and push Docker images + if: "startsWith(github.ref, 'refs/heads/main')" env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | cd workflow - snakemake -j $(grep -c processor /proc/cpuinfo) docker_push + snakemake -j $(nproc) docker_build docker_push cd .. echo "CHANGED=`git diff --exit-code > /dev/null && echo false || echo true`" >> $GITHUB_ENV @@ -342,7 +343,7 @@ jobs: AWS_DEFAULT_REGION: us-west-2 NXF_DEFAULT_DSL: 1 run: | - RUN_NAME="$(echo "$BRANCH" | sed "s/[^a-z]//g")_$(git rev-parse --short HEAD)_${GITHUB_RUN_ATTEMPT}" + RUN_NAME="$(echo "$BRANCH" | sed "s/[^a-zA-Z0-9]//g")_$(git rev-parse --short HEAD)_${GITHUB_RUN_ATTEMPT}" cd /mnt/openproblems-nextflow/cwd/${BRANCH} nextflow run \ -revision v1.2 \ diff --git a/workflow/Snakefile b/workflow/Snakefile index b474090ace..8309746c1e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -26,18 +26,12 @@ rule docker_pull: rule docker_refresh: input: tools.refresh_images -# this is a dirty hack -if "docker_build" in sys.argv: - ruleorder: build_docker_image > refresh_docker_image -else: - ruleorder: refresh_docker_image > build_docker_image - rule refresh_docker_image: input: dockerfile = "{}/{{image}}/refresh.Dockerfile".format(tools.IMAGES_DIR), + requirements = tools.docker_refresh_requirements, output: - temp(touch("{}/{{image}}/.docker_refresh".format(tools.IMAGES_DIR))), - temp(touch("{}/{{image}}/.docker_update".format(tools.IMAGES_DIR))) + temp(touch("{}/{{image}}/.docker_refresh".format(tools.IMAGES_DIR))) params: sourcedir = os.path.dirname(tools.SCRIPTS_DIR), user = "singlecellopenproblems", @@ -69,13 +63,18 @@ RUN cd /usr/src/singlecellopenproblems && sudo git clean -fxdq RUN sudo pip install --no-cache-dir --editable /usr/src/singlecellopenproblems ' > {output}""" +rule update_docker_image: + input: + tools.docker_update_requirements + output: + temp(touch("{}/{{image}}/.docker_update".format(tools.IMAGES_DIR))) + rule build_docker_image: input: dockerfile = "{}/{{image}}/Dockerfile".format(tools.IMAGES_DIR), - requirements = tools.docker_requirements, + requirements = tools.docker_build_requirements, output: - temp(touch("{}/{{image}}/.docker_build".format(tools.IMAGES_DIR))), - temp(touch("{}/{{image}}/.docker_update".format(tools.IMAGES_DIR))) + temp(touch("{}/{{image}}/.docker_build".format(tools.IMAGES_DIR))) params: sourcedir = os.path.dirname(tools.SCRIPTS_DIR), user = "singlecellopenproblems", @@ -100,7 +99,7 @@ rule login_docker: rule push_docker_image: input: - build = tools.docker_push_requirements, + build = "{}/{{image}}/.docker_update".format(tools.IMAGES_DIR), login = ".docker_login", output: temp(touch("{}/{{image}}/.docker_push".format(tools.IMAGES_DIR))) diff --git a/workflow/snakemake_tools.py b/workflow/snakemake_tools.py index 9b2fa5804b..cb1ec17c0b 100644 --- a/workflow/snakemake_tools.py +++ b/workflow/snakemake_tools.py @@ -48,15 +48,6 @@ def build_type(wildcards): return "github_actions" if "GITHUB_ACTIONS" in os.environ else "local" -def image_markers(wildcards): - """Get the appropriate marker for each image.""" - return [ - docker_image_marker(image) - for image in os.listdir(IMAGES_DIR) - if os.path.isdir(os.path.join(IMAGES_DIR, image)) - ] - - def push_images(wildcards): """Get Docker push timestamp for all images.""" images = _images(".docker_push") @@ -321,21 +312,24 @@ def docker_image_marker(image, refresh=True): """Get the file to be created to ensure Docker image exists from the image name.""" docker_path = os.path.join(IMAGES_DIR, image) # possible outputs - docker_refresh = os.path.join(docker_path, ".docker_update") + docker_refresh = os.path.join(docker_path, ".docker_refresh") dockerfile = os.path.join(docker_path, "Dockerfile") no_change = docker_refresh if refresh else dockerfile no_change_text = "refreshing source code only" if refresh else "no change" docker_build = os.path.join(docker_path, ".docker_build") - # inputs to conditional logic - local_imagespec_changed = docker_imagespec_changed(image, dockerfile) - local_codespec_changed = not version_not_changed() - if local_codespec_changed: + if "docker_build" in sys.argv: + # building everything from scratch + requirement_file = docker_build + elif not version_not_changed(): print( "Code version changed: {}".format(openproblems.__version__), file=sys.stderr ) - if local_imagespec_changed or local_codespec_changed: - # spec has changed, let's rebuild + # codebase has changed, let's rebuild + print("{}: rebuilding".format(image), file=sys.stderr) + requirement_file = docker_build + elif docker_imagespec_changed(image, dockerfile): + # image has changed, let's rebuild print("{}: rebuilding".format(image), file=sys.stderr) requirement_file = docker_build elif docker_image_exists(image, local=True) or docker_image_exists( @@ -352,11 +346,13 @@ def docker_image_marker(image, refresh=True): return requirement_file -def _docker_requirements(image, include_self=False, refresh=True): +def _docker_requirements(image, refresh=True): """Get all files to ensure a Docker image is up to date from the image name.""" docker_dir = os.path.join(IMAGES_DIR, image) - dockerfile = os.path.join(docker_dir, "Dockerfile") - requirements = [dockerfile] + requirements = [] + if not refresh: + dockerfile = os.path.join(docker_dir, "Dockerfile") + requirements.append(dockerfile) requirements.extend( [ os.path.join(docker_dir, f) @@ -364,24 +360,25 @@ def _docker_requirements(image, include_self=False, refresh=True): if f.endswith("requirements.txt") ] ) - if include_self: - requirements.append(docker_image_marker(image, refresh=refresh)) base_image = _docker_base(image) if base_image is not None: - requirements.extend( - _docker_requirements(base_image, include_self=True, refresh=refresh) - ) + requirements.append(os.path.join(IMAGES_DIR, base_image, ".docker_update")) return requirements -def docker_requirements(wildcards): +def docker_build_requirements(wildcards): """Get all files to ensure a Docker image is up to date from wildcards.""" - return _docker_requirements(wildcards.image) + return _docker_requirements(wildcards.image, refresh=False) def docker_update_requirements(wildcards): + """Check if we need to refresh or build a docker image.""" + return docker_image_marker(wildcards.image, refresh=True) + + +def docker_refresh_requirements(wildcards): """Get all files to ensure a Docker image is built and up to date from wildcards.""" - return _docker_requirements(wildcards.image, include_self=True, refresh=True) + return _docker_requirements(wildcards.image, refresh=True) def docker_push_requirements(wildcards): @@ -389,11 +386,6 @@ def docker_push_requirements(wildcards): return _docker_requirements(wildcards.image, include_self=True, refresh=False) -def docker_push(wildcards): - """Get the file to be created to ensure Docker image exists from wildcards.""" - return docker_image_marker(docker_image_name(wildcards)) - - def docker_command(wildcards, output): """Get the Docker command to be run given a set of wildcards.""" image = docker_image_name(wildcards)