Update the paths in Dockerfiles (flyteorg#180)

* updated paths in docker-files Signed-off-by: Samhita Alla <aallasamhita@gmail.com> * updated dockerfile in pima diabetes example Signed-off-by: Samhita Alla <aallasamhita@gmail.com> * updated github workflows Signed-off-by: Samhita Alla <aallasamhita@gmail.com>
eapolinario · May 4, 2021 · aea197f · aea197f
1 parent ad490fe
commit aea197f
Show file tree

Hide file tree

Showing 20 changed files with 136 additions and 40 deletions.
diff --git a/.github/workflows/ghcr_k8s_spark.yml b/.github/workflows/ghcr_k8s_spark.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:k8s_spark-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: kubernetes/k8s_spark/Dockerfile
+          context: ./cookbook/integrations/kubernetes/
+          dockerfile: k8s_spark/Dockerfile
diff --git a/.github/workflows/ghcr_kfpytorch.yml b/.github/workflows/ghcr_kfpytorch.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:kfpytorch-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: kubernetes/kfpytorch/Dockerfile
+          context: ./cookbook/integrations/kubernetes/
+          dockerfile: kfpytorch/Dockerfile
diff --git a/.github/workflows/ghcr_pandera.yml b/.github/workflows/ghcr_pandera.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pandera-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: flytekit_plugins/pandera/Dockerfile
+          context: ./cookbook/integrations/flytekit_plugins/
+          dockerfile: pandera/Dockerfile
diff --git a/.github/workflows/ghcr_papermilltasks.yml b/.github/workflows/ghcr_papermilltasks.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:papermilltasks-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: flytekit_plugins/papermilltasks/Dockerfile
+          context: ./cookbook/integrations/flytekit_plugins/
+          dockerfile: papermilltasks/Dockerfile
diff --git a/.github/workflows/ghcr_pima_diabetes.yml b/.github/workflows/ghcr_pima_diabetes.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pima_diabetes-${{ github.sha }}"
-          context: ./cookbook/case_studies
-          dockerfile: ml_training/pima_diabetes/Dockerfile
+          context: ./cookbook/case_studies/ml_training/
+          dockerfile: pima_diabetes/Dockerfile
diff --git a/.github/workflows/ghcr_pod.yml b/.github/workflows/ghcr_pod.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:pod-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: kubernetes/pod/Dockerfile
+          context: ./cookbook/integrations/kubernetes/
+          dockerfile: pod/Dockerfile
diff --git a/.github/workflows/ghcr_sagemaker_pytorch.yml b/.github/workflows/ghcr_sagemaker_pytorch.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:sagemaker_pytorch-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: aws/sagemaker_pytorch/Dockerfile
+          context: ./cookbook/integrations/aws/
+          dockerfile: sagemaker_pytorch/Dockerfile
diff --git a/.github/workflows/ghcr_sagemaker_training.yml b/.github/workflows/ghcr_sagemaker_training.yml
@@ -29,5 +29,5 @@ jobs:
           push_image_and_stages: ${{ github.event_name != 'pull_request' }}
           registry: ghcr.io
           build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:sagemaker_training-${{ github.sha }}"
-          context: ./cookbook/integrations
-          dockerfile: aws/sagemaker_training/Dockerfile
+          context: ./cookbook/integrations/aws/
+          dockerfile: sagemaker_training/Dockerfile
diff --git a/cookbook/case_studies/ml_training/in_container.mk b/cookbook/case_studies/ml_training/in_container.mk
@@ -0,0 +1,24 @@
+SERIALIZED_PB_OUTPUT_DIR := /tmp/output
+
+.PHONY: clean
+clean:
+	rm -rf $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+$(SERIALIZED_PB_OUTPUT_DIR): clean
+	mkdir -p $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: serialize
+serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+.PHONY: fast_serialize
+fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: fast_register
+fast_register: fast_serialize
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/case_studies/ml_training/pima_diabetes/Dockerfile b/cookbook/case_studies/ml_training/pima_diabetes/Dockerfile
@@ -26,15 +26,15 @@ RUN python3.8 -m venv ${VENV}
 RUN ${VENV}/bin/pip install wheel
 
 # Install Python dependencies
-COPY ml_training/pima_diabetes/requirements.txt /root
+COPY pima_diabetes/requirements.txt /root
 RUN ${VENV}/bin/pip install -r /root/requirements.txt
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY ml_training/pima_diabetes/sandbox.config /root
+COPY pima_diabetes/sandbox.config /root
 
 # Copy the actual code
-COPY ml_training/pima_diabetes/ /root/pima_diabetes/
+COPY pima_diabetes/ /root/pima_diabetes/
 
 # Copy over the helper script that the SDK relies on
 RUN cp ${VENV}/bin/flytekit_venv /usr/local/bin/

diff --git a/cookbook/integrations/aws/in_container.mk b/cookbook/integrations/aws/in_container.mk
@@ -0,0 +1,24 @@
+SERIALIZED_PB_OUTPUT_DIR := /tmp/output
+
+.PHONY: clean
+clean:
+	rm -rf $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+$(SERIALIZED_PB_OUTPUT_DIR): clean
+	mkdir -p $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: serialize
+serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+.PHONY: fast_serialize
+fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: fast_register
+fast_register: fast_serialize
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/aws/sagemaker_pytorch/Dockerfile b/cookbook/integrations/aws/sagemaker_pytorch/Dockerfile
@@ -17,18 +17,18 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY aws/sagemaker_pytorch/requirements.txt /root/.
+COPY sagemaker_pytorch/requirements.txt /root/.
 RUN pip install -r /root/requirements.txt
 
 # Setup Sagemaker entrypoints
 ENV SAGEMAKER_PROGRAM /opt/venv/bin/flytekit_sagemaker_runner.py
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY aws/sagemaker_pytorch/sandbox.config /root
+COPY sagemaker_pytorch/sandbox.config /root
 
 # Copy the actual code
-COPY aws/sagemaker_pytorch/ /root/sagemaker_pytorch
+COPY sagemaker_pytorch/ /root/sagemaker_pytorch
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/aws/sagemaker_training/Dockerfile b/cookbook/integrations/aws/sagemaker_training/Dockerfile
@@ -16,18 +16,18 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY aws/sagemaker_training/requirements.txt /root
+COPY sagemaker_training/requirements.txt /root
 RUN pip install -r /root/requirements.txt
 
 # Setup Sagemaker entrypoints
 ENV SAGEMAKER_PROGRAM /opt/venv/bin/flytekit_sagemaker_runner.py
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY aws/sagemaker_training/sandbox.config /root
+COPY sagemaker_training/sandbox.config /root
 
 # Copy the actual code
-COPY aws/sagemaker_training/ /root/sagemaker_training
+COPY sagemaker_training/ /root/sagemaker_training
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/flytekit_plugins/in_container.mk b/cookbook/integrations/flytekit_plugins/in_container.mk
@@ -0,0 +1,24 @@
+SERIALIZED_PB_OUTPUT_DIR := /tmp/output
+
+.PHONY: clean
+clean:
+	rm -rf $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+$(SERIALIZED_PB_OUTPUT_DIR): clean
+	mkdir -p $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: serialize
+serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+.PHONY: fast_serialize
+fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: fast_register
+fast_register: fast_serialize
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/flytekit_plugins/pandera/Dockerfile b/cookbook/integrations/flytekit_plugins/pandera/Dockerfile
@@ -15,15 +15,15 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY flytekit_plugins/pandera/requirements.txt /root/.
+COPY pandera/requirements.txt /root/.
 RUN pip install -r /root/requirements.txt
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY flytekit_plugins/pandera/sandbox.config /root
+COPY pandera/sandbox.config /root
 
 # Copy the actual code
-COPY flytekit_plugins/pandera/ /root/pandera/
+COPY pandera/ /root/pandera/
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/flytekit_plugins/papermilltasks/Dockerfile b/cookbook/integrations/flytekit_plugins/papermilltasks/Dockerfile
@@ -15,15 +15,15 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY flytekit_plugins/papermilltasks/requirements.txt /root/.
+COPY papermilltasks/requirements.txt /root/.
 RUN pip install -r /root/requirements.txt
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY flytekit_plugins/papermilltasks/sandbox.config /root
+COPY papermilltasks/sandbox.config /root
 
 # Copy the actual code
-COPY flytekit_plugins/papermilltasks/ /root/papermilltasks/
+COPY papermilltasks/ /root/papermilltasks/
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/kubernetes/in_container.mk b/cookbook/integrations/kubernetes/in_container.mk
@@ -0,0 +1,24 @@
+SERIALIZED_PB_OUTPUT_DIR := /tmp/output
+
+.PHONY: clean
+clean:
+	rm -rf $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+$(SERIALIZED_PB_OUTPUT_DIR): clean
+	mkdir -p $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: serialize
+serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: register
+register: serialize
+	flyte-cli register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development -v ${VERSION} --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} $(SERIALIZED_PB_OUTPUT_DIR)/*
+
+.PHONY: fast_serialize
+fast_serialize: $(SERIALIZED_PB_OUTPUT_DIR)
+	pyflyte --config /root/sandbox.config serialize fast workflows -f $(SERIALIZED_PB_OUTPUT_DIR)
+
+.PHONY: fast_register
+fast_register: fast_serialize
+	flyte-cli fast-register-files -h ${FLYTE_HOST} ${INSECURE_FLAG} -p ${PROJECT} -d development --kubernetes-service-account ${SERVICE_ACCOUNT} --output-location-prefix ${OUTPUT_DATA_PREFIX} --additional-distribution-dir ${ADDL_DISTRIBUTION_DIR} $(SERIALIZED_PB_OUTPUT_DIR)/*
diff --git a/cookbook/integrations/kubernetes/k8s_spark/Dockerfile b/cookbook/integrations/kubernetes/k8s_spark/Dockerfile
@@ -21,7 +21,7 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY kubernetes/k8s_spark/requirements.txt /root
+COPY k8s_spark/requirements.txt /root
 RUN pip install -r /root/requirements.txt
 
 RUN flytekit_install_spark3.sh
@@ -39,10 +39,10 @@ ENV PYSPARK_DRIVER_PYTHON ${VENV}/bin/python3
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY kubernetes/k8s_spark/sandbox.config /root
+COPY k8s_spark/sandbox.config /root
 
 # Copy the actual code
-COPY kubernetes/k8s_spark/ /root/k8s_spark
+COPY k8s_spark/ /root/k8s_spark
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/kubernetes/kfpytorch/Dockerfile b/cookbook/integrations/kubernetes/kfpytorch/Dockerfile
@@ -18,15 +18,15 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY kubernetes/kfpytorch/requirements.txt /root
+COPY kfpytorch/requirements.txt /root
 RUN pip install -r /root/requirements.txt
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY kubernetes/kfpytorch/sandbox.config /root
+COPY kfpytorch/sandbox.config /root
 
 # Copy the actual code
-COPY kubernetes/kfpytorch/ /root/kfpytorch/
+COPY kfpytorch/ /root/kfpytorch/
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans

diff --git a/cookbook/integrations/kubernetes/pod/Dockerfile b/cookbook/integrations/kubernetes/pod/Dockerfile
@@ -16,15 +16,15 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY kubernetes/pod/requirements.txt /root/.
+COPY pod/requirements.txt /root/.
 RUN pip install -r /root/requirements.txt
 
 # Copy the makefile targets to expose on the container. This makes it easier to register.
 COPY in_container.mk /root/Makefile
-COPY kubernetes/pod/sandbox.config /root
+COPY pod/sandbox.config /root
 
 # Copy the actual code
-COPY kubernetes/pod/ /root/pod/
+COPY pod/ /root/pod/
 
 # This tag is supplied by the build script and will be used to determine the version
 # when registering tasks, workflows, and launch plans