From 6aed57a30e49dc1d14b839eb2876c27ec3716e82 Mon Sep 17 00:00:00 2001 From: Mukul Chandrakant Mahadik Date: Mon, 18 Mar 2024 16:47:19 -0700 Subject: [PATCH 01/56] Create image_build_push.yml --- .github/workflows/image_build_push.yml | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/image_build_push.yml diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml new file mode 100644 index 00000000..11461d22 --- /dev/null +++ b/.github/workflows/image_build_push.yml @@ -0,0 +1,49 @@ +# This is a basic workflow to help you get started with Actions + +name: docker-image-push-public-dash + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ image-push-merge ] + + +# Env variable +env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: docker login + run: | # log into docker hub account + echo "Docker user name: " $DOCKER_USER + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + + - name: Get current date # get the date of the build + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')" + + #Runs a single command using the runners shell + - name: Run a one-line script + run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} + + # Runs a set of commands using the runners shell + - name: build docker image + run: | + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker images + + - name: push docker image + run: | + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From 8b0317fcba25253bf68f4350263a50cc4294d68c Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 21 Mar 2024 17:19:43 -0700 Subject: [PATCH 02/56] Added docker build and push for dashboard and notebook images + Updated viz_scripts Dockerfile viz_scripts Dockerfile contains ENV variables from docker-compose as well. Added docker image commands to image_build_push yml for dashboard and notebook images --- .github/workflows/image_build_push.yml | 6 ++++-- viz_scripts/Dockerfile | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 11461d22..f020887b 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -41,9 +41,11 @@ jobs: # Runs a set of commands using the runners shell - name: build docker image run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_dashboard:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts docker images - name: push docker image run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_dashboard:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 9dea3301..a06183b3 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -3,6 +3,11 @@ FROM shankari/e-mission-server:master_2024-02-10--19-38 VOLUME /plots +ENV DB_HOST=db +ENV WEB_SERVER_HOST=0.0.0.0 +ENV CRON_MODE= +ENV STUDY_CONFIG=stage-program + ADD docker/environment36.dashboard.additions.yml / WORKDIR /usr/src/app From 355db20f75729aaa4d420cfe5136f945f25ee50a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 25 Mar 2024 17:47:57 -0700 Subject: [PATCH 03/56] Changed sed to jq + Renamed docker image in image_push 1. Sed to jq change to make it consistent to what is being used in internal repos. 2. Renamed image pushed to docker hub. --- .github/workflows/image_build_push.yml | 4 ++-- viz_scripts/docker/start_notebook.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index f020887b..857d3f9a 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -41,11 +41,11 @@ jobs: # Runs a set of commands using the runners shell - name: build docker image run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_dashboard:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts docker images - name: push docker image run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_dashboard:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} diff --git a/viz_scripts/docker/start_notebook.sh b/viz_scripts/docker/start_notebook.sh index 33ba7ee6..e2d0dd95 100755 --- a/viz_scripts/docker/start_notebook.sh +++ b/viz_scripts/docker/start_notebook.sh @@ -5,9 +5,9 @@ echo "DB host = "${DB_HOST} if [ -z ${DB_HOST} ] ; then local_host=`hostname -i` - sed "s-localhost-${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf else - sed "s-localhost-${DB_HOST}-" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf fi ### configure the saved-notebooks directory for persistent notebooks From e402a613b67527ebfcd245269e70a91e4256d059 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 26 Mar 2024 10:37:57 -0700 Subject: [PATCH 04/56] Added TODO to change image push branch Currently the branch specified - "image-push-merge" is available locally on my system. I use it to test the automated docker image push mechanism whenever any changes are merged to this branch. Once, everything looks good, need to change this to master or main as per the repo. --- .github/workflows/image_build_push.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 857d3f9a..1756c186 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,6 +6,10 @@ name: docker-image-push-public-dash # events but only for the master branch on: push: + # Mukul: + # I've added a local test branch on my system and using it for testing image push. + # So, for testing purposes, need to checkout a branch "image-push-merge" + # TODO: Need to change to build off master or main once it looks good. branches: [ image-push-merge ] From d6a6809ef8efe0c2b9978cca6fa264ee6a861c8d Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 26 Mar 2024 11:18:21 -0700 Subject: [PATCH 05/56] Removed printing Docker username Had added it initially for testing purposes. Can remove now so it doesn't expose any sensitive info. --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1756c186..7a5e5e84 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -31,7 +31,6 @@ jobs: - uses: actions/checkout@v2 - name: docker login run: | # log into docker hub account - echo "Docker user name: " $DOCKER_USER docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - name: Get current date # get the date of the build From 50a7b6335afa1475f54445176758681bfe5d6b3a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 28 Mar 2024 15:53:19 -0700 Subject: [PATCH 06/56] Added cert.sh + Modified Dockerfiles 1. docker/cert.sh - Uses new environment variable PROD_STAGE variable to determine whether in staging / production environment and only then install certificates, else skip. 2. Dockerfile Added environment variables here, keeping the same default ENV values as the ones in docker-compose.yml. Not adding in docker/Dockerfile.dev, since this change is being done primarily with the objective to aid with the automated build and push to Dockerhub. This pushed image would then be the one that would be used in the internal environments as the base image, which would be based on the non-dev Dockerfile. --- viz_scripts/Dockerfile | 7 ++++++- viz_scripts/docker/cert.sh | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 viz_scripts/docker/cert.sh diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index a06183b3..dbc95b77 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -5,8 +5,9 @@ VOLUME /plots ENV DB_HOST=db ENV WEB_SERVER_HOST=0.0.0.0 -ENV CRON_MODE= +ENV CRON_MODE=TRUE ENV STUDY_CONFIG=stage-program +ENV PROD_STAGE=TRUE ADD docker/environment36.dashboard.additions.yml / @@ -38,6 +39,10 @@ RUN chmod u+x /usr/src/app/.docker/start_notebook.sh ADD docker/crontab /usr/src/app/crontab +ADD docker/cert.sh /usr/src/app/.docker/cert.sh +RUN chmod u+x /usr/src/app/.docker/cert.sh +RUN /usr/src/app/.docker/cert.sh + EXPOSE 8888 CMD ["/bin/bash", "/usr/src/app/.docker/start_notebook.sh"] diff --git a/viz_scripts/docker/cert.sh b/viz_scripts/docker/cert.sh new file mode 100644 index 00000000..36331687 --- /dev/null +++ b/viz_scripts/docker/cert.sh @@ -0,0 +1,6 @@ +if [ -z ${PROD_STAGE} ] ; then + echo “Not in staging / production environment, continuing build...” +elif [ ${PROD_STAGE} = "TRUE" ] ; then + wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem -O /etc/ssl/certs/rds-combined-ca-bundle.pem + echo "In staging / production environment, added AWS certificates" +fi From 2dab7a1bc7e1fcebda7e9aca0f755ef6b957cf5a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 2 Apr 2024 17:12:44 -0700 Subject: [PATCH 07/56] Removing ENV variables from Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other ENV vars that were added in Docker-compose files are currently present in the Dockerfile in the external repo root. Removing them from there and can add them as per requirement. For local testing, add them when “docker run” is used using the -e flag. For usage in stage / production, can be set by cloud team in AWS Codebuild as they currently do. CRON_MODE CRON_MODE can be moved to setting when docker run command is executed using -e flag. This is because this is required in start_notebook.sh in CMD layer in Dockerfile which is executed when the container is run. Hence, CRON_MODE value can be supplied later and not needed to be embedded in image. PROD_STAGE Not setting PROD_TRUE in docker run command since it is required during docker image build to decide whether to add certificates or not. Hence, adding it in the Dockerfile so it’s available during docker build. --- viz_scripts/Dockerfile | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index dbc95b77..9dea3301 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -3,12 +3,6 @@ FROM shankari/e-mission-server:master_2024-02-10--19-38 VOLUME /plots -ENV DB_HOST=db -ENV WEB_SERVER_HOST=0.0.0.0 -ENV CRON_MODE=TRUE -ENV STUDY_CONFIG=stage-program -ENV PROD_STAGE=TRUE - ADD docker/environment36.dashboard.additions.yml / WORKDIR /usr/src/app @@ -39,10 +33,6 @@ RUN chmod u+x /usr/src/app/.docker/start_notebook.sh ADD docker/crontab /usr/src/app/crontab -ADD docker/cert.sh /usr/src/app/.docker/cert.sh -RUN chmod u+x /usr/src/app/.docker/cert.sh -RUN /usr/src/app/.docker/cert.sh - EXPOSE 8888 CMD ["/bin/bash", "/usr/src/app/.docker/start_notebook.sh"] From 89ed59a86d2747f88971980b73b16070583d1b43 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 11 Apr 2024 20:01:50 -0700 Subject: [PATCH 08/56] Removed sed / jq usage from start scripts Can directly set DB_HOST since we can now use environment variables. No need to use jq or sed to replace file contents and copy over files. --- viz_scripts/docker/start_notebook.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/viz_scripts/docker/start_notebook.sh b/viz_scripts/docker/start_notebook.sh index e2d0dd95..5b646fb1 100755 --- a/viz_scripts/docker/start_notebook.sh +++ b/viz_scripts/docker/start_notebook.sh @@ -5,9 +5,8 @@ echo "DB host = "${DB_HOST} if [ -z ${DB_HOST} ] ; then local_host=`hostname -i` - jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf -else - jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf + export DB_HOST=$local_host + echo "Setting db host environment variable to localhost" fi ### configure the saved-notebooks directory for persistent notebooks From e962d13df747b5530693c1d84e3ba7ecac160d8b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 12 Apr 2024 02:47:15 -0700 Subject: [PATCH 09/56] Changing base image to build from redesign server image Created a new branch image-push-merge for e-mission-server in my forked repo. Also modified image push workflow to build and push docker image to docker hub on push to image-push-merge branch. Doing this since admin-dash was failing when was building from internal repo since this was still referring to old server code. Changed Dockerfile and docker/Dockerfile.dev in public-dash and admin-dash to build from this new image mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 Redesigned server image is built from the image-push-merge branch on my personal forked repository. This branch has a workflow run set up to build the docker image and push it to Dockerhub whenever a push or merge happens to image-push-merge branch. Currently, I've been pushing to image-push and then creating a PR to merge into image-push-merge. Doing this, so admin-dash and public-dash can build from the latest redesigned server code. This is not the latest server code but the latest changes from my redesign PR. --- viz_scripts/Dockerfile | 3 ++- viz_scripts/docker/Dockerfile.dev | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 9dea3301..0fc8db11 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,5 +1,6 @@ # python 3 -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 VOLUME /plots diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 2fe505b2..e2e048bf 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,5 +1,6 @@ # python 3 -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 VOLUME /plots From 8d2464d1baccb3f48638cde945720e6e87954232 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 15 Apr 2024 18:39:51 -0700 Subject: [PATCH 10/56] Bumped up base server image tag Changing to build from base server image from my personal Dockerhub repository with redesigned server code. Will need to change to build from Shankari's Dockerhub repository, once all changes are final. --- viz_scripts/Dockerfile | 4 +++- viz_scripts/docker/Dockerfile.dev | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 9dea3301..1b6cc1a2 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,5 +1,7 @@ # python 3 -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 VOLUME /plots diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 2fe505b2..f9ba68ae 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,5 +1,7 @@ # python 3 -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +# Please change once all PR changes are final, so it reads from shankari/e-mission-server +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 VOLUME /plots From baac678814303e7d22fa677c8de17bd3ac801eb1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 15 Apr 2024 20:57:11 -0700 Subject: [PATCH 11/56] Bump up base server image tag --- viz_scripts/Dockerfile | 2 +- viz_scripts/docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 1b6cc1a2..dc03956a 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,7 +1,7 @@ # python 3 # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 VOLUME /plots diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index f9ba68ae..fc20d02a 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,7 +1,7 @@ # python 3 # FROM shankari/e-mission-server:master_2024-02-10--19-38 # Please change once all PR changes are final, so it reads from shankari/e-mission-server -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 VOLUME /plots From fee9437992505c251036e2bf5adea55bdd7f190e Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 01:34:51 -0700 Subject: [PATCH 12/56] Artifact download test - 1 Added working code from join repo to fetch docker image tags using artifact download. --- .github/fetch_runID.py | 67 +++++++++++++++++++ .github/workflows/image_build_push.yml | 89 +++++++++++++++++++++----- 2 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 .github/fetch_runID.py diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py new file mode 100644 index 00000000..c40005d5 --- /dev/null +++ b/.github/fetch_runID.py @@ -0,0 +1,67 @@ + +import json +import logging +import requests +import sys + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + + # Just checking to see if workflows are being fetched; it works!!! + # download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows" + # logging.debug("About to fetch workflows present in e-mission-server from %s" % download_url) + # r = requests.get(download_url) + # if r.status_code != 200: + # logging.debug(f"Unable to fetch workflows, status code: {r.status_code}") + # sys.exit(1) + # else: + # workflows_json = json.loads(r.text) + # logging.debug(f"Successfully fetched workflows") + # print(workflows_json) + + # for workflow_entry in workflows_json["workflows"]: + # print("Workflow name: %s ; id: %s " % (workflow_entry["name"], workflow_entry["id"])) + + + + ''' + Workflow "docker image" uses image_build_push.yml + From above commented out code, and checked via terminal as well, + workflow id for the "docker image" can be fetched using: + https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows + https://api.github.com/repos/e-mission/e-mission-server/actions/workflows + + For MukuFlash03: id = 75506902 + For e-mission-server: id = 35580278 + ''' + download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) + r = requests.get(download_url) + if r.status_code != 200: + logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}") + sys.exit(1) + else: + workflow_runs_json = json.loads(r.text) + logging.debug(f"Successfully fetched workflow runs") + # print(workflow_runs_json) + + workflow_runs = workflow_runs_json["workflow_runs"] + if workflow_runs: + successful_runs = [run for run in workflow_runs \ + if run["status"] == "completed" and \ + run["conclusion"] == "success" and \ + run["head_branch"] == "tags-artifact" + ] + # print(successful_runs) + if successful_runs: + sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) + sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) + # print(sorted_runs) + latest_run_id = sorted_runs[0]["id"] + # print(latest_run_id) + print(f"::set-output name=run_id::{latest_run_id}") + # else: + # print("No successful runs") + # else: + # print("No workflow runs found") diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 7a5e5e84..7e0621ff 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,11 +6,11 @@ name: docker-image-push-public-dash # events but only for the master branch on: push: - # Mukul: - # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "image-push-merge" - # TODO: Need to change to build off master or main once it looks good. - branches: [ image-push-merge ] + # Mukul: + # I've added a local test branch on my system and using it for testing image push. + # So, for testing purposes, need to checkout a branch "tags-artifact" + # TODO: Need to change to build off master or main once it looks good. + branches: [ tags-artifact ] # Env variable @@ -20,11 +20,46 @@ env: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: + + fetch_run_id: + runs-on: ubuntu-latest + + outputs: + run_id: ${{ steps.get_run_id.outputs.run_id }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Run Python script + id: run_script + run: | + echo "Fetching latest successful run ID from e-misison-server docker image workflow" + python .github/fetch_runID.py + + - name: Get Run ID + id: get_run_id + run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" + # This workflow contains a single job called "build" build: + needs: fetch_run_id + # The type of runner that the job will run on runs-on: ubuntu-latest + env: + RUN_ID: ${{needs.fetch_run_id.outputs.run_id}} + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -42,13 +77,37 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - - name: build docker image - run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts - docker images - - - name: push docker image - run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + # - name: build docker image + # run: | + # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts + # docker images + + # - name: push docker image + # run: | + # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Use Run ID from previous fetch_run_id job + run: echo Run ID from previous job ${{ needs.fetch_run_id.outputs.run_id }} + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: docker-image-tag + github-token: ${{ secrets.GH_PAT_TAG }} + repository: MukuFlash03/e-mission-server + run-id: ${{ env.RUN_ID }} + + - name: Print artifact tag + run: cat tag_file.txt + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image-tag + path: tag_file.txt + overwrite: true + + - name: List artifact + run: ls -R From 96222791880cd0f8ebf81e5ccfd91badacfc95aa Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 30 Apr 2024 01:18:19 -0700 Subject: [PATCH 13/56] Bumped up server image tag Bumped up after fixing "url" KeyError bug in this commit in server repo: https://github.com/MukuFlash03/e-mission-server/commit/e778b3f3cf050eec33ef439a1b6763b6aaf533e0 --- viz_scripts/Dockerfile | 2 +- viz_scripts/docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index dc03956a..3fe828e4 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,7 +1,7 @@ # python 3 # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-30--59-58 VOLUME /plots diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index fc20d02a..ac10d32d 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,7 +1,7 @@ # python 3 # FROM shankari/e-mission-server:master_2024-02-10--19-38 # Please change once all PR changes are final, so it reads from shankari/e-mission-server -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-30--59-58 VOLUME /plots From 6e3f2befd242572ed08889ab28674c98b7e32b64 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:04:07 -0700 Subject: [PATCH 14/56] Artifact + Matrix - 1 Added push and workflow_dispatch trigger handling to fetch appropriate docker image tags. Push trigger uses tag from artifact upload while workflow_dispatch uses tag from input parameters. Check commit from admin-dash here: https://github.com/MukuFlash03/op-admin-dashboard/commit/39b289f28d57b3c42b59f916be29ba6a784640c7 --- .github/workflows/image_build_push.yml | 114 ++++++++++++++++--------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 7e0621ff..db2241e2 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -8,10 +8,15 @@ on: push: # Mukul: # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "tags-artifact" + # So, for testing purposes, need to checkout a branch "tags-combo-approach" # TODO: Need to change to build off master or main once it looks good. - branches: [ tags-artifact ] + branches: [ tags-combo-approach ] + workflow_dispatch: + inputs: + docker_image_tag: + description: "Latest Docker image tags passed from e-mission-server repository on image build and push" + required: true # Env variable env: @@ -20,7 +25,6 @@ env: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - fetch_run_id: runs-on: ubuntu-latest @@ -28,7 +32,7 @@ jobs: run_id: ${{ steps.get_run_id.outputs.run_id }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v2 @@ -50,20 +54,70 @@ jobs: id: get_run_id run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" + fetch_tag: + needs: fetch_run_id + runs-on: ubuntu-latest + + env: + RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }} + + outputs: + docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }} + + steps: + - uses: actions/checkout@v4 + + - name: Use Run ID from previous fetch_run_id job + run: echo Run ID from previous job ${{ env.RUN_ID }} + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: docker-image-tag + github-token: ${{ secrets.GH_PAT_TAG }} + repository: MukuFlash03/e-mission-server + run-id: ${{ env.RUN_ID }} + + - name: Print artifact tag + id: get_docker_tag + run: | + cat tag_file.txt + docker_image_tag=$(cat tag_file.txt) + echo $docker_image_tag + echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT + + # - name: Upload Artifact + # uses: actions/upload-artifact@v4 + # with: + # name: docker-image-tag + # path: tag_file.txt + # overwrite: true + + # - name: List artifact + # run: ls -R + # This workflow contains a single job called "build" build: - needs: fetch_run_id + needs: fetch_tag # The type of runner that the job will run on runs-on: ubuntu-latest env: - RUN_ID: ${{needs.fetch_run_id.outputs.run_id}} + DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} + DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Print input docker image tag + run: | + echo "Event name: ${{ github.event_name }}" + echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" + echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + - name: docker login run: | # log into docker hub account docker login -u $DOCKER_USER -p $DOCKER_PASSWORD @@ -77,37 +131,15 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - # - name: build docker image - # run: | - # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend - # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts - # docker images - - # - name: push docker image - # run: | - # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - - - name: Use Run ID from previous fetch_run_id job - run: echo Run ID from previous job ${{ needs.fetch_run_id.outputs.run_id }} - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: docker-image-tag - github-token: ${{ secrets.GH_PAT_TAG }} - repository: MukuFlash03/e-mission-server - run-id: ${{ env.RUN_ID }} - - - name: Print artifact tag - run: cat tag_file.txt - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: docker-image-tag - path: tag_file.txt - overwrite: true - - - name: List artifact - run: ls -R + - name: build docker image + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + else + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + fi + docker images + + - name: push docker image + run: | + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From 1745634710099c54c294941e5703b9bf9830ccf7 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:30:44 -0700 Subject: [PATCH 15/56] Artifact + Matrix - 2 Updating Dockerfiles to use ARG environment variable with latest timestamp that will be passed through: - `docker build --build-arg` command in Github actions in the workflow for automated pushing to Docker hub. - `args: ` config field in docker-compose which will need to be set manually by developers locally. Also, changing branch in fetch_runID and Dockerfiles to tags-combo-approach. --- .github/fetch_runID.py | 2 +- docker-compose.dev.yml | 2 ++ docker-compose.yml | 2 ++ viz_scripts/Dockerfile | 3 ++- viz_scripts/docker/Dockerfile.dev | 3 ++- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index c40005d5..fee53c76 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -51,7 +51,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "tags-artifact" + run["head_branch"] == "tags-combo-approach" ] # print(successful_runs) if successful_runs: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index fdbdbb8c..ec04b026 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -5,6 +5,8 @@ services: build: context: frontend dockerfile: docker/Dockerfile.dev + args: + DOCKER_IMAGE_TAG: '' depends_on: - db ports: diff --git a/docker-compose.yml b/docker-compose.yml index 73a98752..ca3496ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ services: dashboard: image: em-pub-dash-prod/frontend build: frontend + args: + DOCKER_IMAGE_TAG: '' depends_on: - db ports: diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index dc03956a..9bc28c3f 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,7 +1,8 @@ # python 3 # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +ARG DOCKER_IMAGE_TAG +FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} VOLUME /plots diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index fc20d02a..20a6ad7e 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,7 +1,8 @@ # python 3 # FROM shankari/e-mission-server:master_2024-02-10--19-38 # Please change once all PR changes are final, so it reads from shankari/e-mission-server -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +ARG DOCKER_IMAGE_TAG +FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} VOLUME /plots From 21ca992e28b6c4aef1c2b099fbae3e80349743fd Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:49:48 -0700 Subject: [PATCH 16/56] Artifact + Matrix - 3 Public-dash was failing as I had incorrectly replaced the docker build and push commands with the commands used for admin-dash in its YAML file. Public-dash has two images: frontend (dashboard) and viz_scripts (notebook-server) with their separate Dockerfiles. Hence, was getting the error: ERROR: failed to solve: failed to read dockerfile: open /var/lib/docker/tmp/buildkit-mount264844780/Dockerfile: no such file or directory https://github.com/MukuFlash03/em-public-dashboard/actions/runs/8917300905/job/24490148189 --- .github/workflows/image_build_push.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index db2241e2..22e9cad5 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -134,12 +134,15 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts else - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts fi docker images - name: push docker image run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From 58093d30865fd4b67d7cc1796854717900b93621 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 2 May 2024 04:35:36 -0700 Subject: [PATCH 17/56] Artifact + Matrix - 4 For public-dash, admin-dash where ARGS are now being used, need to add the args under build command in the docker compose files. Gives error if arg is at the same hierarchical level as build. Also, public-dash docker-compose.yml (non-dev) version changed to have build: context, dockerfile ; unlike only build: frontend. This allows adding args under build. Similar to how currently being built in docker-compose.dev.yml. Also, args to be added under notebook-server and not dashboard since viz_scripts builds off of server image and not frontend, which is a node image. --- docker-compose.dev.yml | 12 ++++++------ docker-compose.yml | 12 ++++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ec04b026..99f63f04 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,10 +3,8 @@ services: dashboard: image: em-pub-dash-dev/frontend build: - context: frontend - dockerfile: docker/Dockerfile.dev - args: - DOCKER_IMAGE_TAG: '' + context: frontend + dockerfile: docker/Dockerfile.dev depends_on: - db ports: @@ -20,8 +18,10 @@ services: notebook-server: image: em-pub-dash-dev/viz-scripts build: - context: viz_scripts - dockerfile: docker/Dockerfile.dev + context: viz_scripts + dockerfile: docker/Dockerfile.dev + args: + DOCKER_IMAGE_TAG: '' depends_on: - db environment: diff --git a/docker-compose.yml b/docker-compose.yml index ca3496ba..ff5953cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,9 @@ version: "3" services: dashboard: image: em-pub-dash-prod/frontend - build: frontend - args: - DOCKER_IMAGE_TAG: '' + build: + context: frontend + dockerfile: Dockerfile depends_on: - db ports: @@ -16,7 +16,11 @@ services: - emission notebook-server: image: em-pub-dash-prod/viz-scripts - build: viz_scripts + build: + context: viz_scripts + dockerfile: Dockerfile + args: + DOCKER_IMAGE_TAG: '' depends_on: - db environment: From 368900b1a53c3cce5465361dcf8567bba6b7a8ce Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 02:59:03 -0700 Subject: [PATCH 18/56] Artifact + Matrix - 5 Adding .env file which stores only docker image timestamp for the latest dockerhub e-mission-server image already pushed. .env file overwritten in both types of trigger events - push and workflow_dispatch. Added commit and push github actions as well for pushing latest changes to the .env file made via the workflow. Lastly, docker-compose now also mentions the ENV variable name to be read from the .env file for the ARG value in the Dockerfile. No changes required in the Dockerfiles. Had to remove .env from the .gitignore file. --- .env | 1 + .github/workflows/image_build_push.yml | 18 ++++++++++++++++++ .gitignore | 1 - docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..203f8917 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DOCKER_IMAGE_TAG=2024-05-02--16-40 \ No newline at end of file diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 22e9cad5..40f97ff9 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -118,6 +118,24 @@ jobs: echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + - name: Update .env file + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + else + echo "Push event: Restoring latest server image tag in .env" + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + fi + + - name: Add, Commit, Push changes to .env file + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add .env + git commit -m "Updated docker image tag in .env to the latest timestamp: ${{ env.DOCKER_IMAGE_TAG_2 }}" + git push origin + - name: docker login run: | # log into docker hub account docker login -u $DOCKER_USER -p $DOCKER_PASSWORD diff --git a/.gitignore b/.gitignore index 6a0eab58..aa7a8ff4 100644 --- a/.gitignore +++ b/.gitignore @@ -108,7 +108,6 @@ celerybeat.pid *.sage.py # Environments -.env .venv env/ venv/ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 99f63f04..3eb68f99 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,7 +21,7 @@ services: context: viz_scripts dockerfile: docker/Dockerfile.dev args: - DOCKER_IMAGE_TAG: '' + DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} depends_on: - db environment: diff --git a/docker-compose.yml b/docker-compose.yml index ff5953cc..0e18ffa1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: context: viz_scripts dockerfile: Dockerfile args: - DOCKER_IMAGE_TAG: '' + DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} depends_on: - db environment: From bf3e9f74a9d5c32a41931956f115b2fbef3ff140 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 09:59:43 +0000 Subject: [PATCH 19/56] Updated docker image tag in .env to the latest timestamp: --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 203f8917..881e7432 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 \ No newline at end of file +DOCKER_IMAGE_TAG=2024-05-02--16-40 From 6eebf87b7dc60a522db7d1bec653440c1ea25837 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 10:18:20 +0000 Subject: [PATCH 20/56] Updated docker image tag in .env to the latest timestamp: 2024-05-03--14-37 --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 881e7432..03219ee6 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 +DOCKER_IMAGE_TAG=2024-05-03--14-37 From e7605984a1f10cb89d858be4a2522d8da677da20 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 03:36:58 -0700 Subject: [PATCH 21/56] Added TODOs in github actions workflow YAML file Reminder for things to change as per master branch of e-mission-server once changes are finalized. --- .github/fetch_runID.py | 2 ++ .github/workflows/image_build_push.yml | 3 ++- viz_scripts/Dockerfile | 2 +- viz_scripts/docker/Dockerfile.dev | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index fee53c76..56ddf039 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -35,6 +35,7 @@ For e-mission-server: id = 35580278 ''' download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + # TODO: Comment the above line and Uncomment the below line representing the original emission repo # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) @@ -48,6 +49,7 @@ workflow_runs = workflow_runs_json["workflow_runs"] if workflow_runs: + # TODO: Change the head_branch name which identifies only e-mission-server runs triggered by this branch successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 40f97ff9..42d817fc 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -73,6 +73,7 @@ jobs: - name: Download artifact uses: actions/download-artifact@v4 with: + # TODO: Create a token with basic repo permissions name: docker-image-tag github-token: ${{ secrets.GH_PAT_TAG }} repository: MukuFlash03/e-mission-server @@ -133,7 +134,7 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git add .env - git commit -m "Updated docker image tag in .env to the latest timestamp: ${{ env.DOCKER_IMAGE_TAG_2 }}" + git commit -m "Updated docker image tag in .env to the latest timestamp" git push origin - name: docker login diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 9bc28c3f..3fa37394 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,5 +1,5 @@ # python 3 -# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 ARG DOCKER_IMAGE_TAG FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 20a6ad7e..fa86a158 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,6 +1,6 @@ # python 3 +# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -# Please change once all PR changes are final, so it reads from shankari/e-mission-server ARG DOCKER_IMAGE_TAG FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} From 9444e60d85afab80f410b06496eae50ab3470720 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 13:12:35 -0700 Subject: [PATCH 22/56] Artifact + Matrix - 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous Push event triggers run failed Error occurred in GitHub actions git add, commit, push step. If file with no changes operated upon, it leaves an error: “nothing to commit, working tree clean Error: Process completed with exit code 1.” Need to fix. —— Quick fix is to make changes to .env file only if workflow_dispatch event is the trigger. Don’t do anything for push event. So, in case anyone modifies .env file on their own by using their own timestamp during testing, and pushes it as a part of their PR, then Shankari will have to ask them to revert the changes. Else, their custom timestamp will make it to the repo code base. Found something: https://www.reddit.com/r/github/comments/ju3ipr/commit_from_github_action_only_when_changes_exist/ It should work but there’s a drawback of using “exit 0” - it will mask all errors generated during “git commit”. This is bad and we won’t be able to see the reason why something wrong happened as the workflow would be shown as successful with a green tick. Found a solution with git diff: https://github.com/simonw/til/blob/main/github-actions/commit-if-file-changed.md $ git diff --quiet || (git add README.md && git commit -m "Updated README") However, I won’t be able to log any message saying that no changes to commit, tag not modified. Hence, will possibly use just “git diff —quiet” with an if-else block. Expected results: - Push event triggers workflow. - It writes DOCKER_IMAGE_TAG_1 fetched from last successful completed run to .env file. - It sees that there is a difference in the latest committed .env file in the dashboard repo which includes older timestamp. - Hence it runs git commit part of the script to reset to latest server timestamp. --- .env | 2 +- .github/workflows/image_build_push.yml | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 03219ee6..881e7432 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--14-37 +DOCKER_IMAGE_TAG=2024-05-02--16-40 diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 42d817fc..34e1a7be 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -74,6 +74,7 @@ jobs: uses: actions/download-artifact@v4 with: # TODO: Create a token with basic repo permissions + # TODO: Change user / organization to emission instead of MukuFlash03 name: docker-image-tag github-token: ${{ secrets.GH_PAT_TAG }} repository: MukuFlash03/e-mission-server @@ -133,9 +134,13 @@ jobs: run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add .env - git commit -m "Updated docker image tag in .env to the latest timestamp" - git push origin + if git diff --quiet; then + echo "Latest timestamp already present in .env file, no changes to commit" + else + git add .env + git commit -m "Updated docker image tag in .env file to the latest timestamp" + git push origin + fi - name: docker login run: | # log into docker hub account From 40beb806d7f639213deea683923c51fd8cb4c15d Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 20:13:20 +0000 Subject: [PATCH 23/56] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 881e7432..03219ee6 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 +DOCKER_IMAGE_TAG=2024-05-03--14-37 From 29ebf49489384d1fb288d6b3113e196c000f1f48 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 20:45:36 +0000 Subject: [PATCH 24/56] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 03219ee6..8a844199 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--14-37 +DOCKER_IMAGE_TAG=2024-05-03--42-05 From 308eca053d1cdc5320089a669a14e0015cbb4203 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 6 May 2024 22:04:08 +0000 Subject: [PATCH 25/56] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8a844199..ac9b1b4b 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--42-05 +DOCKER_IMAGE_TAG=2024-05-06--00-31 From d919a34fe6d785e565f697192afbdfb507d1a193 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 8 May 2024 18:28:14 +0000 Subject: [PATCH 26/56] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index ac9b1b4b..8a844199 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-06--00-31 +DOCKER_IMAGE_TAG=2024-05-03--42-05 From 11cdafbf656f7f0b2c93b04018a01fdf90b72a07 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 12:36:52 -0600 Subject: [PATCH 27/56] Cleanup image_build_push.yml Preparing the yml for merge with main. --- .github/workflows/image_build_push.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 7a5e5e84..0d95a928 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -1,33 +1,18 @@ -# This is a basic workflow to help you get started with Actions - name: docker-image-push-public-dash -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: - # Mukul: - # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "image-push-merge" - # TODO: Need to change to build off master or main once it looks good. - branches: [ image-push-merge ] - + branches: [ main ] -# Env variable env: DOCKER_USER: ${{secrets.DOCKER_USER}} DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" build: - # The type of runner that the job will run on runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - name: docker login run: | # log into docker hub account @@ -37,11 +22,9 @@ jobs: id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')" - #Runs a single command using the runners shell - name: Run a one-line script run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} - # Runs a set of commands using the runners shell - name: build docker image run: | docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend From 1dda106148f6e13cbd6c38673115995a508404e7 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 12:59:05 -0600 Subject: [PATCH 28/56] Polishing image_build_push.yml Prepped for merge with main --- .github/workflows/image_build_push.yml | 32 ++------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 34e1a7be..6126cfd5 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -1,16 +1,8 @@ -# This is a basic workflow to help you get started with Actions - name: docker-image-push-public-dash -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: - # Mukul: - # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "tags-combo-approach" - # TODO: Need to change to build off master or main once it looks good. - branches: [ tags-combo-approach ] + branches: [ main ] workflow_dispatch: inputs: @@ -18,12 +10,10 @@ on: description: "Latest Docker image tags passed from e-mission-server repository on image build and push" required: true -# Env variable env: DOCKER_USER: ${{secrets.DOCKER_USER}} DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: fetch_run_id: runs-on: ubuntu-latest @@ -74,10 +64,9 @@ jobs: uses: actions/download-artifact@v4 with: # TODO: Create a token with basic repo permissions - # TODO: Change user / organization to emission instead of MukuFlash03 name: docker-image-tag github-token: ${{ secrets.GH_PAT_TAG }} - repository: MukuFlash03/e-mission-server + repository: e-mission/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag @@ -87,31 +76,16 @@ jobs: docker_image_tag=$(cat tag_file.txt) echo $docker_image_tag echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT - - # - name: Upload Artifact - # uses: actions/upload-artifact@v4 - # with: - # name: docker-image-tag - # path: tag_file.txt - # overwrite: true - - # - name: List artifact - # run: ls -R - - # This workflow contains a single job called "build" build: needs: fetch_tag - # The type of runner that the job will run on runs-on: ubuntu-latest env: DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - name: Print input docker image tag @@ -150,11 +124,9 @@ jobs: id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')" - #Runs a single command using the runners shell - name: Run a one-line script run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} - # Runs a set of commands using the runners shell - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then From 66bd0a6d31cea60c0c935560066fc7881778855c Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 13:01:28 -0600 Subject: [PATCH 29/56] Polishing fetch_runID.py Prepping for merge with main. --- .github/fetch_runID.py | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index 56ddf039..ce3d20cb 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -7,23 +7,6 @@ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - # Just checking to see if workflows are being fetched; it works!!! - # download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows" - # logging.debug("About to fetch workflows present in e-mission-server from %s" % download_url) - # r = requests.get(download_url) - # if r.status_code != 200: - # logging.debug(f"Unable to fetch workflows, status code: {r.status_code}") - # sys.exit(1) - # else: - # workflows_json = json.loads(r.text) - # logging.debug(f"Successfully fetched workflows") - # print(workflows_json) - - # for workflow_entry in workflows_json["workflows"]: - # print("Workflow name: %s ; id: %s " % (workflow_entry["name"], workflow_entry["id"])) - - - ''' Workflow "docker image" uses image_build_push.yml From above commented out code, and checked via terminal as well, @@ -34,9 +17,8 @@ For MukuFlash03: id = 75506902 For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" - # TODO: Comment the above line and Uncomment the below line representing the original emission repo - # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + + download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -45,25 +27,16 @@ else: workflow_runs_json = json.loads(r.text) logging.debug(f"Successfully fetched workflow runs") - # print(workflow_runs_json) workflow_runs = workflow_runs_json["workflow_runs"] if workflow_runs: - # TODO: Change the head_branch name which identifies only e-mission-server runs triggered by this branch successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "tags-combo-approach" + run["head_branch"] == "main" ] - # print(successful_runs) if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) - # print(sorted_runs) latest_run_id = sorted_runs[0]["id"] - # print(latest_run_id) print(f"::set-output name=run_id::{latest_run_id}") - # else: - # print("No successful runs") - # else: - # print("No workflow runs found") From 4335f39575d01b26bfa87140c026833651a53e4f Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 16:59:51 -0600 Subject: [PATCH 30/56] Cert copy Per Shankari's permission, copying the cert file every time the viz_scripts dockerfile is run instead of deciding when to copy it based on an environment variable. --- viz_scripts/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index 9732186a..d69eea1e 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -2,6 +2,8 @@ ARG DOCKER_IMAGE_TAG FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +ADD https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem /etc/ssl/certs/ + VOLUME /plots ADD docker/environment36.dashboard.additions.yml / From b330f9675c13d21884e7b86cbb15d22f254af934 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 17:00:45 -0600 Subject: [PATCH 31/56] Delete viz_scripts/docker/cert.sh Removing cert.sh per Shankari's comments --- viz_scripts/docker/cert.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 viz_scripts/docker/cert.sh diff --git a/viz_scripts/docker/cert.sh b/viz_scripts/docker/cert.sh deleted file mode 100644 index 36331687..00000000 --- a/viz_scripts/docker/cert.sh +++ /dev/null @@ -1,6 +0,0 @@ -if [ -z ${PROD_STAGE} ] ; then - echo “Not in staging / production environment, continuing build...” -elif [ ${PROD_STAGE} = "TRUE" ] ; then - wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem -O /etc/ssl/certs/rds-combined-ca-bundle.pem - echo "In staging / production environment, added AWS certificates" -fi From ab0e157f5bf8a05ce3ce99f46914bbb009b9556c Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 21 May 2024 11:59:31 -0600 Subject: [PATCH 32/56] Update docker-compose.yml Removing unnecessary change --- docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e18ffa1..62a575db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,7 @@ version: "3" services: dashboard: image: em-pub-dash-prod/frontend - build: - context: frontend - dockerfile: Dockerfile + build: frontend depends_on: - db ports: From 7c78c184ea630f97298938250fa90e73d2d9927b Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 23 May 2024 15:10:41 -0600 Subject: [PATCH 33/56] Replacing docker build in image_build_push.yml with docker compose Removing the `docker build` command from image_build_push.yml and replacing it with `docker compose`. Since tag names cannot be specified as an input in compose, the names of the tags have to be set in the docker-compose.yml file. Additionally, these tags have to be input as environment variables ahead of the `docker compose` part of the command. I also renamed DOCKER_IMAGE_TAG to SERVER_IMAGE_TAG, since this is a more meaningful name. I will implement this change across the other repositories that use this variable name. --- .env | 2 +- .github/workflows/image_build_push.yml | 6 ++---- docker-compose.yml | 6 +++--- viz_scripts/Dockerfile | 4 ++-- viz_scripts/docker/Dockerfile.dev | 4 ++-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 8a844199..94158a11 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--42-05 +SERVER_IMAGE_TAG=2024-05-03--42-05 diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 4eeb16f7..e23e4215 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -130,11 +130,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build else - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./viz_scripts + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build fi docker images diff --git a/docker-compose.yml b/docker-compose.yml index 62a575db..559ea5b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: dashboard: - image: em-pub-dash-prod/frontend + image: ${FRONTEND_TAG} build: frontend depends_on: - db @@ -13,12 +13,12 @@ services: networks: - emission notebook-server: - image: em-pub-dash-prod/viz-scripts + image: ${VIZ_TAG} build: context: viz_scripts dockerfile: Dockerfile args: - DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} depends_on: - db environment: diff --git a/viz_scripts/Dockerfile b/viz_scripts/Dockerfile index e172dd5a..c872301b 100644 --- a/viz_scripts/Dockerfile +++ b/viz_scripts/Dockerfile @@ -1,7 +1,7 @@ # python 3 -ARG DOCKER_IMAGE_TAG -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +ARG SERVER_IMAGE_TAG +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} ADD https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem /etc/ssl/certs/ diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 689d5b9c..6838856f 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,7 +1,7 @@ # python 3 -ARG DOCKER_IMAGE_TAG -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +ARG SERVER_IMAGE_TAG +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} VOLUME /plots From 27c2f422b4807db10b0d52876faafdb548d013eb Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 23 May 2024 15:23:51 -0600 Subject: [PATCH 34/56] Delete .env --- .env | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 94158a11..00000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -SERVER_IMAGE_TAG=2024-05-03--42-05 From c8b6dca59377e391a3f23334fe869b65e5d3c28c Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 23 May 2024 16:04:46 -0600 Subject: [PATCH 35/56] Remove whitespace changes to docker-compose.dev.yml Removing unnecessary whitespace changes. --- docker-compose.dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3eb68f99..3e24438c 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,8 +3,8 @@ services: dashboard: image: em-pub-dash-dev/frontend build: - context: frontend - dockerfile: docker/Dockerfile.dev + context: frontend + dockerfile: docker/Dockerfile.dev depends_on: - db ports: From 40bf355a7a08bbfac84b16e7eac147f32990e1d9 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 23 May 2024 16:11:27 -0600 Subject: [PATCH 36/56] Removing redundant pip install --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index e23e4215..631ed417 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -31,7 +31,6 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip pip install requests - name: Run Python script From 803e9c2e90cfcac33df7d117c7c36df84fe2c4e1 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 23 May 2024 16:14:29 -0600 Subject: [PATCH 37/56] Updating tag name --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3e24438c..1906101e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,7 +21,7 @@ services: context: viz_scripts dockerfile: docker/Dockerfile.dev args: - DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} depends_on: - db environment: From 7ee8e613578a07ced692b09e2593dfc5112ce7af Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Fri, 24 May 2024 14:44:55 -0600 Subject: [PATCH 38/56] Adding artifact upload Adding a step to upload an artifact with the newly created image tag. This way, the file can be pulled into the internal repo with GH API. --- .github/workflows/image_build_push.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 631ed417..c026b6b6 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -139,3 +139,15 @@ jobs: run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Create a text file + run: | + echo ${{ steps.date.outputs.date }} > public_dash_tag_file.txt + echo "Created tag text file" + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: public-dash-image-tag + path: public_dash_tag_file.txt + overwrite: true From af3c2d33f7e68c2bd6f0b22f40b7bf180e884cd2 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Fri, 24 May 2024 15:13:06 -0600 Subject: [PATCH 39/56] Removing DB_HOST fallback in start_notebook.sh Fallback unnecessary --- viz_scripts/docker/start_notebook.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/viz_scripts/docker/start_notebook.sh b/viz_scripts/docker/start_notebook.sh index 5b646fb1..b6022071 100755 --- a/viz_scripts/docker/start_notebook.sh +++ b/viz_scripts/docker/start_notebook.sh @@ -1,13 +1,7 @@ #!/usr/bin/env bash #Configure web server -#set database URL using environment variable echo "DB host = "${DB_HOST} -if [ -z ${DB_HOST} ] ; then - local_host=`hostname -i` - export DB_HOST=$local_host - echo "Setting db host environment variable to localhost" -fi ### configure the saved-notebooks directory for persistent notebooks From 91d525187646b11ea7ccfc912fe1ad4efa57b067 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Fri, 24 May 2024 15:54:40 -0600 Subject: [PATCH 40/56] DOCKER_IMAGE_TAG --> SERVER_IMAGE_TAG rename --- .github/workflows/image_build_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index c026b6b6..9744e805 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -97,10 +97,10 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" - echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env else echo "Push event: Restoring latest server image tag in .env" - echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env fi - name: Add, Commit, Push changes to .env file From d3bf89f8d65cd790acae6faf3892c63ba3c7cda7 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 00:46:03 -0600 Subject: [PATCH 41/56] Testing workflow I want to see if passing in the tags to the docker-compose in the way I'm trying to will actually work. We'll see! --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 9744e805..2b6a7539 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -2,7 +2,7 @@ name: docker-image-push-public-dash on: push: - branches: [ main ] + branches: [ main, image-push ] workflow_dispatch: inputs: From c58b5dd0b3b0b9cfb2f4667797fe282ebb35638a Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 00:48:05 -0600 Subject: [PATCH 42/56] Testing workflow (again) Trying to test the workflow again. --- .github/workflows/image_build_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 2b6a7539..11ac76b9 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -65,7 +65,8 @@ jobs: # TODO: Create a token with basic repo permissions name: docker-image-tag github-token: ${{ secrets.GH_PAT_TAG }} - repository: e-mission/e-mission-server + # repository: e-mission/e-mission-server + repository: MukuFlash03/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag From 14775b5dddc056d5e2d0d61d696010ad40079f0d Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:32:38 -0600 Subject: [PATCH 43/56] Push test + adding rename step --- .github/fetch_runID.py | 4 ++-- .github/workflows/image_build_push.yml | 11 ++++++++--- docker-compose.yml | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index ce3d20cb..68299901 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -18,7 +18,7 @@ For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -33,7 +33,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "main" + run["head_branch"] == "consolidate-differences" ] if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 11ac76b9..04b91659 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -130,13 +130,18 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-dev.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 FRONTEND_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} VIZ_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-dev.yml build fi docker images - - name: push docker image + - name: rename docker images + run: | + docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash-dev/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: push docker images run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} diff --git a/docker-compose.yml b/docker-compose.yml index 559ea5b0..38a975ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: dashboard: - image: ${FRONTEND_TAG} + image: em-pub-dash-prod/frontend build: frontend depends_on: - db @@ -13,7 +13,7 @@ services: networks: - emission notebook-server: - image: ${VIZ_TAG} + image: em-pub-dash-prod/viz-scripts build: context: viz_scripts dockerfile: Dockerfile From f595b0d75699f684d4c92b264e12e19d74c816a9 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:34:51 -0600 Subject: [PATCH 44/56] dash to dot --- .github/workflows/image_build_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 04b91659..dbaaf685 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -130,9 +130,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose.dev.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose.dev.yml build fi docker images From 2f17e5de6ed1023cfce970ef06ec75b2badec26f Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:37:51 -0600 Subject: [PATCH 45/56] change FROM context --- viz_scripts/docker/Dockerfile.dev | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index 6838856f..dbdf8c18 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,8 +1,8 @@ # python 3 ARG SERVER_IMAGE_TAG -FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} - +# FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} +FROM mukuflash03/e-mission-server:consolidate-differences_${SERVER_IMAGE_TAG} VOLUME /plots ADD docker/environment36.dashboard.additions.yml / From 25dcba158637b261dbccc1c4cb1626444b656b8d Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:54:05 -0600 Subject: [PATCH 46/56] Reverting changes made to test Reverting the changes that I made to test the workflow with docker compose. --- .github/fetch_runID.py | 4 ++-- .github/workflows/image_build_push.yml | 5 ++--- viz_scripts/docker/Dockerfile.dev | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index 68299901..6cde9a25 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -18,7 +18,7 @@ For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -33,7 +33,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "consolidate-differences" + run["head_branch"] == "master" ] if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index dbaaf685..434b049d 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -2,7 +2,7 @@ name: docker-image-push-public-dash on: push: - branches: [ main, image-push ] + branches: [ main ] workflow_dispatch: inputs: @@ -65,8 +65,7 @@ jobs: # TODO: Create a token with basic repo permissions name: docker-image-tag github-token: ${{ secrets.GH_PAT_TAG }} - # repository: e-mission/e-mission-server - repository: MukuFlash03/e-mission-server + repository: e-mission/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag diff --git a/viz_scripts/docker/Dockerfile.dev b/viz_scripts/docker/Dockerfile.dev index dbdf8c18..6838856f 100644 --- a/viz_scripts/docker/Dockerfile.dev +++ b/viz_scripts/docker/Dockerfile.dev @@ -1,8 +1,8 @@ # python 3 ARG SERVER_IMAGE_TAG -# FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} -FROM mukuflash03/e-mission-server:consolidate-differences_${SERVER_IMAGE_TAG} +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} + VOLUME /plots ADD docker/environment36.dashboard.additions.yml / From 82cf4e9463f81c0dfff1e72328a1661875b5ea38 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 28 May 2024 02:54:01 -0600 Subject: [PATCH 47/56] Switching to build prod instead of dev --- .github/workflows/image_build_push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 434b049d..1378cef7 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -129,16 +129,16 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose.dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose.prod.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose.dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose.prod.yml build fi docker images - name: rename docker images run: | - docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - docker image tag em-pub-dash-dev/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash-prod/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash-prod/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker images run: | From 5d7f6d81713ee363dd583a2f5b3d8012e4591d93 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 16:49:36 -0700 Subject: [PATCH 48/56] Modified tag variable names to be more relevant These store tags differently depending on the trigger event - Push OR Workflow dispatch --- .github/workflows/image_build_push.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1378cef7..4315582f 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -81,8 +81,8 @@ jobs: runs-on: ubuntu-latest env: - DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} - DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} + DOCKER_TAG_FROM_PUSH: ${{ needs.fetch_tag.outputs.docker_image_tag }} + DOCKER_TAG_FROM_WORKFLOW_DISPATCH: ${{ github.event.inputs.docker_image_tag }} steps: - uses: actions/checkout@v4 @@ -90,17 +90,17 @@ jobs: - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" - echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" - echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + echo "Latest docker image tag (push): ${{ env.DOCKER_TAG_FROM_PUSH }}" + echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_TAG_FROM_WORKFLOW_DISPATCH }}" - name: Update .env file run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env else echo "Push event: Restoring latest server image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH" > .env fi - name: Add, Commit, Push changes to .env file @@ -129,9 +129,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose.prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose.prod.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose.prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose.prod.yml build fi docker images From cedf7b1cb4e1d3fafde86be49fe9f58fd8c004f8 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 16:54:25 -0700 Subject: [PATCH 49/56] Updated username to clarify that env file is being updated --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 4315582f..d54eda3f 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -106,7 +106,7 @@ jobs: - name: Add, Commit, Push changes to .env file run: | git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.name "Github Actions bot to update .env with latest tags" if git diff --quiet; then echo "Latest timestamp already present in .env file, no changes to commit" else From 6849f84cc0aa3b52c80579d843013121d72e88f5 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 18:02:19 -0700 Subject: [PATCH 50/56] Re-added .env file This had been deleted in this commit: https://github.com/MukuFlash03/em-public-dashboard/commit/27c2f422b4807db10b0d52876faafdb548d013eb Added it with latest server image tag. Hence forth, once PRs are merged, it will read updated tags. --- .env | 1 + 1 file changed, 1 insertion(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..93050391 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +SERVER_IMAGE_TAG=2024-08-12--15-15 \ No newline at end of file From f325eb35b406824ac4db1155546706fa45d333ec Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 19:59:14 -0700 Subject: [PATCH 51/56] Changing images to use docker-compose DEV file Earlier commit had changed it to use PROD file but this file doesn't exist. The prod version of the file we have is docker-compose.yml instead. Using dev version only. --- .github/workflows/image_build_push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index d54eda3f..b8c1cf0e 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -129,16 +129,16 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose.prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose.dev.yml build else - SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose.prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose.dev.yml build fi docker images - name: rename docker images run: | - docker image tag em-pub-dash-prod/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - docker image tag em-pub-dash-prod/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash-dev/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker images run: | From eb4672c9e97e71025e9b0c229a8dac1202546c45 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:03:01 -0700 Subject: [PATCH 52/56] Tag and Push frontend dashboard image only on Push event Workflow dispatch event is triggered on changes to server image. This should build the notebook server image as it uses the server image. But frontend is Javascript nodejs based image and is unrelated to the server. Check this review comment: https://github.com/e-mission/em-public-dashboard/pull/125#discussion_r1714430059 --- .github/workflows/image_build_push.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index b8c1cf0e..3bfdc06b 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -137,12 +137,16 @@ jobs: - name: rename docker images run: | - docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + if [ "${{ github.event_name }}" == "push" ]; then + docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + fi docker image tag em-pub-dash-dev/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker images run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + if [ "${{ github.event_name }}" == "push" ]; then + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + fi docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: Create a text file From e87058d0f6207275669d2a7d295b7e5ff89e3867 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:22:42 -0700 Subject: [PATCH 53/56] Hardcoded WEB_SERVER_HOST to 0.0.0.0 Check comment https://github.com/e-mission/em-public-dashboard/pull/125#discussion_r1716119410 --- viz_scripts/docker/start_notebook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viz_scripts/docker/start_notebook.sh b/viz_scripts/docker/start_notebook.sh index b6022071..c6d1ad6d 100755 --- a/viz_scripts/docker/start_notebook.sh +++ b/viz_scripts/docker/start_notebook.sh @@ -30,7 +30,7 @@ cd saved-notebooks # tail -f /dev/null if [ -z ${CRON_MODE} ] ; then echo "Running notebook in docker, change host:port to localhost:47962 in the URL below" - PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=${WEB_SERVER_HOST} --allow-root + PYTHONPATH=/usr/src/app jupyter notebook --no-browser --ip=0.0.0.0 --allow-root else echo "Running crontab without user interaction, setting python path" export PYTHONPATH=/usr/src/app From 7483f2adff2e162d4ca454e0a275af2b7cbfbe31 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:30:23 -0700 Subject: [PATCH 54/56] Changing to use non-dev version Based on this commit https://github.com/e-mission/em-public-dashboard/pull/125/commits/82cf4e9463f81c0dfff1e72328a1661875b5ea38 --- .github/workflows/image_build_push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 3bfdc06b..1f39aacb 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -129,18 +129,18 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose.dev.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose.yml build else - SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose.dev.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose.yml build fi docker images - name: rename docker images run: | if [ "${{ github.event_name }}" == "push" ]; then - docker image tag em-pub-dash-dev/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} fi - docker image tag em-pub-dash-dev/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker images run: | From 2434648abebf309ea98bd738d43023d7ffb77fdf Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 22:41:44 -0700 Subject: [PATCH 55/56] Removing artifact method This was mainly needed for Push event but since Workflow dispatch event would be setting the latest server image tag in .env file, the push event can read from this file directly. --- .github/fetch_runID.py | 42 --------------- .github/workflows/image_build_push.yml | 71 +++----------------------- 2 files changed, 6 insertions(+), 107 deletions(-) delete mode 100644 .github/fetch_runID.py diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py deleted file mode 100644 index 6cde9a25..00000000 --- a/.github/fetch_runID.py +++ /dev/null @@ -1,42 +0,0 @@ - -import json -import logging -import requests -import sys - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - - ''' - Workflow "docker image" uses image_build_push.yml - From above commented out code, and checked via terminal as well, - workflow id for the "docker image" can be fetched using: - https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows - https://api.github.com/repos/e-mission/e-mission-server/actions/workflows - - For MukuFlash03: id = 75506902 - For e-mission-server: id = 35580278 - ''' - - download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" - logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) - r = requests.get(download_url) - if r.status_code != 200: - logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}") - sys.exit(1) - else: - workflow_runs_json = json.loads(r.text) - logging.debug(f"Successfully fetched workflow runs") - - workflow_runs = workflow_runs_json["workflow_runs"] - if workflow_runs: - successful_runs = [run for run in workflow_runs \ - if run["status"] == "completed" and \ - run["conclusion"] == "success" and \ - run["head_branch"] == "master" - ] - if successful_runs: - sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) - sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) - latest_run_id = sorted_runs[0]["id"] - print(f"::set-output name=run_id::{latest_run_id}") diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1f39aacb..f667bdb4 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -15,78 +15,20 @@ env: DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} jobs: - fetch_run_id: - runs-on: ubuntu-latest - - outputs: - run_id: ${{ steps.get_run_id.outputs.run_id }} - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install Python dependencies - run: | - pip install requests - - - name: Run Python script - id: run_script - run: | - echo "Fetching latest successful run ID from e-misison-server docker image workflow" - python .github/fetch_runID.py - - - name: Get Run ID - id: get_run_id - run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" - - fetch_tag: - needs: fetch_run_id - runs-on: ubuntu-latest - - env: - RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }} - - outputs: - docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }} - - steps: - - uses: actions/checkout@v4 - - - name: Use Run ID from previous fetch_run_id job - run: echo Run ID from previous job ${{ env.RUN_ID }} - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - # TODO: Create a token with basic repo permissions - name: docker-image-tag - github-token: ${{ secrets.GH_PAT_TAG }} - repository: e-mission/e-mission-server - run-id: ${{ env.RUN_ID }} - - - name: Print artifact tag - id: get_docker_tag - run: | - cat tag_file.txt - docker_image_tag=$(cat tag_file.txt) - echo $docker_image_tag - echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT build: - needs: fetch_tag - runs-on: ubuntu-latest env: - DOCKER_TAG_FROM_PUSH: ${{ needs.fetch_tag.outputs.docker_image_tag }} DOCKER_TAG_FROM_WORKFLOW_DISPATCH: ${{ github.event.inputs.docker_image_tag }} steps: - uses: actions/checkout@v4 + - name: Set docker image tag from .env file + run: | + set -a; source .env; set +a + echo "DOCKER_TAG_FROM_PUSH=${SERVER_IMAGE_TAG}" >> $GITHUB_ENV + - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" @@ -99,8 +41,7 @@ jobs: echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env else - echo "Push event: Restoring latest server image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH" > .env + echo "Push event: Restoring latest server image tag from .env" fi - name: Add, Commit, Push changes to .env file From b98cf531d678b16c0e557f6a05abe72e02bc3d38 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 14 Aug 2024 12:05:14 -0700 Subject: [PATCH 56/56] Added suffix to frontend image --- .github/workflows/image_build_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index f667bdb4..47e0203a 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -79,14 +79,14 @@ jobs: - name: rename docker images run: | if [ "${{ github.event_name }}" == "push" ]; then - docker image tag em-pub-dash/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker image tag em-pub-dash/frontend:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_frontend:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} fi docker image tag em-pub-dash/viz-scripts:latest $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker images run: | if [ "${{ github.event_name }}" == "push" ]; then - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_frontend:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} fi docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}_notebook:${GITHUB_REF##*/}_${{ steps.date.outputs.date }}