Build and push and test Docker containers (#10)

* organize Dockerfiles

* update action according to the dockerfile re-org

* try to fix docker platform

* add test-docker-compose.yml

* load a default raster table

* update readme

* test action 1

* test action 2

* test action 3

* fix db

* fix something

* test action

* remove used stuff

.github/workflows/build-and-push-docker.yml

@@ -1,4 +1,4 @@
-name: build and push docker
+name: Build and push docker
 
 on:
   push:
@@ -25,24 +25,50 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
-        name: Build and push amd64
+        name: Build and push GWS amd64
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/Dockerfile
+          file: ./docker/gws-amd64/Dockerfile
           push: true
+          platforms: linux/amd64
+          provenance: false
           tags: gplates/gws:amd64-${{github.ref_name}}
       -
-        name: Build and push arm64
+        name: Build and push GWS arm64
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/Dockerfile-arm64
+          file: ./docker/gws-arm64/Dockerfile
           push: true
+          platforms: linux/arm64
+          provenance: false
           tags: gplates/gws:arm64-${{github.ref_name}}
-
       -
-        name: Create manifest
+        name: Build and push PostGIS amd64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/gws-postgis-amd64/Dockerfile
+          push: true
+          platforms: linux/amd64
+          provenance: false
+          tags: gplates/postgis:amd64-${{github.ref_name}}
+      -
+        name: Build and push PostGIS arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/gws-postgis-arm64/Dockerfile
+          push: true
+          platforms: linux/arm64
+          provenance: false
+          tags: gplates/postgis:arm64-${{github.ref_name}}
+      -
+        name: Create cross-platform images
         run: |
+            docker buildx imagetools create -t gplates/postgis:${{github.ref_name}} gplates/postgis:amd64-${{github.ref_name}} gplates/postgis:arm64-${{github.ref_name}}
+            docker buildx imagetools create -t gplates/postgis:latest gplates/postgis:amd64-${{github.ref_name}} gplates/postgis:arm64-${{github.ref_name}}
+            docker buildx imagetools create -t gplates/gws:latest gplates/gws:amd64-${{github.ref_name}} gplates/gws:arm64-${{github.ref_name}}
             docker buildx imagetools create -t gplates/gws:${{github.ref_name}} gplates/gws:amd64-${{github.ref_name}} gplates/gws:arm64-${{github.ref_name}}
             

.github/workflows/push-docker-latest-and-test-docker-compose.yml

@@ -0,0 +1,50 @@
+name: Push Docker latest and test Docker Compose
+
+on:
+  release:
+    types: [published]
+#on:
+#  push:
+#    branches: 
+#      - dev
+
+jobs:
+  push-docker-latest:
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        name: Push docker latest
+        run: |
+            docker buildx imagetools create -t gplates/postgis:latest gplates/postgis:amd64-${{ github.event.release.tag_name }} gplates/postgis:arm64-${{ github.event.release.tag_name }}
+            docker buildx imagetools create -t gplates/gws:latest gplates/gws:amd64-${{ github.event.release.tag_name }} gplates/gws:arm64-${{ github.event.release.tag_name }}
+    
+  test-docker-compose:
+    runs-on: ubuntu-latest
+
+    steps:
+      - 
+        name: Check out repository code
+        uses: actions/checkout@v4
+      -
+        name: Start Docker compose
+        run: |   
+          cd docker
+          docker-compose up -d
+      - 
+        name: Sleep for 30 seconds
+        run: sleep 30s
+        shell: bash
+      -  
+        name: Check Docker compose
+        if: always()
+        run: |
+           docker ps
+           docker network ls
+           docker inspect docker_gws_1 -f "{{json .NetworkSettings.Networks }}"
+           wget http://localhost:18000
+           curl --fail -s "http://localhost:18000/raster/query?lon=128.86&lat=-12.42&raster_name=crustal_thickness" || exit 1
+           
+
+          
+

.github/workflows/test.yml

@@ -72,9 +72,36 @@ jobs:
         name: Run testcases
         run: |     
           docker run --name test -v`pwd`:/workspace --network ${{ job.container.network }} --env GWS_SERVER_URL=http://gws:80 gplates/gws /workspace/test/test-server.sh    
+          docker exec test cat /workspace/test/testcases/logs/*
       -
         name: Check log
         if: always()
         run: |
           docker exec gws cat /var/log/apache2/error.log
+  
+  test-docker-compose:
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        name: Start Docker compose
+        run: |   
+          cd docker
+          docker-compose up -d
+      - 
+        name: Sleep for 30 seconds
+        run: sleep 30s
+        shell: bash
+      -  
+        name: Check Docker compose
+        if: always()
+        run: |
+           docker ps
+           docker network ls
+           docker inspect gws -f "{{json .NetworkSettings.Networks }}"
+           wget http://localhost:18000
+           curl --fail -s http://localhost:18000/raster/query?lon=128.86&lat=-12.42&raster_name=crustal_thickness
+           
+
+          
 

README.md

@@ -1,5 +1,7 @@
 # GPlates Web Service
 
+![Test](https://github.com/gplates/gplates-web-service/actions/workflows/test.yml/badge.svg)
+
 The **🔥GPlates Web Service🔥** is a part of the on-going GPlates project funded by [AuScope](https://www.auscope.org.au/).
 The web service is built upon [pygplates](https://www.gplates.org/docs/pygplates/index.html).
 It allows users to utilize the pygplates functionalities without installing pygplates locally.

django/GWS/env.template

@@ -4,7 +4,7 @@ DB_NAME=gplates
 DB_USER=gplates
 DB_HOST=gws-postgis
 DB_PORT=5432
-DB_PASSWORD=zaq12wsx
+DB_PASSWORD=gplates
 THROTTLE=false
 DEFAULT_THROTTLE_ANON_RATE=1000/second
 DEFAULT_THROTTLE_USER_RATE=10000/second

docker/README.md

@@ -1,29 +1,35 @@
 ## 👉 Quick start 
 
-If you would like to try the GPlates Web Service very quickly, follow the steps in this "quick start" section. The steps will start the docker container using the built-in source code. And the PostGIS database is not available. See "PostGIS Database and docker network" section for database configuration.
+If you would like to try the GPlates Web Service very quickly, follow the steps in this "quick start" section. The steps will start the docker container using the built-in source code. 
 
-- run the service as daemon in the background
+- run the with docker-compose
 
-    `docker run -d -p 80:80 --restart always gplates/gws`
-
-- or run the service interactively (you can check the error messages in the console)
-
-    `docker run --rm -it -p 80:80 gplates/gws`
+    - `git clone https://github.com/GPlates/gplates-web-service gplates-web-service.git`
+    - `cd gplates-web-service.git`
+    - `cd docker`
+    - `docker-compose up -d`
 
 - verify the service is up and running.
 
-    - `wget -O test.json "http://localhost/reconstruct/reconstruct_points/?points=95,54,142,-33&time=140&model=SETON2012" `
-    - or `curl "http://localhost/reconstruct/reconstruct_points/?points=95,54,142,-33&time=140&model=SETON2012" `
+    - `wget -O test.json "http://localhost:18000/reconstruct/reconstruct_points/?points=95,54,142,-33&time=140&model=SETON2012" `
+    - or `curl "http://localhost:18000/reconstruct/reconstruct_points/?points=95,54,142,-33&time=140&model=SETON2012" `
     - or use web browser if you have GUI
 
 ## 👉 Use the latest code from github.com to run the service 
 
 - `git clone https://github.com/GPlates/gplates-web-service gplates-web-service.git`
-- `cd gplates-web-service.git`
-- ``docker run --rm -it -v `pwd`:/gws  gplates/gws /bin/bash -c "pmm download all /gws/django/GWS/data/model-repo/"``
-- ``docker run -it --rm -v `pwd`:/gws -p 80:80 gplates/gws``
+- `cd gplates-web-service.git/docker`
+- `docker-compose run --rm --service-ports gws-postgis`
+- `docker-compose run --rm --service-ports redis`
+- ``docker run -it --rm -v `pwd`:/gws -p 18000:80 gplates/gws``
 - verify the service is up and running with wget, curl or web browser
 
+Alternatively, 
+
+you can write a customized docker-compose.yml, like Michael Chin did. See [docker-compose-mc.yml](docker-compose-mc.yml)
+
+## 👀 Warning: The notes from this line below are meant for Michael Chin. Other people might fail to understand them. Ask him! 👀 
+
 ## 👉 Setup PostGIS database
 
 - pull the postgis docker image from docker hub
@@ -82,7 +88,7 @@ If you would like to try the GPlates Web Service very quickly, follow the steps
 - `curl "http://localhost:18000/raster/query?lon=99.50&lat=-40.24&raster_name=age_grid_geek_2007"`
 - `curl "http://localhost:18000/reconstruct/reconstruct_points/?points=95,54,142,-33&time=140&model=SETON2012"`
 
-**IMPORTANT: ⚠ Make sure the BEDUG is set to True in .env**
+**👀 IMPORTANT: Make sure the BEDUG is set to True in .env 👀**
 
 
 ## 👉 Build the docker images

docker/create-gws-db.sql

@@ -5,3 +5,7 @@ CREATE EXTENSION postgis;
 CREATE EXTENSION postgis_topology;
 CREATE EXTENSION postgis_raster;
 
+create schema raster;
+ALTER SCHEMA raster OWNER TO gplates;
+
+

docker/docker-compose-mc.yml

@@ -0,0 +1,65 @@
+# This is for Michael Chin only... 
+version: '3.9'
+
+volumes:
+  gws-code:
+    external: true
+    name: gws-code
+  gws-db-data:
+    external: true
+    name: gws-db-data-new
+
+services:
+  gws-postgis:
+    image: gplates/postgis
+    networks:
+      - gws-net
+    hostname: gws-postgis
+    volumes:
+      - gws-db-data:/var/lib/postgresql/15/main
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD="this is not a real password! go away!"
+    restart: on-failure
+    healthcheck:
+      test: pg_isready -U gplates -d gplates
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+
+  gws:
+    image: gplates/gws
+    networks:
+      - gws-net
+    hostname: gws
+    depends_on:
+      - gws-postgis
+      - gws-redis
+    volumes:
+      - gws-code:/gws
+    ports:
+      - 18000:80
+    restart: on-failure
+    healthcheck:
+      test: curl --fail -s "http://localhost:80/raster/query?lon=128.86&lat=-12.42&raster_name=crustal_thickness" || exit 1
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+
+  gws-redis:
+    image: redis
+    networks:
+      - gws-net
+    hostname: gws-redis
+    restart: on-failure
+    healthcheck:
+      test: redis-cli -h gws-redis ping  
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+
+
+networks:
+  gws-net:
+    external: true

docker/docker-compose.yml

@@ -1,23 +1,9 @@
 version: '3.9'
 
-volumes:
-  gws-code:
-    external: true
-    name: gws-code
-  gws-db-data:
-    external: true
-    name: gws-db-data-new
-
 services:
   gws-postgis:
     image: gplates/postgis
-    networks:
-      - gws-net
     hostname: gws-postgis
-    volumes:
-      - gws-db-data:/var/lib/postgresql/15/main
-    ports:
-      - 5432:5432
     environment:
       - POSTGRES_PASSWORD="this is not a real password! go away!"
     restart: on-failure
@@ -29,14 +15,10 @@ services:
 
   gws:
     image: gplates/gws
-    networks:
-      - gws-net
     hostname: gws
     depends_on:
       - gws-postgis
       - gws-redis
-    volumes:
-      - gws-code:/gws
     ports:
       - 18000:80
     restart: on-failure
@@ -48,8 +30,6 @@ services:
 
   gws-redis:
     image: redis
-    networks:
-      - gws-net
     hostname: gws-redis
     restart: on-failure
     healthcheck:
@@ -58,7 +38,3 @@ services:
       timeout: 10s
       retries: 3
 
-
-networks:
-  gws-net:
-    external: true

docker/Dockerfile → docker/gws-amd64/Dockerfile

@@ -2,13 +2,16 @@
 # to build this docker container image
 # step 1: git clone https://github.com/GPlates/gplates-web-service.git
 # step 2: go into folder gplates-web-service
-# step 3: docker build -f docker/Dockerfile -t gplates/gws .
+# step 3: docker build -f docker/gws-amd64/Dockerfile -t gplates/gws:amd64 .
 #
 
 # if error, try with --no-cache option
 
 FROM ubuntu:22.04
 
+LABEL org.opencontainers.image.authors="michael.chin@sydney.edu.au"
+LABEL version="0.1.0"
+
 RUN apt-get update -y
 ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=Australia 
@@ -50,7 +53,7 @@ RUN pip3 install plate-model-manager --upgrade
 RUN pip3 install redis
 
 ADD . /gws
-
+RUN mv /gws/django/GWS/env.template /gws/django/GWS/.env
 RUN pmm download all /gws/django/GWS/data/model-repo/
 
 # Add Tini
@@ -69,7 +72,7 @@ RUN mkdir /tmp/gws
 RUN chown www-data /tmp/gws
 RUN chown www-data /var/www
 
-HEALTHCHECK CMD curl --fail -s http://localhost:80/raster/query?lon=128.86&lat=-12.42&raster_name=crustal_thickness || exit 1
+HEALTHCHECK CMD curl --fail -s "http://localhost:80/raster/query?lon=128.86&lat=-12.42&raster_name=crustal_thickness" || exit 1
 
 EXPOSE 80