Merge pull request #28 from Terradue/feature/deploy-undeploy-ogcapi-r…

…oute

OGC API - Processes - Part 2: Deploy, Replace, Undeploy

.github/workflows/docker-dru-image.yml

@@ -0,0 +1,70 @@
+name: Docker Image CI (DRU support)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: docker/setup-buildx-action@v2
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: ${{ runner.os }}-buildx-
+    - name: docker login
+      env:
+        DOCKER_USER: ${{secrets.DOCKER_USER}}
+        DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+      run: |
+        docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+    - name: Build and push
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        push: true
+        file: Dockerfile-DRU
+        tags: ${{secrets.DOCKER_USER}}/zookernel:dru-${{ github.sha }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+    - name: Build and push (official image)
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        push: true
+        file: Dockerfile-DRU
+        tags: zooproject/zoo-project:dru-${{ github.sha }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+    - name: Move cache
+      run: |
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+    - name: Build and push latest
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        push: true
+        file: Dockerfile-DRU
+        tags: ${{secrets.DOCKER_USER}}/zookernel:dru-latest
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+    - name: Build and push latest
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        push: true
+        file: Dockerfile-DRU
+        tags: zooproject/zoo-project:dru-latest
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+

.gitignore

@@ -1 +1,37 @@
 .idea/
+
+### C++ ###
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# End of c++

Dockerfile

@@ -357,6 +357,8 @@ COPY --from=builder1 /zoo-project/zoo-project/zoo-services/cgal/examples/ /var/w
 COPY --from=builder1 /zoo-project/zoo-project/zoo-services/utils/open-api/templates/index.html /var/www/index.html
 COPY --from=builder1 /zoo-project/zoo-project/zoo-services/utils/open-api/static /var/www/html/static
 COPY --from=builder1 /zoo-project/zoo-project/zoo-services/echo-py/cgi-env/ /usr/lib/cgi-bin/
+COPY --from=builder1 /zoo-project/zoo-project/zoo-services/deploy-py/cgi-env/ /usr/lib/cgi-bin/
+COPY --from=builder1 /zoo-project/zoo-project/zoo-services/undeploy-py/cgi-env/ /usr/lib/cgi-bin/
 COPY --from=builder1 /zoo-project/docker/.htaccess /var/www/html/.htaccess
 COPY --from=builder1 /zoo-project/docker/default.conf /000-default.conf
 COPY --from=builder1 /zoo-project/zoo-project/zoo-services/utils/open-api/server/publish.py /usr/lib/cgi-bin/publish.py
@@ -431,6 +433,7 @@ RUN set -ex \
 # service namespaces parent folder
 RUN mkdir -p /opt/zooservices_namespaces && chmod -R 700 /opt/zooservices_namespaces && chown -R www-data /opt/zooservices_namespaces
 
+
 # For using another port than 80, change the value below.
 # remember to also change the ports in docker-compose.yml
 EXPOSE 80

Dockerfile-DRU

@@ -0,0 +1,162 @@
+FROM ubuntu:focal-20220113
+
+# to build:
+# docker build --rm -t ades:latest .
+# to run:
+# docker run --rm -ti -p 80:80  ades:latest
+
+# procadesdev:latest
+ENV DEBIAN_FRONTEND noninteractive
+
+########################################
+### DEV TOOLS
+RUN apt-get update -qqy --no-install-recommends \
+# Various cli tools
+ && apt-get install -qqy --no-install-recommends wget mlocate tree \
+# C++ and CMAKE
+ gcc mono-mcs cmake \
+ build-essential libcgicc-dev gdb \
+#Install Docker CE CLI
+ curl apt-transport-https ca-certificates gnupg2 lsb-release \
+ && curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | apt-key add - 2>/dev/null \
+ && echo "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list \
+ && apt-get update -qqy --no-install-recommends \
+ && apt-get install -qqy --no-install-recommends docker-ce-cli && \
+ apt-get clean -qqy
+
+ARG PY_VER=3.8
+# Miniconda
+RUN wget -nv \
+    https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \
+    && bash Miniconda3-py39_4.10.3-Linux-x86_64.sh -b -p /usr/miniconda3 \
+    && rm -f Miniconda3-py39_4.10.3-Linux-x86_64.sh
+ENV PATH="/usr/miniconda3/envs/ades-dev/bin:/usr/miniconda3/bin:${PATH}"
+COPY docker/dru/ades-dev_env.yaml /tmp/ades-dev_env.yaml
+RUN conda install mamba -n base -c conda-forge && \
+    mamba env create --file /tmp/ades-dev_env.yaml &&\
+    rm /tmp/ades-dev_env.yaml
+
+########################################
+# ZOO_Prerequisites
+
+RUN apt-get update -qqy  --no-install-recommends  && \
+apt-get install -qqy --no-install-recommends  software-properties-common && \
+add-apt-repository ppa:ubuntugis/ubuntugis-unstable && \
+add-apt-repository ppa:ubuntugis/ppa && \
+apt-get update -qqy  --no-install-recommends && \
+apt-get install -qqy  --no-install-recommends software-properties-common\
+	git\
+	wget\
+	vim\
+	flex\
+	bison\
+	libfcgi-dev\
+	libxml2\
+	libxml2-dev\
+	curl\
+	libssl-dev\
+	autoconf\
+	apache2\
+	subversion\
+	libmozjs185-dev\
+	python3-dev\
+	python3-setuptools\
+	build-essential\
+	libxslt1-dev\
+	uuid-dev\
+	libjson-c-dev\
+	libmapserver-dev\
+	libgdal-dev\
+	libaprutil1-dev \
+	librabbitmq-dev\
+	libapache2-mod-fcgid\
+	wget \
+	pkg-config\
+# if you remove --with-db-backend from the configure command, uncomment the following line
+#RUN ln -s /usr/lib/x86_64-linux-gnu/libfcgi.a /usr/lib/
+&& a2enmod actions fcgid alias proxy_fcgi \
+&& /etc/init.d/apache2 restart \
+&& rm -rf /var/lib/apt/lists/*
+
+########################################
+# ZOO_KERNEL
+#ARG ZOO_PRJ_GIT_BRANCH='feature/deploy-undeploy-ogcapi-route'
+#RUN cd /opt && git clone --depth 1 https://github.com/terradue/ZOO-Project.git -b $ZOO_PRJ_GIT_BRANCH
+COPY . /opt/ZOO-Project
+WORKDIR /opt/ZOO-Project
+RUN make -C ./thirds/cgic206 libcgic.a
+RUN cd ./zoo-project/zoo-kernel \
+     && autoconf \
+     #&& grep MS_VERSION_ -rni /usr/ \
+     && ./configure --with-dru=yes \
+     	--with-python=/usr/miniconda3/envs/ades-dev \
+	--with-pyvers=$PY_VER \
+	--with-js=/usr \
+	--with-mapserver=/usr \
+	--with-ms-version=7 \
+	--with-json=/usr \
+	--prefix=/usr \
+	--with-metadb=yes \
+	--with-db-backend \
+	--with-rabbitmq=yes \
+     && sed -i "s/-DACCEPT_USE_OF_DEPRECATED_PROJ_API_H/-DPROJ_VERSION_MAJOR=8/g" ./ZOOMakefile.opts \
+     && make -j4\
+     && make install \
+     && cp main.cfg /usr/lib/cgi-bin \
+     && cp zoo_loader.cgi /usr/lib/cgi-bin \
+     && cp zoo_loader_fpm /usr/lib/cgi-bin \
+     && cp oas.cfg /usr/lib/cgi-bin \
+    \
+    # Install Basic Authentication sample
+    # TODO: is this still required?
+    && cd ../zoo-services/utils/security/basicAuth \
+    && make \
+    && cp cgi-env/* /usr/lib/cgi-bin \
+    \
+     && sed -i "s%http://www.zoo-project.org/zoo/%http://127.0.0.1%g" /usr/lib/cgi-bin/main.cfg \
+     && sed -i "s%../tmpPathRelativeToServerAdress/%http://localhost/temp/%g" /usr/lib/cgi-bin/main.cfg \
+     && echo "\n[env]\nPYTHONPATH=/usr/miniconda3/envs/ades-dev/lib/python${PY_VER}/site-packages"\
+          >> /usr/lib/cgi-bin/main.cfg \
+     && a2enmod cgi rewrite \
+     && sed "s:AllowOverride None:AllowOverride All:g" -i /etc/apache2/apache2.conf \
+     && cd /opt/ZOO-Project \
+     && cp ./docker/.htaccess /var/www/html/.htaccess \
+     && cp -r zoo-project/zoo-services/utils/open-api/templates/index.html /var/www/index.html \
+     && cp -r zoo-project/zoo-services/utils/open-api/static /var/www/html/ \
+     && cp zoo-project/zoo-services/utils/open-api/cgi-env/* /usr/lib/cgi-bin/ \
+     && cp zoo-project/zoo-services/utils/security/dru/* /usr/lib/cgi-bin/ \
+     && ln -s /tmp/ /var/www/html/temp \
+     && mkdir /var/www/html/examples/ \
+     # update the securityIn.zcfg
+     && sed "s:serviceType = C:serviceType = Python:g;s:serviceProvider = security_service.zo:serviceProvider = service:g" -i /usr/lib/cgi-bin/securityIn.zcfg \
+     && curl -o /var/www/html/examples/deployment-job.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/snuggs/app-deploy-body.json \
+     && curl -o /var/www/html/examples/deployment-job1.json https://raw.githubusercontent.com/EOEPCA/proc-ades/1b55873dad2684f3333842aea77efb6fb33aa210/test/sample_apps/dNBR/app-deploy-body1.json \
+     && curl -o /var/www/html/examples/deployment-job2.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/dNBR/app-deploy-body.json \
+     && curl -o /var/www/html/examples/job_order1.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/snuggs/app-execute-body.json \
+     && curl -o /var/www/html/examples/job_order2.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/snuggs/app-execute-body2.json \
+     && curl -o /var/www/html/examples/job_order3.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/snuggs/app-execute-body3.json \
+     && curl -o /var/www/html/examples//app-package.cwl https://raw.githubusercontent.com/EOEPCA/app-snuggs/main/app-package.cwl \
+     && cd .. && rm -rf ZOO-Project
+
+#
+# Install Swagger-ui
+#
+RUN git clone --depth 1 https://github.com/swagger-api/swagger-ui.git \
+    && mv swagger-ui /var/www/html/swagger-ui \
+    && sed "s=https://petstore.swagger.io/v2/swagger.json=http://localhost:8080/ogc-api/api=g" -i /var/www/html/swagger-ui/dist/* \
+    && mv /var/www/html/swagger-ui/dist /var/www/html/swagger-ui/oapip
+
+COPY docker/default.conf /etc/apache2/sites-available/000-default.conf
+COPY zoo-project/zoo-services/utils/open-api/dru/DeployProcess.py /usr/lib/cgi-bin
+COPY zoo-project/zoo-services/utils/open-api/dru/DeployProcess.zcfg /usr/lib/cgi-bin
+COPY zoo-project/zoo-services/utils/open-api/dru/UndeployProcess.py /usr/lib/cgi-bin
+COPY zoo-project/zoo-services/utils/open-api/dru/UndeployProcess.zcfg /usr/lib/cgi-bin
+COPY zoo-project/zoo-services/utils/open-api/dru/deploy_util.py /usr/lib/cgi-bin
+
+RUN chmod -R 777 /usr/lib/cgi-bin
+
+RUN mkdir /tmp/cookiecutter-templates && \
+    chmod -R 777 /tmp/cookiecutter-templates
+
+EXPOSE 80
+CMD ["apachectl", "-D", "FOREGROUND"]

docker/.htaccess

@@ -1,7 +1,7 @@
 RewriteEngine On
 SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1
 SetEnvIf Request_URI "^\/?(\w+)\/ogc-api(.*)" SERVICES_NAMESPACE=$1
-RewriteRule ^\/?(\w+)\/ogc-api/(.*) /ogc-api/$2 [PT]
+RewriteRule ^(.*)/ogc-api/(.*) /ogc-api/$2 [PT]
 RewriteCond %{REQUEST_FILENAME} !-f
 RewriteCond %{REQUEST_FILENAME} !-d
 RewriteRule ^ogc-api/api.html$ /cgi-bin/zoo_loader.cgi?/api.html [L,QSA]

docker/dru/ades-dev_env.yaml

@@ -0,0 +1,22 @@
+name: ades-dev
+channels:
+  - anaconda
+  - conda-forge
+  - terradue
+  - eoepca
+  - terradue/label/dev
+dependencies:
+  - python=3.8
+  - jinja2
+  - cookiecutter=1.7.2
+  - boto3
+  - pycalrissian
+  - zoo-calrissian-runner=0.1.7
+  - python-kubernetes
+  - cwl-wrapper=0.10.1
+  - coverage
+  - pip
+  - pip:
+    - cwl_utils==0.14
+    - cheetah3
+    - psycopg2-binary

docker/main.cfg

@@ -70,3 +70,5 @@ async_worker=20
 
 [servicesNamespace]
 path=/opt/zooservices_namespaces
+deploy_service_provider=DeployPy
+undeploy_service_provider=UndeployPy

docs/_static/minikube_upgrade_output.txt

@@ -0,0 +1,12 @@
+Release "zoo-project-dru" does not exist. Installing it now.
+NAME: zoo-project-dru
+LAST DEPLOYED: Mon Aug 21 14:26:22 2023
+NAMESPACE: zoo-project-dru
+STATUS: deployed
+REVISION: 1
+NOTES:
+1. Get the application URL by running these commands:
+  export POD_NAME=$(kubectl get pods --namespace zoo-project-dru -l "app.kubernetes.io/name=ades,app.kubernetes.io/instance=zoo-project-dru" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace zoo-project-dru $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace zoo-project-dru port-forward $POD_NAME 8080:$CONTAINER_PORT

docs/_static/my-minio-fs.yaml

@@ -0,0 +1,49 @@
+## Create persistent volume claim for minio to store data.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: my-minio-fs-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+## Run minio fs deployment.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-minio-fs
+spec:
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: my-minio-fs
+  template:
+    metadata:
+      labels:
+        app: my-minio-fs
+    spec:
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: my-minio-fs-pvc
+      containers:
+      - name: my-minio-fs
+        volumeMounts:
+        - name: data 
+          mountPath: "/data"
+        image: minio/minio:RELEASE.2017-11-22T19-55-46Z
+        args:
+        - server
+        - /data
+        env:
+        - name: MINIO_ACCESS_KEY
+          value: "minio"
+        - name: MINIO_SECRET_KEY
+          value: "minio123"
+        ports:
+        - containerPort: 9000
+          hostPort: 9000