Update HPGC support

Pass the current response to filter_out for possible modification

Support returning response from a filter_in process

Publish the eoAPI proxy-like process

docker/hpgc/Dockerfile

@@ -1,9 +1,7 @@
 FROM ubuntu:focal
 
 # to build:
-# docker build --rm -t ades:latest .
-# to run:
-# docker run --rm -ti -p 80:80  ades:latest
+# docker build --rm -t zooproject/zoo-project:hpgc-latest . -f docker/hpgc/Dockerfile
 
 # procadesdev:latest
 ENV DEBIAN_FRONTEND noninteractive
@@ -38,10 +36,11 @@ RUN wget -nv \
     && bash Miniconda3-py39_4.10.3-Linux-x86_64.sh -b -p /usr/miniconda3 \
     && rm -f Miniconda3-py39_4.10.3-Linux-x86_64.sh
 ENV PATH="/usr/miniconda3/envs/ades-dev/bin:/usr/miniconda3/bin:${PATH}"
-COPY docker/dru/ades-dev_env.yaml /tmp/ades-dev_env.yaml
+COPY docker/hpgc/ades-dev_env.yaml /tmp/ades-dev_env.yaml
 RUN conda install mamba -n base -c conda-forge && \
     mamba env create --file /tmp/ades-dev_env.yaml &&\
     rm /tmp/ades-dev_env.yaml
+RUN ln -s /usr/miniconda3/envs/ades-dev/bin/kubectl /usr/bin/kubectl
 
 ########################################
 # ZOO_Prerequisites
@@ -72,6 +71,7 @@ apt-get install -qqy  --no-install-recommends software-properties-common\
 	libxslt1-dev\
 	uuid-dev\
 	libjson-c-dev\
+	mapserver-bin \
 	libmapserver-dev\
 	libgdal-dev\
 	libaprutil1-dev \
@@ -127,6 +127,8 @@ RUN curl -o libssh2-1.11.0.tar.bz2 https://github.com/libssh2/libssh2/releases/d
     \
     # Install Basic Authentication sample
     # TODO: is this still required?
+    && cp ../zoo-services/utils/open-api/server/publish.py /usr/lib/cgi-bin/ \
+    && cp ../zoo-services/utils/open-api/server/subscriber.py /usr/lib/cgi-bin/ \
     && cp ../zoo-services/utils/hpc/examples/callback.py /usr/lib/cgi-bin/ \
     && chmod 755 /usr/lib/cgi-bin/callback.py \
     && cd ../zoo-services/utils/security/basicAuth \
@@ -163,6 +165,8 @@ RUN curl -o libssh2-1.11.0.tar.bz2 https://github.com/libssh2/libssh2/releases/d
      && curl -o /var/www/html/examples/job_order3.json https://raw.githubusercontent.com/EOEPCA/proc-ades/master/test/sample_apps/v2/snuggs/app-execute-body3.json \
      && curl -o /var/www/html/examples//app-package.cwl https://raw.githubusercontent.com/EOEPCA/app-snuggs/main/app-package.cwl \
      && mkdir /opt/zooservices_namespaces \
+     && sed "s=/usr/bin/python3=/usr/miniconda3/envs/ades-dev/bin/python=g" -i /usr/lib/cgi-bin/subscriber.py /usr/lib/cgi-bin/publish.py \
+     && ln -s $(which mapserv) /usr/lib/cgi-bin/ \
      && chown www-data:www-data -R /opt/zooservices_namespaces\
      && a2enmod ssl \
      #&& sed "s:Listen 80:Listen 8702:g;s:443:8703:g" -i /etc/apache2/ports.conf \

docker/hpgc/ades-dev_env.yaml

@@ -0,0 +1,26 @@
+name: ades-dev
+channels:
+  - anaconda
+  - conda-forge
+  - terradue
+  - eoepca
+  - terradue/label/dev
+dependencies:
+  - python=3.8
+  - jinja2
+  - cookiecutter=2.1.1
+  - boto3
+  - pycalrissian
+  - zoo-calrissian-runner=0.1.8
+  - python-kubernetes
+  - cwl-wrapper=0.12.1
+  - coverage
+  - pip
+  - pip:
+    - cwl_utils==0.14
+    - cheetah3
+    - psycopg2-binary
+    - redis
+    - certifi>=2023.7.22
+    - cryptography>=39.0.1
+    - wheel>=0.38.1

docker/hpgc/docker-compose.yml

@@ -17,8 +17,6 @@ services:
       #- ./zoo-project/zoo-services/echo-py/cgi-env:/usr/lib/cgi-bin/org/n52/javaps/test
       - ./zoo-project/zoo-services/echo-py/cgi-env/echo_service.py:/usr/lib/cgi-bin/echo_service.py
       - ./zoo-project/zoo-services/echo-py/cgi-env/echo.zcfg:/usr/lib/cgi-bin/echo.zcfg
-      - ./zoo-project/zoo-services/utils/open-api/server/publish.py:/usr/lib/cgi-bin/publish.py
-      - ./zoo-project/zoo-services/utils/open-api/server/subscriber.py:/usr/lib/cgi-bin/subscriber.py
       - ./docker/mapserver.conf:/mapserver/etc/mapserver.conf
       - ./docker/.htaccess:/var/www/html/.htaccess
       - ./zoo-project/zoo-services/utils/open-api/templates/index.html:/var/www/index.html
@@ -46,7 +44,7 @@ services:
       - ./docker/tmp:/tmp/zTmp
       - ./docker/mapserver.conf:/mapserver/etc/mapserver.conf
       - ./docker/com:/usr/com/zoo-project
-      - ./docker/startUp.sh:/startUp.sh
+      - ./docker/hpgc/startUp.sh:/startUp.sh
       # Uncomment below to expose a process named org.n52.javaps.test.EchoProcess
       #- ./zoo-project/zoo-services/echo-py/cgi-env:/usr/lib/cgi-bin/org/n52/javaps/test
       - ./zoo-project/zoo-services/echo-py/cgi-env/echo_service.py:/usr/lib/cgi-bin/echo_service.py
@@ -65,7 +63,7 @@ services:
     depends_on:
       - redis
     ports:
-      - 8801:8801
+      - 8801:8888
     environment:
       - ZOO_REDIS_HOST=redis
     volumes:

docker/hpgc/main.cfg

@@ -1,15 +1,15 @@
 [main]
 encoding = utf-8
 version = 1.0.0
-serverAddress = http://tb19-dev.geolabs.fr:8701/cgi-bin/zoo_loader.cgi
+serverAddress = http://localhost/cgi-bin/zoo_loader.cgi
 language = en-US
 lang = fr-FR,en-CA,en-US
 tmpPath=/tmp/zTmp/
-tmpUrl = http://tb19-dev.geolabs.fr:8701/temp/
+tmpUrl = http://localhost/temp/
 dataPath = /usr/com/zoo-project
 cacheDir = /tmp/zTmp/
 templatesPath = /var/www/
-mapserverAddress = http://tb19-dev.geolabs.fr:8701/cgi-bin/mapserv
+mapserverAddress = http://localhost/cgi-bin/mapserv
 msOgcVersion=1.0.0
 cors=true
 memory=protect
@@ -96,7 +96,7 @@ ssh_host=anvil.rcac.purdue.edu
 ssh_port=22
 ssh_user=x-gfenoy
 ssh_key=/var/www/.ssh/id_rsa.pub
-ssh_password=
+essh_password=
 remote_data_path=/home/x-gfenoy/wps_executions/data
 remote_persitent_data_path=/home/x-gfenoy/wps_executions/datap
 remote_work_path=/home/x-gfenoy/wps_executions/script
@@ -111,7 +111,7 @@ ssh_host=anvil.rcac.purdue.edu
 ssh_port=22
 ssh_user=x-gfenoy
 ssh_key=/var/www/.ssh/id_rsa.pub
-ssh_password=
+essh_password=
 remote_data_path=/home/x-gfenoy/wps_executions/data
 remote_persitent_data_path=/home/x-gfenoy/wps_executions/datap
 remote_work_path=/home/x-gfenoy/wps_executions/script
@@ -120,3 +120,10 @@ jobscript_body=/usr/lib/cgi-bin/config-hpc1_body.txt
 sbatch_substr=Submitted batch job
 billing_nb_cpu=1
 remote_command_opt=Account%30,AdminComment%30,AllocCPUS%30,AllocNodes%30,AllocTRES%30,AssocID%30,AveCPU%30,AveCPUFreq%30,AveDiskRead%30,AveDiskWrite%30,AvePages%30,AveRSS%30,AveVMSize%30,BlockID%30,Cluster%30,Comment%30,Constraints%30,ConsumedEnergy%30,ConsumedEnergyRaw%30,Container%30,CPUTime%30,CPUTimeRAW%30,DBIndex%30,DerivedExitCode%30,Elapsed%30,ElapsedRaw%30,Eligible%30,End%30,ExitCode%30,Extra%30,FailedNode%30,Flags%30,GID%30,Group%30,JobID%30,JobIDRaw%30,JobName%30,Layout%30,Licenses%30,MaxDiskRead%30,MaxDiskReadNode%30,MaxDiskReadTask%30,MaxDiskWrite%30,MaxDiskWriteNode%30,MaxDiskWriteTask%30,MaxPages%30,MaxPagesNode%30,MaxPagesTask%30,MaxRSS%30,MaxRSSNode%30,MaxRSSTask%30,MaxVMSize%30,MaxVMSizeNode%30,MaxVMSizeTask%30,McsLabel%30,MinCPU%30,MinCPUNode%30,MinCPUTask%30,NCPUS%30,NNodes%30,NodeList%30,NTasks%30,Partition%30,Planned%30,PlannedCPU%30,PlannedCPURAW%30,Priority%30,QOS%30,QOSRAW%30,Reason%30,ReqCPUFreq%30,ReqCPUFreqGov%30,ReqCPUFreqMax%30,ReqCPUFreqMin%30,ReqCPUS%30,ReqMem%30,ReqNodes%30,ReqTRES%30,Reservation%30,ReservationId%30,Start%30,State%30,Submit%30,SubmitLine%30,Suspended%30,SystemComment%30,SystemCPU%30,Timelimit%30,TimelimitRaw%30,TotalCPU%30,TRESUsageInAve%30,TRESUsageInMax%30,TRESUsageInMaxNode%30,TRESUsageInMaxTask%30,TRESUsageInMin%30,TRESUsageInMinNode%30,TRESUsageInMinTask%30,TRESUsageInTot%30,TRESUsageOutAve%30,TRESUsageOutMax%30,TRESUsageOutMaxNode%30,TRESUsageOutMaxTask%30,TRESUsageOutMin%30,TRESUsageOutMinNode%30,TRESUsageOutMinTask%30,TRESUsageOutTot%30,UID%30,User%30,UserCPU%30,WCKey%30,WCKeyID%30,WorkDir%30
+
+[headers]
+X-Powered-By=ZOO-Project-HPGC
+Access-Control-Allow-Origin=*
+Access-Control-Allow-Methods=GET,PUT,POST,DELETE
+Access-Control-Allow-Headers=Content-Type,Authorization
+

docker/hpgc/oas.cfg

@@ -12,7 +12,7 @@ license_name=OGC license
 license_url=https://raw.githubusercontent.com/opengeospatial/ogcapi-processes/master/LICENSE
 full_html_support=false
 partial_html_support=false
-wsUrl=ws://tb19-dev.geolabs.fr:8888/
+wsUrl=ws://localhost:8801/
 publisherUrl=http://zookernel/cgi-bin/publish.py?jobid=
 link_href=http://zoo-project.org/dl/link.json
 tags=Home,API,Conformance,GetCapabilities,Deploy,Replace,Undeploy,ProcessDescription,ExecuteEndpoint,JobList,GetStatus,GetResult,Dismiss
@@ -139,15 +139,16 @@ abstract=Information about the available processes
 tags=GetCapabilities
 tags_description=
 parameters=/components/parameters/limitParam,/components/parameters/skipParam
-schema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/rel-1.0/core/openapi/responses/ProcessList.yaml
+schema=https://raw.githubusercontent.com/opengeospatial/ogcapi-processes/master/openapi/responses/processes-core/rProcessList.yaml
+aschema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/rel-1.0/core/openapi/responses/ProcessList.yaml
 method_1=post
 code_1=201
 title_1=deploy a new processes
 abstract_1=Deploy a new processes
 tags_1=Deploy
 tags_description_1=
 parameters_1=/components/parameters/wParam
-schema_1=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/extensions/deploy_replace_undeploy/standard/openapi/responses/DeployProcess.yaml
+schema_1=https://raw.githubusercontent.com/opengeospatial/ogcapi-processes/master/openapi/responses/processes-dru/rDeployProcess.yaml
 ecode_1=403,409,500
 isecured_1=BasicAuth
 requestBody_length_1=2
@@ -269,7 +270,7 @@ schema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/rel-1.0/core/o
 [requestBodyPkg]
 abstract=Mandatory OGC Application Package in JSON format
 type=application/ogcapppkg+json
-schema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/extensions/deploy_replace_undeploy/standard/openapi/schemas/swagger/ogcapppkg.yaml
+schema=https://raw.githubusercontent.com/opengeospatial/ogcapi-processes/master/openapi/schemas/processes-dru/ogcapppkg.yaml
 
 [requestBodyCwl]
 abstract=Mandatory OGC Application Package in CWL format

docker/hpgc/startUp.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Author : Gérald FENOY
+#
+# Copyright 2021 GeoLabs SARL. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+mkdir -p /tmp/zTmp/statusInfos
+cp /var/www/html/data/* /usr/com/zoo-project
+chown www-data:www-data -R /tmp/zTmp /usr/com/zoo-project
+chmod 777 -R /tmp/zTmp
+
+CMD="curl -o toto.out http://rabbitmq:15672"
+$CMD
+cat toto.out
+if [ -e toto.out ]; then echo "Should start" ; else echo wait; sleep 1; $CMD ; fi 
+
+while [ ! -e toto.out ]; do echo wait; sleep 1; $CMD ;  done
+
+
+echo "START FPM in 5 seconds"
+
+sleep 5
+
+cd /usr/lib/cgi-bin
+touch /var/log/zoofpm.log
+chown www-data:www-data -R /var/www/.ssh
+chown www-data:www-data /var/log/zoofpm.log
+su www-data -s /bin/bash -c 'cd /usr/lib/cgi-bin; ./zoo_loader_fpm ./main.cfg 2> /var/log/zoofpm.log >> /var/log/zoofpm.log'

zoo-project/HISTORY.txt

@@ -1,4 +1,14 @@
 Version 2.0.0
+  * Add a trivial DeployOnHpc service for automating the installation
+  and the deployment of a singularity container using the image
+  parameter provided in the executionUnit used when deploying the
+  process
+  * Set the key json_response_object with the JSON response that the
+  ZOO-Kernel produced for the request, it can be updated from a
+  filter_out process 
+  * Update filter_in to support returning a response directly by
+  adding a reponse key in the lenv section (binary is supported for
+  this response)
   * Provide an initial Dockefile dedicated to the DRU support with
   remote HPC execution support
   * Add a USE_HPC_NESTEDOUTPUTS build option to activate the nested

zoo-project/zoo-kernel/service_json.c

@@ -3196,6 +3196,8 @@ extern "C" {
 	      json_object *cc=json_object_new_array();
 	      json_object_array_add(cc,json_object_new_string(vMap->value));
 	      json_object_object_add(methodc,"tags",cc);
+	      if(pmName==NULL)
+		pmName=getMapArray(tmpMaps->content,"operationId",i);
 	      if(pmName!=NULL){
 		char* pcaOperationId=(char*)malloc((strlen(vMap->value)+strlen(pmName->value)+1)*sizeof(char));
 		sprintf(pcaOperationId,"%s%s",vMap->value,pmName->value);
@@ -3436,6 +3438,8 @@ extern "C" {
 
 		    json_object_object_add(pajPost,"requestBody",pajRBody);
 		    json_object_object_add(pajPost,"responses",pajResponse);
+		    if(pmName==NULL)
+		      pmName=getMapArray(tmpMaps->content,"operationId",i);
 		    if(pmName!=NULL){
 		      char* pcaOperationId=(char*)malloc((strlen(pmState->value)+strlen(pmName->value)+1)*sizeof(char));
 		      sprintf(pcaOperationId,"%s%s",pmState->value,pmName->value);

zoo-project/zoo-kernel/sshapi.c

@@ -533,7 +533,7 @@ int ssh_exec(maps* conf,const char* command,int cnt){
   map* uuid=getMapFromMaps(conf,"lenv","usid");
   char *logPath=(char*)malloc((strlen(tmpPath->value)+strlen(uuid->value)+11)*sizeof(char));
   sprintf(logPath,"%s/exec_out_%s",tmpPath->value,uuid->value);
-  
+
   FILE* logFile=fopen(logPath,"wb");
   free(logPath);
   while(true){