Merge branch 'RESTAPI-966-zip-dirs-2' into 'master'

Add /compress and /extract endpoints

See merge request firecrest/firecrest!281

CHANGELOG.md

@@ -9,7 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the SLURM scheduling queue.
+- Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the scheduling queue.
+- Added endpoints `POST /utilities/compress`, `POST /utilities/extract`, `POST /storage/xfer-internal/compress` and `POST /storage/xfer-internal/extract` for file compression and extraction.
+
 ### Changed
 
 ### Fixed

deploy/test-build/cluster/Dockerfile

@@ -117,10 +117,12 @@ ADD cluster/test_sbatch.sh /srv/f7t/.
 ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_forbidden.sh
 ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_rm.sh
 ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_mv.sh
+COPY cluster/test_zip.tar.gz /srv/f7t/test_zip.tar.gz
 RUN chmod 777 /srv/f7t
 RUN chmod 555 /srv/f7t/test_sbatch.sh
 RUN chmod 777 /srv/f7t/test_sbatch_rm.sh
 RUN chmod 777 /srv/f7t/test_sbatch_mv.sh
+RUN chmod 777 /srv/f7t/test_zip.tar.gz
 RUN chmod 700 /srv/f7t/test_sbatch_forbidden.sh
 
 ENTRYPOINT ["supervisord"]

deploy/test-build/cluster/ssh/ssh_command_wrapper.sh

@@ -35,7 +35,7 @@ case "$command" in
     tmp2=${tmp1#* }
     command2=${tmp2%% *}   # remove options
     case "$command2" in
-      base64|cat|chmod|chown|cp|curl|file|head|id|ln|ls|mkdir|mv|rm|sacct|sbatch|scancel|scontrol|sha256sum|squeue|stat|tail|touch)
+      base64|cat|chmod|chown|cp|curl|file|head|id|ln|ls|mkdir|mv|rm|sacct|sbatch|scancel|scontrol|sha256sum|squeue|stat|tail|touch|tar|unzip)
         ;;
       rsvmgmt)
         # advance reservation command

doc/openapi/firecrest-api.yaml

@@ -1323,6 +1323,161 @@ paths:
               description: Command has finished with timeout signal
               schema:
                 type: string
+  '/utilities/compress':
+    parameters:
+      - in: header
+        name: X-Machine-Name
+        description: The system name
+        required: true
+        schema:
+          type: string
+    post:
+      summary: Compress file/directory
+      description: >-
+        Compress files using gzip compression. You can name the output file as you like, but typically these files have a .tar.gz extension.
+      tags:
+        - Utilities
+      requestBody:
+        required: true
+        content:
+          'multipart/form-data':
+            schema:
+              type: object
+              properties:
+                sourcePath:
+                  type: string
+                  description: Absolute filesystem path to file to be compressed
+                targetPath:
+                  type: string
+                  description: Absolute filesystem path of the compressed file
+              required:
+                - sourcePath
+                - targetPath
+              example:
+                sourcePath: /home/user/directory
+                targetPath: /home/user/file.tar.gz
+      responses:
+        '201':
+          description: Object compress succesful
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Utilities-ok'
+        '400':
+          description: Error in compress operation
+          content:
+            text/plain:
+              schema:
+                $ref: '#/components/schemas/Utilities-notok'
+          headers:
+            X-Machine-Does-Not-Exist:
+              description: Machine does not exist
+              schema:
+                type: string
+            X-Machine-Not-Available:
+              description: Machine is not available
+              schema:
+                type: string
+            X-Permission-Denied:
+              description: User does not have permissions to access machine or paths
+              schema:
+                type: string
+            X-Invalid-Path:
+              description: <path> is an invalid path
+              schema:
+                type: string
+            X-Timeout:
+              description: Command has finished with timeout signal
+              schema:
+                type: string
+            X-Exists:
+              description: <targetPath> already exists
+              schema:
+                type: string
+            X-Not-Found:
+              description: <sourcePath> not found
+              schema:
+                type: string
+  '/utilities/extract':
+    parameters:
+      - in: header
+        name: X-Machine-Name
+        description: The system name
+        required: true
+        schema:
+          type: string
+    post:
+      summary: Extract file
+      description: >-
+        Extract files. If you don't select the `extension`, FirecREST will try to guess the right command based on the extension of the sourcePath. Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`.
+      tags:
+        - Utilities
+      requestBody:
+        required: true
+        content:
+          'multipart/form-data':
+            schema:
+              type: object
+              properties:
+                sourcePath:
+                  type: string
+                  description: Absolute filesystem path to file to be extracted
+                targetPath:
+                  type: string
+                  description: Absolute filesystem path where the {sourcePath} is extracted
+                extension:
+                  type: string
+                  description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`
+                  default: "auto"
+              required:
+                - sourcePath
+                - targetPath
+              example:
+                sourcePath: /home/user/file.tar.gz
+                targetPath: /home/user/directory
+                extension: .tar.gz
+      responses:
+        '201':
+          description: Object extract succesful
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Utilities-ok'
+        '400':
+          description: Error in extract operation
+          content:
+            text/plain:
+              schema:
+                $ref: '#/components/schemas/Utilities-notok'
+          headers:
+            X-Machine-Does-Not-Exist:
+              description: Machine does not exist
+              schema:
+                type: string
+            X-Machine-Not-Available:
+              description: Machine is not available
+              schema:
+                type: string
+            X-Permission-Denied:
+              description: User does not have permissions to access machine or paths
+              schema:
+                type: string
+            X-Invalid-Path:
+              description: <path> is an invalid path
+              schema:
+                type: string
+            X-Timeout:
+              description: Command has finished with timeout signal
+              schema:
+                type: string
+            X-Exists:
+              description: <targetPath> already exists
+              schema:
+                type: string
+            X-Not-Found:
+              description: <sourcePath> not found
+              schema:
+                type: string
   '/compute/jobs/upload':
     parameters:
       - in: header
@@ -2035,6 +2190,160 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Task-Creation-Error'
+  '/storage/xfer-internal/compress':
+    parameters:
+      - in: header
+        name: X-Machine-Name
+        description: The system name
+        required: true
+        schema:
+          type: string
+    post:
+      summary: compress files
+      description: >-
+        'Compress files using gzip compression. You can name the output file
+        as you like, but typically these files have a .tar.gz or .tgz extension.
+        Possible to stage-out jobs providing the workload manager Id of a
+        production job. Reference:
+        https://user.cscs.ch/storage/data_transfer/internal_transfer/'
+      tags:
+        - Storage
+      requestBody:
+        required: true
+        content:
+          'multipart/form-data':
+            schema:
+              type: object
+              properties:
+                sourcePath:
+                  type: string
+                  description: source path to the location filesystem
+                targetPath:
+                  type: string
+                  description: Absolute path to destination
+                jobName:
+                  type: string
+                  description: job name the copy operation
+                  default: cp-job
+                time:
+                  type: string
+                  description: >-
+                    'Limit on the total run time of the copy. Acceptable time formats
+                    \'minutes\',   \'minutes:seconds\',   \'hours:minutes:seconds\',
+                    \'days-hours\', \'days-hours:minutes\' and
+                    \'days-hours:minutes:seconds\'. Note: for stage-in queue a
+                    xfer job.'
+                  default: '02:00:00'
+                stageOutJobId:
+                  type: string
+                  description: Copy data after job with id {stageOutJobId} is completed
+                  default: null
+                account:
+                  type: string
+                  description: Name of the account associated to the user in the scheduler. If not set, system default is taken.
+                  default: null
+              required:
+                - sourcePath
+                - targetPath
+              example:
+                sourcePath: /home/user/origin
+                targetPath: /home/user/destination
+                jobName: cp-firecrest-job
+                stageOutJobId: "123456"
+                time: "2-03:00:00"
+      responses:
+        '201':
+          description: Task created
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Task-Creation-Success'
+        '400':
+          description: Task creation error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Task-Creation-Error'
+  '/storage/xfer-internal/extract':
+    parameters:
+      - in: header
+        name: X-Machine-Name
+        description: The system name
+        required: true
+        schema:
+          type: string
+    post:
+      summary: extract files
+      description: >-
+        'Extract files. If you don't select the `extension`, FirecREST will
+        try to guess the right command based on the extension of the sourcePath.
+        Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`.
+        Possible to stage-out jobs providing the workload manager Id of a
+        production job. Reference:
+        https://user.cscs.ch/storage/data_transfer/internal_transfer/'
+      tags:
+        - Storage
+      requestBody:
+        required: true
+        content:
+          'multipart/form-data':
+            schema:
+              type: object
+              properties:
+                sourcePath:
+                  type: string
+                  description: source path to the location filesystem
+                targetPath:
+                  type: string
+                  description: Absolute path to destination
+                jobName:
+                  type: string
+                  description: job name the copy operation
+                  default: cp-job
+                time:
+                  type: string
+                  description: >-
+                    'Limit on the total run time of the copy. Acceptable time formats
+                    \'minutes\',   \'minutes:seconds\',   \'hours:minutes:seconds\',
+                    \'days-hours\', \'days-hours:minutes\' and
+                    \'days-hours:minutes:seconds\'. Note: for stage-in queue a
+                    xfer job.'
+                  default: '02:00:00'
+                stageOutJobId:
+                  type: string
+                  description: Copy data after job with id {stageOutJobId} is completed
+                  default: null
+                account:
+                  type: string
+                  description: Name of the account associated to the user in the scheduler. If not set, system default is taken.
+                  default: null
+                extension:
+                  type: string
+                  description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`
+                  default: "auto"
+              required:
+                - sourcePath
+                - targetPath
+              example:
+                sourcePath: /home/user/origin
+                targetPath: /home/user/destination
+                jobName: cp-firecrest-job
+                stageOutJobId: "123456"
+                time: "2-03:00:00"
+                extension: ".tar.gz"
+      responses:
+        '201':
+          description: Task created
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Task-Creation-Success'
+        '400':
+          description: Task creation error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Task-Creation-Error'
   '/storage/xfer-external/upload':
     parameters:
       - in: header
@@ -2829,7 +3138,6 @@ components:
           enum:
             - 200
             - 400
-
     Filesystems:
       type: object
       properties:
@@ -2840,8 +3148,6 @@ components:
             type: array
             items:
               $ref: '#/components/schemas/Filesystem'
-
-
     Job:
       type: object
       required: