Feat: release-v2.11.0 - Adds save_logs and removes spot options from job command

.github/workflows/ci.yml

@@ -86,7 +86,7 @@ jobs:
         INSTANCE_TYPE: "c5.large"
       run: |
         JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER"
-        cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE --spot 2>&1 | tee out.txt
+        cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt
         JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev)
         cloudos job status --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --job-id $JOB_ID
   workflow_list:

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## lifebit-ai/cloudos-cli: changelog
 
+## v2.11.0 (2024-04-16)
+
+- Now, the default `cloudos job run` command will save job process logs. To prevent saving process logs, you can use the new flag `--do-not-save-logs`.
+- Removes unsupported `--spot` option from `cloudos job run`.
+
 ## v2.10.0 (2024-04-11)
 
 - Adds the new parameter `--workflow-docs-link` to add a documentation link to the imported workflow.

README.md

@@ -105,6 +105,12 @@ Options:
   --job-name TEXT                 The name of the job. Default=new_job.
   --resumable                     Whether to make the job able to be resumed
                                   or not.
+  --do-not-save-logs              Avoids process log saving. If you select
+                                  this option, your job process logs will not
+                                  be stored.
+  --spot                          [Deprecated in 2.11.0] This option has been
+                                  deprecated and has no effect. Spot instances
+                                  are no longer available in CloudOS.
   --batch                         [Deprecated in 2.7.0] Since v2.7.0, the
                                   default executor is AWSbatch so there is no
                                   need to use this flag. It is maintained for
@@ -120,7 +126,6 @@ Options:
                                   dard_D4as_v4(azure).
   --instance-disk INTEGER         The amount of disk storage to configure.
                                   Default=500.
-  --spot                          Whether to make a spot instance.
   --storage-mode TEXT             Either 'lustre' or 'regular'. Indicates if
                                   the user wants to select regular or lustre
                                   storage. Default=regular.
@@ -199,8 +204,7 @@ cloudos job run \
     --project-name "$PROJECT_NAME" \
     --workflow-name $WORKFLOW_NAME \
     --job-config $JOB_PARAMS \
-    --resumable \
-    --spot
+    --resumable
 ```
 
 In addition, parameters can also be specified using the command-line `-p` or `--parameter`. For instance,
@@ -216,8 +220,7 @@ cloudos job run \
     --parameter reads=s3://lifebit-featured-datasets/pipelines/rnatoy-data \
     --parameter genome=s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.Ggal71.500bpflank.fa \
     --parameter annot=s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.bed.gff \
-    --resumable \
-    --spot
+    --resumable
 ```
 
 > NOTE: options `--job-config` and `--parameter` are completely compatible and complementary, so you can use a
@@ -253,7 +256,6 @@ cloudos job run \
     --workflow-name $WORKFLOW_NAME \
     --job-config $JOB_PARAMS \
     --resumable \
-    --spot \
     --wait-completion
 ```
 
@@ -304,7 +306,6 @@ If your CloudOS is configured to
 use Azure, you will need to take into consideration the following:
 
 - When sending jobs to CloudOS using `cloudos job run` or `cloudos job run-curated-examples` commands, please use the option `--execution-platform azure`.
-- Option `--spot` is not taking effect when using Azure execution platform.
 - Due to the lack of AWS batch queues in Azure, `cloudos queue list` command is not working.
 
 Other than that, `cloudos-cli` will work very similarly. For instance, this is a typical send job command:
@@ -344,9 +345,9 @@ cloudos job run \
 
 Please, note that HPC execution do not support the following parameters and all of them will be ignored:
 
-- `--resumable`
 - `--job-queue`
-- `--instance-type` | `--instance-disk` | `--spot` | `--cost-limit`
+- `--resumable | --do-not-save-logs`
+- `--instance-type` | `--instance-disk` | `--cost-limit`
 - `--storage-mode` | `--lustre-size`
 - `--wdl-mainfile` | `--wdl-importsfile` | `--cromwell-token`
 

cloudos/__main__.py

@@ -137,6 +137,14 @@ def queue():
 @click.option('--resumable',
               help='Whether to make the job able to be resumed or not.',
               is_flag=True)
+@click.option('--do-not-save-logs',
+              help=('Avoids process log saving. If you select this option, your job process ' +
+                    'logs will not be stored.'),
+              is_flag=True)
+@click.option('--spot',
+              help=('[Deprecated in 2.11.0] This option has been deprecated and has no effect. ' +
+                    'Spot instances are no longer available in CloudOS.'),
+              is_flag=True)
 @click.option('--batch',
               help=('[Deprecated in 2.7.0] Since v2.7.0, the default executor is AWSbatch ' +
                     'so there is no need to use this flag. It is maintained for ' +
@@ -157,9 +165,6 @@ def queue():
               help='The amount of disk storage to configure. Default=500.',
               type=int,
               default=500)
-@click.option('--spot',
-              help='Whether to make a spot instance.',
-              is_flag=True)
 @click.option('--storage-mode',
               help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' +
                     'regular or lustre storage. Default=regular.'),
@@ -226,13 +231,14 @@ def run(apikey,
         git_tag,
         job_name,
         resumable,
+        do_not_save_logs,
+        spot,
         batch,
         ignite,
         job_queue,
         nextflow_profile,
         instance_type,
         instance_disk,
-        spot,
         storage_mode,
         lustre_size,
         wait_completion,
@@ -251,6 +257,13 @@ def run(apikey,
     """Submit a job to CloudOS."""
     print('Executing run...')
     verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert)
+    if spot:
+        print('\n[Message] You have specified spot instances but they are no longer available ' +
+              'in CloudOS. Option ignored.\n')
+    if do_not_save_logs:
+        save_logs = False
+    else:
+        save_logs = True
     if instance_type == 'NONE_SELECTED':
         if execution_platform == 'aws':
             instance_type = 'c5.xlarge'
@@ -260,7 +273,6 @@ def run(apikey,
             instance_type = None
     if execution_platform == 'azure' or execution_platform == 'hpc':
         batch = None
-        spot = None
     elif ignite:
         batch = None
         print('\n[Warning] You have specified ignite executor. Please, note that ignite is being ' +
@@ -274,14 +286,15 @@ def run(apikey,
             raise ValueError('Please, specify your HPC ID using --hpc parameter')
         print('[Message] Please, take into account that HPC execution do not support ' +
               'the following parameters and all of them will be ignored:\n' +
-              '\t--resumable\n' +
               '\t--job-queue\n' +
-              '\t--instance-type | --instance-disk | --spot | --cost-limit\n' +
+              '\t--resumable | --do-not-save-logs\n' +
+              '\t--instance-type | --instance-disk | --cost-limit\n' +
               '\t--storage-mode | --lustre-size\n' +
               '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n')
         wdl_mainfile = None
         wdl_importsfile = None
         storage_mode = 'regular'
+        save_logs = False
     if verbose:
         print('\t...Detecting workflow type')
     cl = Cloudos(cloudos_url, apikey, cromwell_token)
@@ -340,12 +353,12 @@ def run(apikey,
                       git_tag=git_tag,
                       job_name=job_name,
                       resumable=resumable,
+                      save_logs=save_logs,
                       batch=batch,
                       job_queue_id=job_queue_id,
                       nextflow_profile=nextflow_profile,
                       instance_type=instance_type,
                       instance_disk=instance_disk,
-                      spot=spot,
                       storage_mode=storage_mode,
                       lustre_size=lustre_size,
                       execution_platform=execution_platform,
@@ -403,6 +416,14 @@ def run(apikey,
 @click.option('--resumable',
               help='Whether to make the job able to be resumed or not.',
               is_flag=True)
+@click.option('--do-not-save-logs',
+              help=('Avoids process log saving. If you select this option, your job process ' +
+                    'logs will not be stored.'),
+              is_flag=True)
+@click.option('--spot',
+              help=('[Deprecated in 2.11.0] This option has been deprecated and has no effect. ' +
+                    'Spot instances are no longer available in CloudOS.'),
+              is_flag=True)
 @click.option('--batch',
               help=('[Deprecated in 2.7.0] Since v2.7.0, the default executor is AWSbatch ' +
                     'so there is no need to use this flag. It is maintained for ' +
@@ -421,9 +442,6 @@ def run(apikey,
               help='The amount of disk storage to configure. Default=500.',
               type=int,
               default=500)
-@click.option('--spot',
-              help='Whether to make a spot instance.',
-              is_flag=True)
 @click.option('--storage-mode',
               help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' +
                     'regular or lustre storage. Default=regular.'),
@@ -469,11 +487,12 @@ def run_curated_examples(apikey,
                          workspace_id,
                          project_name,
                          resumable,
+                         do_not_save_logs,
+                         spot,
                          batch,
                          ignite,
                          instance_type,
                          instance_disk,
-                         spot,
                          storage_mode,
                          lustre_size,
                          execution_platform,
@@ -495,14 +514,20 @@ def run_curated_examples(apikey,
     runnable_curated_workflows = [
         w for w in curated_workflows if w['workflowType'] == 'nextflow' and len(w['parameters']) > 0
     ]
+    if spot:
+        print('\n[Message] You have specified spot instances but they are no longer available ' +
+              'in CloudOS. Option ignored.\n')
+    if do_not_save_logs:
+        save_logs = False
+    else:
+        save_logs = True
     if instance_type == 'NONE_SELECTED':
         if execution_platform == 'aws':
             instance_type = 'c5.xlarge'
         if execution_platform == 'azure':
             instance_type = 'Standard_D4as_v4'
     if execution_platform == 'azure':
         batch = None
-        spot = None
     elif ignite:
         batch = None
         print('\n[Warning] You have specified ignite executor. Please, note that ignite is being ' +
@@ -517,10 +542,10 @@ def run_curated_examples(apikey,
         j_id = j.send_job(example_parameters=workflow['parameters'],
                           job_name=f"{workflow['name']}|Example_Run",
                           resumable=resumable,
+                          save_logs=save_logs,
                           batch=batch,
                           instance_type=instance_type,
                           instance_disk=instance_disk,
-                          spot=spot,
                           storage_mode=storage_mode,
                           lustre_size=lustre_size,
                           execution_platform=execution_platform,

cloudos/_version.py

@@ -1 +1 @@
-__version__ = '2.10.0'
+__version__ = '2.11.0'

cloudos/jobs/job.py

@@ -190,12 +190,12 @@ def convert_nextflow_to_json(self,
                                  workflow_id,
                                  job_name,
                                  resumable,
+                                 save_logs,
                                  batch,
                                  job_queue_id,
                                  nextflow_profile,
                                  instance_type,
                                  instance_disk,
-                                 spot,
                                  storage_mode,
                                  lustre_size,
                                  execution_platform,
@@ -231,6 +231,8 @@ def convert_nextflow_to_json(self,
             The name to assign to the job.
         resumable: bool
             Whether to create a resumable job or not.
+        save_logs : bool
+            Whether to save job logs or not.
         batch: bool
             Whether to create a batch job or an ignite one.
         job_queue_id : string
@@ -241,8 +243,6 @@ def convert_nextflow_to_json(self,
             Name of the AMI to choose.
         instance_disk : int
             The disk space of the instance, in GB.
-        spot : bool
-            Whether to create a spot instance or not.
         storage_mode : string
             Either 'lustre' or 'regular'. Indicates if the user wants to select regular
             or lustre storage.
@@ -351,16 +351,6 @@ def convert_nextflow_to_json(self,
         if len(example_parameters) > 0:
             for example_param in example_parameters:
                 workflow_params.append(example_param)
-        if spot:
-            instance_type_block = {
-                "instanceType": instance_type,
-                "onDemandFallbackInstanceType": instance_type
-            }
-            instance = "spotInstances"
-        else:
-            instance_type_block = instance_type
-            instance = "instanceType"
-
         if git_tag is not None and git_commit is not None:
             raise ValueError('Please, specify none or only one of --git-tag' +
                              ' or --git-commit options but not both.')
@@ -391,6 +381,7 @@ def convert_nextflow_to_json(self,
             "workflow": workflow_id,
             "name": job_name,
             "resumable": resumable,
+            "saveProcessLogs": save_logs,
             "batch": {
                 "dockerLogin": False,
                 "enabled": batch,
@@ -408,7 +399,7 @@ def convert_nextflow_to_json(self,
             "storageMode": storage_mode,
             "revision": revision_block,
             "profile": nextflow_profile,
-            instance: instance_type_block
+            "instanceType": instance_type
         }
         if execution_platform != 'hpc':
             params['masterInstance'] = {
@@ -427,12 +418,12 @@ def send_job(self,
                  git_tag=None,
                  job_name='new_job',
                  resumable=False,
+                 save_logs=True,
                  batch=True,
                  job_queue_id=None,
                  nextflow_profile=None,
                  instance_type='c5.xlarge',
                  instance_disk=500,
-                 spot=False,
                  storage_mode='regular',
                  lustre_size=1200,
                  execution_platform='aws',
@@ -463,8 +454,10 @@ def send_job(self,
             commit of the default branch will be used.
         job_name : string
             The name to assign to the job.
-        resumable: bool
+        resumable : bool
             Whether to create a resumable job or not.
+        save_logs : bool
+            Whether to save job logs or not.
         batch: bool
             Whether to create a batch job or an ignite one.
         job_queue_id : string
@@ -475,8 +468,6 @@ def send_job(self,
             Type of the AMI to choose.
         instance_disk : int
             The disk space of the instance, in GB.
-        spot : bool
-            Whether to create a spot instance or not.
         storage_mode : string
             Either 'lustre' or 'regular'. Indicates if the user wants to select regular
             or lustre storage.
@@ -522,12 +513,12 @@ def send_job(self,
                                                workflow_id,
                                                job_name,
                                                resumable,
+                                               save_logs,
                                                batch,
                                                job_queue_id,
                                                nextflow_profile,
                                                instance_type,
                                                instance_disk,
-                                               spot,
                                                storage_mode,
                                                lustre_size,
                                                execution_platform,

tests/test_data/convert_nextflow_to_json_params.json

@@ -1 +1 @@
-{"parameters": [{"prefix": "--", "name": "reads", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data"}, {"prefix": "--", "name": "genome", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.Ggal71.500bpflank.fa"}, {"prefix": "--", "name": "annot", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.bed.gff"}], "project": "6054754029b82f0112762b9c", "workflow": "60b0ca54303ee601a69b42d1", "name": "new_job", "resumable": true, "batch": {"dockerLogin": false, "enabled": false, "jobQueue": null}, "cromwellCloudResources": null, "executionPlatform": "aws", "hpc": null ,"storageSizeInGb": 500, "execution": {"computeCostLimit": -1, "optim": "test"}, "lusterFsxStorageSizeInGb": 1200, "storageMode": "regular", "revision": "", "profile": null, "spotInstances": {"instanceType": "c5.xlarge", "onDemandFallbackInstanceType": "c5.xlarge"}, "masterInstance": {"requestedInstance": {"type": "c5.xlarge", "asSpot": false}}}
+{"parameters": [{"prefix": "--", "name": "reads", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data"}, {"prefix": "--", "name": "genome", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.Ggal71.500bpflank.fa"}, {"prefix": "--", "name": "annot", "parameterKind": "textValue", "textValue": "s3://lifebit-featured-datasets/pipelines/rnatoy-data/ggal_1_48850000_49020000.bed.gff"}], "project": "6054754029b82f0112762b9c", "workflow": "60b0ca54303ee601a69b42d1", "name": "new_job", "resumable": true, "saveProcessLogs": true, "batch": {"dockerLogin": false, "enabled": false, "jobQueue": null}, "cromwellCloudResources": null, "executionPlatform": "aws", "hpc": null ,"storageSizeInGb": 500, "execution": {"computeCostLimit": -1, "optim": "test"}, "lusterFsxStorageSizeInGb": 1200, "storageMode": "regular", "revision": "", "profile": null, "instanceType": "c5.xlarge", "masterInstance": {"requestedInstance": {"type": "c5.xlarge", "asSpot": false}}}