[hailctl] clarify suggested use of -- and allow it in hailctl batch s…

…ubmit

CHANGELOG: Resolves hail-is#13446. In all three submit commands (batch, dataproc, and hdinsight), Hail now allows and encourages the use of -- to separate arguments meant for the user script from those meant for hailctl. In hailctl batch submit, option-like arguments, for example "--foo", are now supported before "--" if and only if they do not conflict with a hailctl option.

hail/python/hailtop/hailctl/batch/cli.py

@@ -149,16 +149,23 @@ def job(batch_id: int, job_id: int, output: StructuredFormatOption = StructuredF
             print(f"Job with ID {job_id} on batch {batch_id} not found")
 
 
-@app.command()
+@app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
 def submit(
+    ctx: typer.Context,
     script: str,
-    arguments: Ann[Optional[List[str]], Arg()] = None,
+    arguments: Ann[Optional[List[str]], Arg(help='You should use -- if you want to pass option-like arguments through.')] = None,
     files: Ann[
         Optional[List[str]], Opt(help='Files or directories to add to the working directory of the job.')
     ] = None,
     name: Ann[str, Opt(help='The name of the batch.')] = '',
     image_name: Ann[Optional[str], Opt(help='Name of Docker image for the job (default: hailgenetics/hail)')] = None,
     output: StructuredFormatPlusTextOption = StructuredFormatPlusText.TEXT,
 ):
-    '''Submit a batch with a single job that runs SCRIPT with the arguments ARGUMENTS.'''
-    asyncio.run(_submit.submit(name, image_name, files or [], output, script, arguments or []))
+    '''Submit a batch with a single job that runs SCRIPT with the arguments ARGUMENTS.
+
+    If you wish to pass option-like arguments you should use "--". For example:
+
+        hailctl batch submit --image-name docker.io/image my_script.py -- some-argument --animal dog
+    '''
+    raise ValueError((name, image_name, files or [], output, script, [*(arguments or []), *ctx.args]))
+    asyncio.run(_submit.submit(name, image_name, files or [], output, script, [*(arguments or []), *ctx.args]))

hail/python/hailtop/hailctl/dataproc/cli.py

@@ -317,13 +317,20 @@ def submit(
     ] = None,
     dry_run: DryRunOption = False,
     region: Ann[Optional[str], Opt(help='Compute region for the cluster.')] = None,
+    arguments: Ann[Optional[List[str]], Arg(help='You should use -- if you want to pass option-like arguments through.')] = None,
 ):
+    '''Submit the Python script at path SCRIPT to a running Dataproc cluster with name NAME.
+
+    You may pass arguments to the script being submitted by listing them after the script; however,
+    if you wish to pass option-like arguments you should use "--". For example:
+
+
+
+    $ hailctl dataproc submit name --image-name docker.io/image my_script.py -- some-argument --animal dog
+
     '''
-    Submit the Python script at path SCRIPT to a running Dataproc cluster with
-    name NAME. To pass arguments to the script being submitted, just list them
-    after the name of the script.
-    '''
-    dataproc_submit(name, script, files, pyfiles, properties, gcloud_configuration, dry_run, region, ctx.args)
+    raise ValueError((name, script, files, pyfiles, properties, gcloud_configuration, dry_run, region, [*(arguments or []), *ctx.args]))
+    dataproc_submit(name, script, files, pyfiles, properties, gcloud_configuration, dry_run, region, [*(arguments or []), *ctx.args])
 
 
 @app.command()

hail/python/hailtop/hailctl/hdinsight/cli.py

@@ -153,11 +153,19 @@ def submit(
     storage_account: Ann[str, Arg(help="Storage account in which the cluster's container exists.")],
     http_password: Ann[str, Arg(help='Web password for the cluster')],
     script: Ann[str, Arg(help='Path to script.')],
+    arguments: Ann[Optional[List[str]], Arg(help='You should use -- if you want to pass option-like arguments through.')] = None,
 ):
     '''
     Submit a job to an HDInsight cluster configured for Hail.
+
+    If you wish to pass option-like arguments you should use "--". For example:
+
+
+
+    $ hailctl hdinsight submit name account password script.py --image-name docker.io/image my_script.py -- some-argument --animal dog
     '''
-    hdinsight_submit(name, storage_account, http_password, script, ctx.args)
+    raise ValueError((name, storage_account, http_password, script, [*(arguments or []), *ctx.args]))
+    hdinsight_submit(name, storage_account, http_password, script, [*(arguments or []), *ctx.args])
 
 
 @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})