Cleans up the operator's redundant functionality.

ads/opctl/backend/ads_ml_job.py

@@ -16,8 +16,6 @@
 from distutils import dir_util
 from typing import Dict, Tuple, Union
 
-from jinja2 import Environment, PackageLoader
-
 from ads.common.auth import AuthContext, AuthType, create_signer
 from ads.common.oci_client import OCIClientFactory
 from ads.jobs import (
@@ -199,36 +197,6 @@ def run(self) -> Dict:
             print("JOB RUN OCID:", run_id)
             return {"job_id": job_id, "run_id": run_id}
 
-    def init_operator(self):
-        # TODO: check if folder is empty, check for force overwrite
-        # TODO: check that command is being run from advanced-ds repo (important until ads released)
-
-        operator_folder = self.config["execution"].get("operator_folder_path")
-        os.makedirs(operator_folder, exist_ok=True)
-
-        operator_folder_name = os.path.basename(os.path.normpath(operator_folder))
-        docker_tag = f"{os.path.join(self.config['infrastructure'].get('docker_registry'), operator_folder_name)}:latest"
-
-        self.config["execution"]["operator_folder_name"] = operator_folder_name
-        self.config["execution"]["docker_tag"] = docker_tag
-
-        operator_slug = self.config["execution"].get("operator_slug")
-        self._jinja_write(operator_slug, operator_folder)
-
-        # DONE
-        print(
-            "\nInitialization Successful.\n"
-            f"All code should be written in main.py located at: {os.path.join(operator_folder, 'main.py')}\n"
-            f"Additional libraries should be added to environment.yaml located at: {os.path.join(operator_folder, 'environment.yaml')}\n"
-            "Any changes to main.py will require re-building the docker image, whereas changes to args in the"
-            " runtime section of the yaml file do not. Write accordingly.\n"
-            "Run this cluster with:\n"
-            f"\tdocker build -t {docker_tag} -f {os.path.join(operator_folder, 'Dockerfile')} .\n"
-            f"\tads opctl publish-image {docker_tag} \n"
-            f"\tads opctl run -f {os.path.join(operator_folder, operator_slug + '.yaml')} \n"
-        )
-        return operator_folder
-
     def delete(self):
         """
         Delete Job or Job Run from OCID.
@@ -264,25 +232,6 @@ def watch(self):
             run = DataScienceJobRun.from_ocid(run_id)
             run.watch(interval=interval, wait=wait)
 
-    def _jinja_write(self, operator_slug, operator_folder):
-        # TODO AH: fill in templates with relevant details
-        env = Environment(
-            loader=PackageLoader("ads", f"opctl/operators/{operator_slug}")
-        )
-
-        for setup_file in [
-            "Dockerfile",
-            "environment.yaml",
-            "main.py",
-            "run.py",
-            "start_scheduler.sh",
-            "start_worker.sh",
-            "dask_cluster.yaml",
-        ]:
-            template = env.get_template(setup_file + ".jinja2")
-            with open(os.path.join(operator_folder, setup_file), "w") as ff:
-                ff.write(template.render(config=self.config))
-
     def _create_payload(self, infra=None, name=None) -> Job:
         if not infra:
             infra = self.config.get("infrastructure", {})

ads/opctl/cli.py

@@ -24,7 +24,6 @@
 from ads.opctl.cmds import deactivate as deactivate_cmd
 from ads.opctl.cmds import delete as delete_cmd
 from ads.opctl.cmds import init as init_cmd
-from ads.opctl.cmds import init_operator as init_operator_cmd
 from ads.opctl.cmds import init_vscode as init_vscode_cmd
 from ads.opctl.cmds import predict as predict_cmd
 from ads.opctl.cmds import run as run_cmd
@@ -54,11 +53,12 @@ def commands():
 @click.help_option("--help", "-h")
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def configure(debug):
+    """Sets up the initial configurations for the ADS OPCTL."""
     suppress_traceback(debug)(configure_cmd)()
 
 
 @commands.command()
-@click.argument("image-type", type=click.Choice(["job-local", "ads-ops-base"]))
+@click.argument("image-type", type=click.Choice(["job-local"]))
 @click.help_option("--help", "-h")
 @click.option(
     "--gpu",
@@ -68,23 +68,10 @@ def configure(debug):
     default=False,
     required=False,
 )
-@click.option(
-    "--source-folder",
-    "-s",
-    help="when building custom operator image, source folder of the custom operator",
-    default=None,
-    required=False,
-)
-@click.option(
-    "--image",
-    "-i",
-    help="image name, used when building custom image",
-    default=None,
-    required=False,
-)
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
-def build_image(image_type, gpu, source_folder, image, debug):
-    suppress_traceback(debug)(build_image_cmd)(image_type, gpu, source_folder, image)
+def build_image(image_type, gpu, debug):
+    """Builds the local Data Science Jobs image."""
+    suppress_traceback(debug)(build_image_cmd)(image_type, gpu)
 
 
 @commands.command()
@@ -101,6 +88,7 @@ def build_image(image_type, gpu, source_folder, image, debug):
 @click.help_option("--help", "-h")
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def publish_image(**kwargs):
+    """Publishes image to the OCI Container Registry."""
     debug = kwargs.pop("debug")
     if kwargs.get("registry", None):
         registry = kwargs["registry"]
@@ -449,30 +437,8 @@ def check(file, **kwargs):
     suppress_traceback(debug)(run_diagnostics_cmd)(config, **kwargs)
 
 
-@commands.command()
-@click.argument("operator_slug", nargs=1)
-@click.option(
-    "--folder_path",
-    "-fp",
-    help="the name of the folder wherein to put the operator code",
-    multiple=True,
-    required=False,
-    default=None,
-)
-@add_options(_options)
-def init_operator(**kwargs):
-    suppress_traceback(kwargs["debug"])(init_operator_cmd)(**kwargs)
-
-
 @commands.command()
 @click.argument("ocid", nargs=1)
-@add_options(_model_deployment_options)
-@click.option(
-    "--conda-pack-folder",
-    required=False,
-    default=None,
-    help="folder where conda packs are saved",
-)
 @click.option(
     "--auth",
     "-a",
@@ -487,6 +453,7 @@ def init_operator(**kwargs):
 )
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def delete(**kwargs):
+    """Deletes a data science service resource."""
     suppress_traceback(kwargs["debug"])(delete_cmd)(**kwargs)
 
 
@@ -513,6 +480,7 @@ def delete(**kwargs):
 )
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def cancel(**kwargs):
+    """Aborts the execution of the OCI resource run."""
     suppress_traceback(kwargs["debug"])(cancel_cmd)(**kwargs)
 
 
@@ -566,7 +534,7 @@ def cancel(**kwargs):
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def watch(**kwargs):
     """
-    ``tail`` logs form a job run, dataflow run or pipeline run.
+    Tails the logs form a job run, data flow run or pipeline run.
     Connects to the logging service that was configured with the JobRun, Application Run or Pipeline Run and streams the logs.
     """
     suppress_traceback(kwargs["debug"])(watch_cmd)(**kwargs)
@@ -575,13 +543,6 @@ def watch(**kwargs):
 @commands.command()
 @click.argument("ocid", nargs=1)
 @click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False)
-@add_options(_model_deployment_options)
-@click.option(
-    "--conda-pack-folder",
-    required=False,
-    default=None,
-    help="folder where conda packs are saved",
-)
 @click.option(
     "--auth",
     "-a",
@@ -597,7 +558,7 @@ def watch(**kwargs):
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def activate(**kwargs):
     """
-    Activates a data science service.
+    Activates a data science service resource.
     """
     suppress_traceback(kwargs["debug"])(activate_cmd)(**kwargs)
 
@@ -627,7 +588,7 @@ def activate(**kwargs):
 @click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
 def deactivate(**kwargs):
     """
-    Deactivates a data science service.
+    Deactivates a data science service resource.
     """
     suppress_traceback(kwargs["debug"])(deactivate_cmd)(**kwargs)
 

ads/opctl/cmds.py

@@ -332,31 +332,6 @@ def _update_env_vars(config, env_vars: List):
     return config
 
 
-def init_operator(**kwargs) -> str:
-    """
-    Initialize the resources for an operator
-
-    Parameters
-    ----------
-    kwargs: dict
-        keyword argument, stores command line args
-    Returns
-    -------
-    folder_path: str
-        a path to the folder with all of the resources
-    """
-    # TODO: confirm that operator slug is in the set of valid operator slugs
-    assert kwargs["operator_slug"] == "dask_cluster"
-
-    if kwargs.get("folder_path"):
-        kwargs["operator_folder_path"] = kwargs.pop("folder_path")[0]
-    else:
-        kwargs["operator_folder_path"] = kwargs["operator_slug"]
-    p = ConfigProcessor().step(ConfigMerger, **kwargs)
-    print(f"config check: {p.config}")
-    return _BackendFactory(p.config).backend.init_operator()
-
-
 def delete(**kwargs) -> None:
     """
     Delete a MLJob/DataFlow run.

ads/opctl/conda/cli.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import click
@@ -17,6 +17,7 @@
 @click.group("conda")
 @click.help_option("--help", "-h")
 def commands():
+    "The CLI to assist in the management of conda environments."
     pass
 
 

ads/opctl/constants.py

@@ -12,10 +12,8 @@
 DEFAULT_MODEL_FOLDER = "~/.ads_ops/models"
 CONDA_PACK_OS_PREFIX_FORMAT = "oci://<bucket>@<namespace>/<prefix>"
 DEFAULT_ADS_CONFIG_FOLDER = "~/.ads_ops"
-OPS_IMAGE_BASE = "ads-operators-base"
 ML_JOB_IMAGE = "ml-job"
 ML_JOB_GPU_IMAGE = "ml-job-gpu"
-OPS_IMAGE_GPU_BASE = "ads-operators-gpu-base"
 DEFAULT_MANIFEST_VERSION = "1.0"
 ADS_CONFIG_FILE_NAME = "config.ini"
 ADS_JOBS_CONFIG_FILE_NAME = "ml_job_config.ini"

ads/opctl/distributed/cli.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import os
 
@@ -21,6 +21,7 @@
 @click.group("distributed-training")
 @click.help_option("--help", "-h")
 def commands():
+    "The CLI to assist in the management of the distributed training."
     pass