Sync main and develop branches

.github/workflows/run-unittests-default_setup.yml

@@ -0,0 +1,65 @@
+name: Unit Tests unitary/default_setup/*
+
+on:
+  workflow_dispatch:
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}, tests/unitary/default_setup
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Caching python libraries installed with pip
+      # https://github.com/actions/cache/blob/main/examples.md#python---pip
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+#      - name: "Test config setup"
+#        shell: bash
+#        env:
+#          ADS_OCI_CONFIG_REALLIKE: ${{ secrets.ADS_OCI_CONFIG_REALLIKE }}
+#          ADS_OCI_PRIVATE_KEY_REALLIKE: ${{ secrets.ADS_OCI_PRIVATE_KEY_REALLIKE }}
+#        run: |
+#          set -x # print commands that are executed
+#          mkdir -p /home/runner/.oci
+#          ls -lha
+#          echo "$ADS_OCI_CONFIG_REALLIKE" > "/home/runner/.oci/config"
+#          echo "$ADS_OCI_PRIVATE_KEY_REALLIKE" > "/home/runner/.oci/oci_ads_user.pem"
+#          echo "key_file=/home/runner/.oci/oci_ads_user.pem" >> "/home/runner/.oci/config"
+
+#      - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
+#        timeout-minutes: 15
+#        shell: bash
+#        env:
+#          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
+#          NoDependency: True
+#        run: |
+#          set -x # print commands that are executed
+#          $CONDA/bin/conda init
+#          source /home/runner/.bashrc
+#          pip install -r test-requirements.txt
+#          python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup

.github/workflows/run-unittests.yml

@@ -0,0 +1,94 @@
+name: Unit Tests unitary
+
+on:
+  workflow_dispatch:
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}, ${{ matrix.test-path }}, ignore ${{ matrix.ignore-path }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+        test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
+        include:
+          - test-path: "tests/unitary"
+            ignore-path: "tests/unitary/with_extras/model"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Caching python libraries installed with pip
+      # https://github.com/actions/cache/blob/main/examples.md#python---pip
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+#      - name: "Test config setup"
+#        shell: bash
+#        env:
+#          ADS_OCI_CONFIG_REALLIKE: ${{ secrets.ADS_OCI_CONFIG_REALLIKE }}
+#          ADS_OCI_PRIVATE_KEY_REALLIKE: ${{ secrets.ADS_OCI_PRIVATE_KEY_REALLIKE }}
+#        run: |
+#          set -x # print commands that are executed
+#          mkdir -p /home/runner/.oci
+#          ls -lha
+#          echo "$ADS_OCI_CONFIG_REALLIKE" > "/home/runner/.oci/config"
+#          echo "$ADS_OCI_PRIVATE_KEY_REALLIKE" > "/home/runner/.oci/oci_ads_user.pem"
+#          echo "key_file=/home/runner/.oci/oci_ads_user.pem" >> "/home/runner/.oci/config"
+
+#      - name: "Test env setup"
+#        timeout-minutes: 20
+#        shell: bash
+#        run: |
+#          set -x # print commands that are executed
+#          sudo apt-get install libkrb5-dev graphviz
+#          $CONDA/bin/conda init
+#          source /home/runner/.bashrc
+#          pip install setuptools
+#          pip install -r dev-requirements.txt
+
+#      - name: "Run unitary tests folder with maximum ADS dependencies"
+#        timeout-minutes: 60
+#        shell: bash
+#        env:
+#          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
+#          CONDA_PREFIX: /usr/share/miniconda
+#        run: |
+#          python -m pytest -v -p no:warnings --durations=5 \
+#          ${{ matrix.test-path }} \
+#          --ignore "${{ matrix.ignore-path }}" \
+#          --cov --cov-append --cov-report=html
+
+#       Uploading test artifacts
+#       https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts#uploading-build-and-test-artifacts
+#      - name: "Save html coverage report"
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: code-coverage-report
+#          path: htmlcov/
+#          retention-days: 10

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI](https://img.shields.io/pypi/v/oracle-ads.svg)](https://pypi.org/project/oracle-ads/) [![Python](https://img.shields.io/pypi/pyversions/oracle-ads.svg?style=plastic)](https://pypi.org/project/oracle-ads/)
 
-The [Oracle Accelerated Data Science (ADS) SDK](https://accelerated-data-science.readthedocs.io/en/latest/index.html) is maintained by the Oracle Cloud Infrastructure (OCI) [Data Science service](https://docs.oracle.com/en-us/iaas/data-science/using/data-science.htm) team. It speeds up common data science activities by providing tools that automate and simplify common data science tasks. Additionally, provides data scientists a friendly pythonic interface to OCI services. Some of the more notable services are OCI Data Science, Model Catalog, Model Deployment, Jobs, Data Flow, Object Storage, Vault, Big Data Service, Data Catalog, and the Autonomous Database. ADS gives you an interface to manage the life cycle of machine learning models, from data acquisition to model evaluation, interpretation, and model deployment.
+The [Oracle Accelerated Data Science (ADS) SDK](https://accelerated-data-science.readthedocs.io/en/latest/index.html) is maintained by the Oracle Cloud Infrastructure (OCI) [Data Science service](https://docs.oracle.com/en-us/iaas/data-science/using/data-science.htm) team. It speeds up common data science activities by providing tools that automate and simplify common data science tasks. Additionally, provides data scientists a friendly pythonic interface to OCI services. Some of the more notable services are OCI Data Science, Model Catalog, Model Deployment, Jobs, ML Pipelines, Data Flow, Object Storage, Vault, Big Data Service, Data Catalog, and the Autonomous Database. ADS gives you an interface to manage the life cycle of machine learning models, from data acquisition to model evaluation, interpretation, and model deployment.
 
 With ADS you can:
 
@@ -14,6 +14,7 @@ With ADS you can:
  - Deploy models as HTTP endpoints with [Model Deployment](https://docs.oracle.com/en-us/iaas/data-science/using/model-dep-about.htm).
  - Launch distributed ETL, data processing, and model training jobs in Spark with [OCI Data Flow](https://docs.oracle.com/en-us/iaas/data-flow/using/home.htm).
  - Train machine learning models in OCI Data Science [Jobs](https://docs.oracle.com/en-us/iaas/data-science/using/jobs-about.htm).
+ - Define and run an end-to-end machine learning orchestration covering all the steps of machine learning lifecycle in a repeatable, continuous [ML Pipelines](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/pipeline/overview.html#).
  - Manage the life cycle of conda environments through the `ads conda` command line interface (CLI).
 
 ## Installation
@@ -166,7 +167,7 @@ This example uses SQL injection safe binding variables.
 
 ## Contributing
 
-This project welcomes contributions from the community. Before submitting a pull request, please review our contribution guide [CONTRIBUTING.md](https://github.com/oracle/accelerated-data-science/blob/main/CONTRIBUTING.md).
+This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./CONTRIBUTING.md)
 
 Find Getting Started instructions for developers in [README-development.md](https://github.com/oracle/accelerated-data-science/blob/main/README-development.md)
 

SECURITY.md

@@ -6,11 +6,10 @@ and privacy of all our users.
 
 Please do NOT raise a GitHub Issue to report a security vulnerability. If you
 believe you have found a security vulnerability, please submit a report to
-[secalert_us@oracle.com](mailto:secalert_us@oracle.com) preferably with a proof of concept. 
-Please review some additional information on 
-[how to report security vulnerabilities to Oracle](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html).
+[secalert_us@oracle.com][1] preferably with a proof of concept. Please review
+some additional information on [how to report security vulnerabilities to Oracle][2].
 We encourage people who contact Oracle Security to use email encryption using
-[our encryption key](https://www.oracle.com/security-alerts/encryptionkey.html).
+[our encryption key][3].
 
 We ask that you do not use other channels or contact the project maintainers
 directly.
@@ -22,15 +21,18 @@ security features are welcome on GitHub Issues.
 
 Security updates will be released on a regular cadence. Many of our projects
 will typically release security fixes in conjunction with the
-[Oracle Critical Patch Update](https://www.oracle.com/security-alerts/encryptionkey.html) program. 
-Security updates are released on the Tuesday closest to the 17th day of January, April, July and October. 
-A pre-release announcement will be published on the Thursday preceding each release. Additional
-information, including past advisories, is available on our 
-[security alerts](https://www.oracle.com/security-alerts/) page.
+[Oracle Critical Patch Update][3] program. Additional
+information, including past advisories, is available on our [security alerts][4]
+page.
 
 ## Security-related information
 
 We will provide security related information such as a threat model, considerations
 for secure use, or any known security issues in our documentation. Please note
 that labs and sample code are intended to demonstrate a concept and may not be
 sufficiently hardened for production use.
+
+[1]: mailto:secalert_us@oracle.com
+[2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html
+[3]: https://www.oracle.com/security-alerts/encryptionkey.html
+[4]: https://www.oracle.com/security-alerts/

ads/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from __future__ import print_function, division, absolute_import
@@ -26,6 +26,7 @@
 from ads.feature_engineering.accessor.dataframe_accessor import ADSDataFrameAccessor
 from ads.common import auth
 from ads.common.auth import set_auth
+from ads.common.config import Config
 
 os.environ["GIT_PYTHON_REFRESH"] = "quiet"
 

ads/ads_version.json

@@ -1,3 +1,3 @@
 {
-  "version": "2.8.0"
+  "version": "2.8.3"
 }

ads/automl/driver.py

@@ -1,16 +1,19 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from __future__ import absolute_import, division, print_function
 
+import copy
+
 import numpy as np
 import pandas as pd
 import scipy
-import copy
+
 from ads.common import logger, utils
+from ads.common.decorator.deprecate import deprecated
 from ads.common.model import ADSModel
 from ads.dataset import helper
 from ads.dataset.classification_dataset import (
@@ -37,6 +40,10 @@
 }
 
 
+@deprecated(
+    details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+    raise_error=True,
+)
 def get_ml_task_type(X, y, classes):
     """
     Gets the ML task type and returns it.
@@ -79,6 +86,10 @@ def get_ml_task_type(X, y, classes):
 
 
 class AutoML:
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(
         self,
         training_data,
@@ -87,7 +98,7 @@ def __init__(
         baseline="dummy",
         client=None,
     ):
-        r"""
+        """
         Creates an Automatic machine learning object.
 
         Parameters

ads/automl/provider.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
@@ -41,6 +41,10 @@ class AutoMLProvider(ABC):
     implement train() and get_transformer_pipeline().
     """
 
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(self):
         self.X_train = None
         self.y_train = None
@@ -143,6 +147,10 @@ class BaselineModel(object):
     interface. Labels (y) are encoded using DataFrameLabelEncoder.
     """
 
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(self, est):
         self.est = est
         self.df_label_encoder = DataFrameLabelEncoder()
@@ -250,6 +258,10 @@ def get_transformer_pipeline(self):
         msg = "Baseline"
         return [("automl_preprocessing", AutoMLPreprocessingTransformer(msg))]
 
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(self, est):
         """
         Generates a baseline model using the Zero Rule algorithm by default. For a classification
@@ -309,6 +321,7 @@ class OracleAutoMLProvider(AutoMLProvider, ABC):
     @deprecated(
         "2.6.7",
         details="Oracle AutoML is recommended to be directly instantiated by importing automlx package",
+        raise_error=True,
     )
     def __init__(
         self, n_jobs=-1, loglevel=None, logger_override=None, model_n_jobs: int = 1
@@ -870,6 +883,10 @@ def visualize_tuning_trials(self, ylabel=None):
 
 
 class AutoMLPreprocessingTransformer(object):  # pragma: no cover
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(self, msg):
         self.msg = msg
 
@@ -915,6 +932,10 @@ def __repr__(self):
 
 
 class AutoMLFeatureSelection(object):  # pragma: no cover
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
     def __init__(self, msg):
         self.msg = msg