Support Python 3.11

- initial commit - moving some code from ODSC-54710/python311_312_support branch
- removed dask from dependencies and tests

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
+name: "[Py3.8-3.12] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -60,5 +60,8 @@ jobs:
           source /home/runner/.bashrc
           conda install python=${{ matrix.python-version }}
           pip install -r test-requirements.txt
+          if [[ ${{ matrix.python-version }} == "3.12" ]]; then
+            pip install setuptools
+          fi
           conda list
           python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup

.github/workflows/run-unittests-py38-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] tests/unitary/**"
+name: "[Py3.8][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:

.github/workflows/run-unittests-py39-py310.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9][Py3.10] - tests/unitary/**"
+name: "[Py3.9-3.12] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"

dev-requirements.txt

@@ -1,3 +1,5 @@
+# Do not add test dependencies here. Use pyproject.toml [project.optional-dependencies] 'testsuite' section.
+# Reason - it is flexible to specify different version for specific python
 -r test-requirements.txt
 -e ".[aqua,bds,data,geo,huggingface,llm,notebook,onnx,opctl,optuna,spark,tensorflow,text,torch,viz]"
 -e ".[testsuite]"

pyproject.toml

@@ -48,6 +48,7 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
 ]
 
 # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/
@@ -79,7 +80,7 @@ dependencies = [
 # Copied from extras_require list in setup.py, setup.py got removed in favor of this config file
 bds = ["hdfs[kerberos]", "ibis-framework[impala]", "sqlalchemy"]
 boosted = [
-  "lightgbm<4.0.0", # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm",
   "xgboost",
 ]
 data = [
@@ -95,12 +96,12 @@ geo = ["geopandas", "oracle_ads[viz]"]
 huggingface = ["transformers"]
 notebook = ["ipython>=7.23.1, <8.0", "ipywidgets~=7.6.3"]
 onnx = [
-  "lightgbm<4.0.0",            # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm",
   "onnx>=1.12.0",
   "onnxmltools>=1.10.0",
-  "onnxruntime>=1.10.0,<1.16", # v1.16 introduced issues https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime>=1.10.0,!=1.16.0", # avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
   "oracle_ads[viz]",
-  "protobuf<=3.20",
+  "protobuf",
   "skl2onnx>=1.10.4",
   "tf2onnx",
   "xgboost<=1.7",
@@ -120,7 +121,10 @@ opctl = [
 optuna = ["optuna==2.9.0", "oracle_ads[viz]"]
 spark = ["pyspark>=3.0.0"]
 tensorflow = ["oracle_ads[viz]", "tensorflow"]
-text = ["spacy", "wordcloud>=1.8.1"]
+text = [
+  "spacy>=3.4.2",   # the first version of spacy that supports python 3.11 is spacy v3.4.2
+  "wordcloud>=1.8.1"
+]
 torch = ["oracle_ads[viz]", "torch", "torchvision"]
 viz = [
   "bokeh>=3.0.0, <3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
@@ -182,27 +186,24 @@ aqua = ["jupyter_server"]
 # trying to use. Ref - https://pip.pypa.io/en/stable/topics/dependency-resolution/#possible-ways-to-reduce-backtracking.
 # Revisit this section continuously and update to recent version of libraries. focus on pyt3.9/3.10 versions.
 testsuite = [
-  "dask==2023.5.0; python_version=='3.8'",
-  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
   "arff",
   "category_encoders==2.6.3",  # set version to avoid backtracking
   "cohere==4.53",  # set version to avoid backtracking
-  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
-  "dask==2023.5.0; python_version=='3.8'",
   "faiss-cpu",
   "fastparquet==2024.2.0",  # set version to avoid backtracking
   "imbalanced-learn",
+  "kubernetes",
   "lxml",
   "mysql-connector-python",
   "nltk",
   "notebook==6.4.12",
   "opensearch-py",
   "pdfplumber",
   "py4j",
-  "pyarrow",
+  "pyarrow>=15.0.0",
   "statsmodels; python_version=='3.8'",
   "statsmodels>=0.14.1; python_version>='3.9'",  # cython3.0 compatibility added in v0.14.1
-  "tables",
+  "tables>3.9.0",
   "xlrd>=1.2.0",
 ]
 

test-requirements.txt

@@ -1,3 +1,5 @@
+# Include here only libraries required to run test methods, like mock, pytest, coverage etc.
+# Add dependencies for ADS to pyproject.toml [project.optional-dependencies] 'testsuite' section.
 -e .
 click
 coverage
@@ -7,5 +9,4 @@ parameterized
 pip
 pytest
 pytest-cov
-pytest-xdist
-kubernetes
+pytest-xdist

tests/unitary/default_setup/common/test_common_ADSData.py

@@ -149,22 +149,6 @@ def test_ADSData_build_valid_input_using_target_vector(self):
         assert expected.y.name == "target"
         assert expected.y.shape == (3,)
 
-    @pytest.mark.skipif("NoDependency" in os.environ, reason="skip for dependency test")
-    def test_ADSData_build_valid_input_dask_dataframe(self):
-        """
-        Ensures build method takes dask dataframe
-        """
-        import dask
-
-        X = dask.datasets.timeseries().drop("y", axis=1)
-        y = dask.datasets.timeseries()["y"]
-        expected = ADSData.build(X, y)
-        assert sorted(expected.X.columns.tolist()) == sorted(["id", "name", "x"])
-        assert expected.X.shape[0] == 2592000
-        assert expected.X.shape[1] == 3
-        assert expected.y.name == "y"
-        assert expected.y.shape[0] == 2592000
-
     @pytest.mark.skip("api change. this test should be re-written.")
     def test_ADSData_build_with_data(self):
         """