Resolve issue #2018

.github/actions/azureml-test/action.yml

@@ -69,7 +69,7 @@ runs:
   using: "composite"
   steps:
     - name: Setup python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.8"
     - name: Install azureml-core and azure-cli on a GitHub hosted server
@@ -82,43 +82,32 @@ runs:
     - name: Install wheel package
       shell: bash
       run: pip install --quiet wheel
-    - name: Submit CPU tests to AzureML
+    - name: Submit tests to AzureML
       shell: bash
-      if: contains(inputs.TEST_GROUP, 'cpu')
       run: >-
-          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
-          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
-          --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}}
-          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
-          --disable-warnings --sha "${GITHUB_SHA}"
-    - name: Submit GPU tests to AzureML
-      shell: bash
-      if: contains(inputs.TEST_GROUP, 'gpu')
-      run: >-
-          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
-          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
-          --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}}
-          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
-          --disable-warnings --sha "${GITHUB_SHA}"
-    - name: Submit PySpark tests to AzureML
-      shell: bash
-      if: contains(inputs.TEST_GROUP, 'spark')
-      run: >-
-          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
-          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }}
-          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
-          --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}}
-          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
-          --disable-warnings --sha "${GITHUB_SHA}"
+        python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py \
+          --subid ${{inputs.AZUREML_TEST_SUBID}} \
+          --reponame "recommenders" \
+          --branch ${{ github.ref }} \
+          --rg ${{inputs.RG}} \
+          --wsname ${{inputs.WS}} \
+          --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} \
+          --testlogs ${{inputs.TEST_LOGS_PATH}} \
+          --testkind ${{inputs.TEST_KIND}} \
+          --conda_pkg_python ${{inputs.PYTHON_VERSION}} \
+          --testgroup ${{inputs.TEST_GROUP}} \
+          --disable-warnings \
+          --sha "${GITHUB_SHA}" \
+          --clustername $(if [[ ${{inputs.TEST_GROUP}} =~ "gpu" ]]; then echo "${{inputs.GPU_CLUSTER_NAME}}"; else echo "${{inputs.CPU_CLUSTER_NAME}}"; fi) \
+          $(if [[ ${{inputs.TEST_GROUP}} =~ "gpu" ]]; then echo "--add_gpu_dependencies"; fi) \
+          $(if [[ ${{inputs.TEST_GROUP}} =~ "spark" ]]; then echo "--add_spark_dependencies"; fi)
     - name: Get exit status
       shell: bash
       id: exit_status
       run: echo "code=$(cat ${{inputs.PYTEST_EXIT_CODE}})" >> $GITHUB_OUTPUT
     - name: Check Success/Failure
       if: ${{ steps.exit_status.outputs.code != 0 }}
-      uses: actions/github-script@v3
+      uses: actions/github-script@v7
       with:
         script: |
             core.setFailed('All tests did not pass!')

.github/workflows/azureml-cpu-nightly.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get test group names
         id: get_test_groups
         uses: ./.github/actions/get-test-groups
@@ -71,7 +71,7 @@ jobs:
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Execute tests
         uses: ./.github/actions/azureml-test
         id: execute_tests

.github/workflows/azureml-gpu-nightly.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get test group names
         id: get_test_groups
         uses: ./.github/actions/get-test-groups
@@ -71,7 +71,7 @@ jobs:
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Execute tests
         uses: ./.github/actions/azureml-test
         id: execute_tests

.github/workflows/azureml-release-pipeline.yml

@@ -33,9 +33,9 @@ jobs:
     needs: [unit-test-workflow, cpu-nightly-workflow, gpu-nightly-workflow, spark-nightly-workflow]
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with: 
           python-version: "3.8"
       - name: Install wheel package

.github/workflows/azureml-spark-nightly.yml

@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get test group names
         id: get_test_groups
         uses: ./.github/actions/get-test-groups
@@ -70,7 +70,7 @@ jobs:
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Execute tests
         uses: ./.github/actions/azureml-test
         id: execute_tests

.github/workflows/azureml-unit-tests.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get test group names
         id: get_test_groups
         uses: ./.github/actions/get-test-groups
@@ -60,7 +60,7 @@ jobs:
         test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Execute tests
         uses: ./.github/actions/azureml-test
         id: execute_tests

.github/workflows/sarplus.yml

@@ -41,10 +41,10 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -96,15 +96,15 @@ jobs:
       - name: Upload Python wheel as GitHub artifact when merged into main
         # Upload the whl file of the specific python version
         if: github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: pysarplus-${{ env.sarplus_version }}-cp${{ matrix.python-version }}-wheel
           path: ${{ env.PYTHON_ROOT }}/dist/*.whl
 
       - name: Upload Python source as GitHub artifact when merged into main
         # Only one pysarplus source tar file is needed
         if: github.ref == 'refs/heads/main' && matrix.python-version == '3.10'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: pysarplus-${{ env.sarplus_version }}-source
           path: ${{ env.PYTHON_ROOT }}/dist/*.tar.gz
@@ -131,7 +131,7 @@ jobs:
             hadoop-version: "3.3.1"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Test
         run: |
@@ -180,7 +180,7 @@ jobs:
 
       - name: Upload Scala bundle as GitHub artifact when merged into main
         if: github.ref == 'refs/heads/main'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: sarplus-${{ env.sarplus_version }}-bundle_2.12-spark-${{ matrix.spark-version }}-jar
           path: ${{ env.SCALA_ROOT }}/target/scala-2.12/*bundle*.jar

.github/workflows/update_documentation.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.10
 

recommenders/models/deeprec/models/sequential/rnn_cell_implement.py

@@ -601,9 +601,9 @@ def __init__(
     ):
         self._build_bias = build_bias
 
-        if args is None or (nest.is_sequence(args) and not args):
+        if args is None or (nest.is_nested(args) and not args):
             raise ValueError("`args` must be specified")
-        if not nest.is_sequence(args):
+        if not nest.is_nested(args):
             args = [args]
             self._is_sequence = False
         else:

setup.py

@@ -58,7 +58,7 @@
 extras_require = {
     "gpu": [
         "nvidia-ml-py3>=7.352.0",
-        "tensorflow==2.8.4",  # FIXME: Temporarily pinned due to issue with TF version > 2.10.1 See #2018
+        "tensorflow>=2.8.4,!=2.9.0.*,!=2.9.1,!=2.9.2,!=2.10.0.*,<3",
         "tf-slim>=1.1.0",
         "torch>=1.13.1",  # for CUDA 11 support
         "fastai>=1.0.46,<2",
@@ -72,6 +72,7 @@
         "pytest>=3.6.4",
         "pytest-cov>=2.12.1",
         "pytest-mock>=3.6.1",  # for access to mock fixtures in pytest
+        "packaging>=20.9",     # for version comparison in test_dependency_security.py
     ],
 }
 # For the brave of heart

tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py

@@ -37,7 +37,6 @@
 """
 import argparse
 import logging
-import glob
 
 from azureml.core.authentication import AzureCliAuthentication
 from azureml.core import Workspace
@@ -146,7 +145,6 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes)
 
 def create_run_config(
     cpu_cluster,
-    docker_proc_type,
     add_gpu_dependencies,
     add_spark_dependencies,
     conda_pkg_jdk,
@@ -165,7 +163,6 @@ def create_run_config(
                                                 the following:
                                                 - Reco_cpu_test
                                                 - Reco_gpu_test
-            docker_proc_type (str)          : processor type, cpu or gpu
             add_gpu_dependencies (bool)     : True if gpu packages should be
                                         added to the conda environment, else False
             add_spark_dependencies (bool)   : True if PySpark packages should be
@@ -179,7 +176,39 @@ def create_run_config(
     run_azuremlcompute = RunConfiguration()
     run_azuremlcompute.target = cpu_cluster
     run_azuremlcompute.environment.docker.enabled = True
-    run_azuremlcompute.environment.docker.base_image = docker_proc_type
+    if not add_gpu_dependencies:
+        # https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04
+        run_azuremlcompute.environment.docker.base_image = "mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04"
+    else:
+        run_azuremlcompute.environment.docker.base_image = None
+        # Use the latest CUDA
+        # See
+        # * https://learn.microsoft.com/en-us/azure/machine-learning/how-to-train-with-custom-image?view=azureml-api-1#use-a-custom-dockerfile-optional
+        # * https://github.com/Azure/AzureML-Containers/blob/master/base/gpu/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04
+        run_azuremlcompute.environment.docker.base_dockerfile = r"""
+FROM nvcr.io/nvidia/cuda:12.3.1-devel-ubuntu22.04
+USER root:root
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && \
+    apt-get install -y wget git-all && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+# Conda Environment
+ENV MINICONDA_VERSION py38_23.3.1-0
+ENV PATH /opt/miniconda/bin:$PATH
+ENV CONDA_PACKAGE 23.5.0
+RUN wget -qO /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -bf -p /opt/miniconda && \
+    conda install conda=${CONDA_PACKAGE} -y && \
+    conda update --all -c conda-forge -y && \
+    conda clean -ay && \
+    rm -rf /opt/miniconda/pkgs && \
+    rm /tmp/miniconda.sh && \
+    find / -type d -name __pycache__ | xargs rm -rf
+"""
 
     # Use conda_dependencies.yml to create a conda environment in
     # the Docker image for execution
@@ -195,6 +224,7 @@ def create_run_config(
 
     # install recommenders
     reco_extras = "dev"
+    conda_dep.add_conda_package("anaconda::git")
     if add_gpu_dependencies and add_spark_dependencies:
         conda_dep.add_channel("conda-forge")
         conda_dep.add_conda_package(conda_pkg_jdk)
@@ -326,13 +356,6 @@ def create_arg_parser():
         default="STANDARD_D3_V2",
         help="Set the size of the VM either STANDARD_D3_V2",
     )
-    # cpu or gpu
-    parser.add_argument(
-        "--dockerproc",
-        action="store",
-        default="cpu",
-        help="Base image used in docker container",
-    )
     # Azure subscription id, when used in a pipeline, it is stored in keyvault
     parser.add_argument(
         "--subid", action="store", default="123456", help="Azure Subscription ID"
@@ -421,16 +444,6 @@ def create_arg_parser():
 
     logger = logging.getLogger("submit_groupwise_azureml_pytest.py")
     args = create_arg_parser()
-
-    if args.dockerproc == "cpu":
-        from azureml.core.runconfig import DEFAULT_CPU_IMAGE
-
-        docker_proc_type = DEFAULT_CPU_IMAGE
-    else:
-        from azureml.core.runconfig import DEFAULT_GPU_IMAGE
-
-        docker_proc_type = DEFAULT_GPU_IMAGE
-
     cli_auth = AzureCliAuthentication()
 
     workspace = setup_workspace(
@@ -450,7 +463,6 @@ def create_arg_parser():
 
     run_config = create_run_config(
         cpu_cluster=cpu_cluster,
-        docker_proc_type=docker_proc_type,
         add_gpu_dependencies=args.add_gpu_dependencies,
         add_spark_dependencies=args.add_spark_dependencies,
         conda_pkg_jdk=args.conda_pkg_jdk,