Make sycl optional

.github/workflows/build_and_run.yml

@@ -40,7 +40,7 @@ jobs:
           python-version: ${{ matrix.python }}
           mamba-version: "*"
           activate-environment: dpbench-dev
-          environment-file: dpbench-env-linux.yml
+          environment-file: environments/conda-linux-sycl.yml
 
       - name: Conda info
         shell: bash -l {0}
@@ -51,7 +51,8 @@ jobs:
       - name: Build dpbench
         shell: bash -l {0}
         run: |
-          CC=icx CXX=icpx python setup.py develop -- -Dpybind11_DIR=$(python -m pybind11 --cmakedir) -DDPCTL_MODULE_PATH=$(python -m dpctl --cmakedir)
+          CC=icx CXX=icpx DPBENCH_SYCL=1 pip install \
+          --no-index --no-deps --no-build-isolation -e . -v
 
       - name: Run benchmarks
         shell: bash -l {0}

README.md

@@ -9,100 +9,145 @@ SPDX-License-Identifier: Apache-2.0
 
 # DPBench - Benchmarks to evaluate Data-Parallel Extensions for Python
 
-* __*_numba_*.py__ : This file contains Numba implementations of the benchmarks. There are three modes: nopython-mode, nopython-mode-parallel and nopython-mode-parallel-range.
-* __*_numba_dpex_*.py__ : This file contains Numba-Dpex implementations of the benchmarks. There are three modes: kernel-mode, numpy-mode and prange-mode.
-* __*_dpnp_*.py__ : This file contains dpnp implementations of the benchmarks.
-* __*_native_ext/_sycl/__kernel_*.hpp__ : This file contains native dpcpp implementations of the benchmarks.
+* **\<benchmark\>\_numba\_\<mode\>.py** : This file contains Numba implementations of the benchmarks. There are three modes: nopython-mode, nopython-mode-parallel and nopython-mode-parallel-range.
+* **\<benchmark\>\_numba_dpex\_\<mode\>.py** : This file contains Numba-Dpex implementations of the benchmarks. There are three modes: kernel-mode, numpy-mode and prange-mode.
+* **\<benchmark\>\_dpnp\_\<mode\>.py** : This file contains dpnp implementations of the benchmarks.
+* **\<benchmark\>\_native_ext/\<benchmark\>\_sycl/_\<benchmark\>_kernel.hpp** : This file contains native dpcpp implementations of the benchmarks.
+* **\<benchmark\>\_numpy.py** : This file contains numpy implementations of the benchmarks. It should take benefits of numpy arrays and should avoid loops over arrays.
+* **\<benchmark\>\_python.py** : This file contains naive python implementations of the benchmarks. Should be run only for small presets, otherwise it will take long execution time.
+* **\<benchmark\>\_numba_mlir\_\<mode\>.py** : This file contains Numba-MLIR implementations of the benchmarks. There are three modes: kernel-mode, numpy-mode and prange-mode. Experimental.
 
 ## Examples of setting up and running the benchmarks
-1. Setting up conda environment and installing dependencies:
-
-        $ conda create -n dpbench-dev
-        $ conda activate dpbench-dev
-        $ conda install python
-        $ conda install -c intel tbb dpcpp_linux-64
-        $ conda install numpy numba cython cmake ninja scikit-build pandas
-        $ conda install scipy scikit-learn pybind11 tomli
-        # do not miss the quotes!
-        $ conda install -c pkgs/main libgcc-ng">=11.2.0" libstdcxx-ng">=11.2.0" libgomp">=11.2.0"
-        $ conda install -c dppy/label/dev -c intel -c main dpctl numba-dpex dpnp
-        $ pip install alembic
-
-2. Build and run DPBench
-    - To build:
-        ```bash
-        $  CC=icx CXX=icpx python setup.py develop -- -Dpybind11_DIR=$(python -m pybind11 --cmakedir) -DDPCTL_MODULE_PATH=$(python -m dpctl --cmakedir)
-        ```
-    - To run, taking black_scholes for example:
-        ```bash
-        $  dpbench -b black_scholes run
-        ```
-    - Similarly, to run all the cases in DPBench:
-        ```bash
-        $  dpbench -a run
-        ```
+1. Clone the repository
 
-3. Device Customization
-
-   If a framework is SYCL based, an extra configuration option `sycl_device` may be set in the
-   framework JSON file to control what device the framework uses for execution. The `sycl_device`
-   value should be a legal
-   [SYCL device filter ](https://intel.github.io/llvm-docs/EnvironmentVariables.html#sycl_device_filter)
-   string. The dpcpp, dpnp, and numba_dpex frameworks support the sycl_device option.
+    ```bash
+    git clone https://github.com/IntelPython/dpbench
+    cd dpbench
+    ```
 
-   Here is an example:
+2. Setting up conda environment and installing dependencies:
 
-    ```json
-        {
-            "framework": {
-                "simple_name": "dpcpp",
-                "full_name": "dpcpp",
-                "prefix": "dp",
-                "postfix": "dpcpp",
-                "class": "DpcppFramework",
-                "arch": "gpu",
-                "sycl_device": "level_zero:gpu:0"
-            }
-        }
+    ```bash
+    conda env create -n dpbench -f ./environments/conda.yml
     ```
 
-    > **_NOTE:_**  The `arch` option is deprecated and not used by dpbench.
+    - SYCL
 
-   To run with customized framework JSON file, pass it as an argument to the `run_benchmark` or
-   `run_benchmarks` functions.
+        If you want to build sycl benchmarks as well:
+        ```bash
+        conda env create -n dpbench -f ./environments/conda-linux-sycl.yml
+        CC=icx CXX=icpx DPBENCH_SYCL=1 pip install --no-index --no-deps --no-build-isolation -e . -v
+        ```
 
-   TODO: current way not working anymore.
+3. Build DPBench
 
     ```bash
-    $ python -c "import dpbench; dpbench.run_benchmark(\"black_scholes\", "<absolute path to json file>")"
+    pip install --no-index --no-deps --no-build-isolation -e . -v
     ```
 
-## Running numba-mlir benchmarks
-1. Setting up conda environment and installing dependencies:
+    Alternatively you can build it with `setup.py`, but pip version is preferable:
 
-    Use same instructions as for usual dpbench setup.
-
-    Install latest `numba-mlir` dev package:
+    ```bash
+    python setup.py develop
+    ```
 
-        $ conda install numba-mlir -c dppy/label/dev -c intel
+    - SYCL
 
-2. Build and run DPBench
+        For sycl build use:
+        ```bash
+        CC=icx CXX=icpx DPBENCH_SYCL=1 pip install --no-index --no-deps --no-build-isolation -e . -v
+        ```
 
-    Use same commands to setup and run dpbench:
+        or
 
         ```bash
-        $  dpbench -b black_scholes run
+        CC=icx CXX=icpx DPBENCH_SYCL=1 python setup.py develop
         ```
 
-    or, to run specific version:
+4. Run specific benchmark, e.g. black_scholes
 
+    ```bash
+    dpbench -b black_scholes run
+    ```
+5. Run all benchmarks
 
-        ```bash
-        $  dpbench -b black_scholes -i numba_mlir_k run
-        ```
+    ```bash
+    dpbench -a run
+    ```
 
-    to run all `numba-mlir` benchmarks:
+6. Device Customization
 
-        ```bash
-        $  dpbench -b black_scholes -i numba_mlir_k,numba_mlir_n,numba_mlir_p run
-        ```
+   If a framework is SYCL based, an extra configuration option `sycl_device` may be set in the
+   framework config file or by passing `--sycl-device` argument to `dpbench run` to control what device the framework uses for execution. The `sycl_device`
+   value should be a legal
+   [SYCL device filter ](https://intel.github.io/llvm-docs/EnvironmentVariables.html#sycl_device_filter)
+   string. The dpcpp, dpnp, and numba_dpex frameworks support the sycl_device option.
+
+   Here is an example:
+
+    ```json
+        dpbench -b black_scholes -i dpnp run --sycl-device=level_zero:gpu:0
+    ```
+
+7. All available options are available using `dpbench --help` and `dpbench <command> --help`:
+
+    ```
+    usage: dpbench [-h] [-b [BENCHMARKS]] [-i [IMPLEMENTATIONS]] [-a | --all-implementations | --no-all-implementations] [--version] [-r [RUN_ID]] [--last-run | --no-last-run]
+                [-d [RESULTS_DB]] [--log-level [{critical,fatal,error,warning,info,debug}]]
+                {run,report,config} ...
+
+    positional arguments:
+    {run,report,config}
+
+    options:
+    -h, --help            show this help message and exit
+    -b [BENCHMARKS], --benchmarks [BENCHMARKS]
+                            Comma separated list of benchmarks. Leave empty to load all benchmarks.
+    -i [IMPLEMENTATIONS], --implementations [IMPLEMENTATIONS]
+                            Comma separated list of implementations. Use --all-implementations to load all available implementations.
+    -a, --all-implementations, --no-all-implementations
+                            If set, all available implementations will be loaded. (default: False)
+    --version             show program's version number and exit
+    -r [RUN_ID], --run-id [RUN_ID]
+                            run_id to perform actions on. Use --last-run to use latest available run, or leave empty to create new one.
+    --last-run, --no-last-run
+                            Sets run_id to the latest run_id from the database. (default: False)
+    -d [RESULTS_DB], --results-db [RESULTS_DB]
+                            Path to a database to store results.
+    --log-level [{critical,fatal,error,warning,info,debug}]
+                            Log level.
+    ```
+
+    ```
+    usage: dpbench run [-h] [-p [{S,M,L}]] [-s | --validate | --no-validate] [--dpbench | --no-dpbench] [--npbench | --no-npbench] [--polybench | --no-polybench] [-r [REPEAT]] [-t [TIMEOUT]]
+                    [--precision [{single,double}]] [--print-results | --no-print-results] [--save | --no-save] [--sycl-device [SYCL_DEVICE]]
+                    [--skip-expected-failures | --no-skip-expected-failures]
+
+    Subcommand to run benchmark executions.
+
+    options:
+    -h, --help            show this help message and exit
+    -p [{S,M,L}], --preset [{S,M,L}]
+                            Preset to use for benchmark execution.
+    -s, --validate, --no-validate
+                            Set if the validation will be run for each benchmark. (default: True)
+    --dpbench, --no-dpbench
+                            Set if run dpbench benchmarks. (default: True)
+    --npbench, --no-npbench
+                            Set if run npbench benchmarks. (default: False)
+    --polybench, --no-polybench
+                            Set if run polybench benchmarks. (default: False)
+    -r [REPEAT], --repeat [REPEAT]
+                            Number of repeats for each benchmark.
+    -t [TIMEOUT], --timeout [TIMEOUT]
+                            Timeout time in seconds for each benchmark execution.
+    --precision [{single,double}]
+                            Data precision to use for array initialization.
+    --print-results, --no-print-results
+                            Show the result summary or not (default: True)
+    --save, --no-save     Either to save execution into database. (default: True)
+    --sycl-device [SYCL_DEVICE]
+                            Sycl device to overwrite for framework configurations.
+    --skip-expected-failures, --no-skip-expected-failures
+                            Either to save execution into database. (default: True)
+    ```

dpbench/config/reader.py

@@ -356,7 +356,11 @@ def read_benchmark_implementations(
                 if hasattr(impl_mod, func):
                     func_name = func
                     break
-        except Exception:
+        except Exception as e:
+            logging.warn(f"Could not import module: {e}")
+            import traceback
+
+            traceback.print_exc()
             continue
 
         config.implementations.append(

dpbench/console/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Console tools."""

dpbench/console/config.py

@@ -5,6 +5,7 @@
 """Report subcommand package."""
 
 import argparse
+import logging
 
 from ._namespace import Namespace
 
@@ -33,6 +34,8 @@ def execute_config(args: Namespace):
         args: object with all input arguments.
         conn: database connection.
     """
+    import importlib
+
     import dpbench.config as cfg
 
     cfg.GLOBAL = cfg.read_configs(
@@ -42,7 +45,15 @@ def execute_config(args: Namespace):
         with_polybench=True,
     )
 
-    if args.color:
+    color_output = args.color
+
+    if color_output and not importlib.util.find_spec("textwrap"):
+        logging.warn(
+            "pygments not found. If you want color output - install it using pip/conda etc"
+        )
+        color_output = False
+
+    if color_output:
         from pprint import pformat
 
         from pygments import highlight

dpbench/console/run.py

@@ -136,9 +136,9 @@ def execute_run(args: Namespace, conn: sqlalchemy.Engine):
     )
 
     if args.all_implementations:
-        args.implementations = {
+        args.implementations = [
             impl.postfix for impl in cfg.GLOBAL.implementations
-        }
+        ]
 
     if args.sycl_device:
         for framework in cfg.GLOBAL.frameworks:

dpbench/migrations/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0

environments/conda-linux-sycl.yml

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: 2022 - 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: dpbench-dev
+channels:
+  - dppy/label/dev
+  - intel
+  - conda-forge
+  - nodefaults
+dependencies:
+  - conda-forge::pip
+  - conda-forge::tomli
+  - conda-forge::alembic>=1.11.0
+  - conda-forge::sqlalchemy>=2.0.0
+  - conda-forge::scipy
+  - conda-forge::scikit-learn
+  - conda-forge::pandas
+  - numpy
+  - numba
+  - dpctl
+  - dpnp
+  - numba-dpex
+  - numba-mlir
+  - dpcpp_linux-64
+  - cython
+  - cmake>=3.22
+  - ninja
+  - scikit-build
+  # https://github.com/scikit-build/scikit-build/issues/981
+  - setuptools>=42,<64
+  - pybind11
+  - conda-forge::libgcc-ng
+  - conda-forge::libstdcxx-ng
+  - conda-forge::libgomp

dpbench-env-linux.yml → environments/conda.yml

@@ -7,27 +7,18 @@ channels:
   - dppy/label/dev
   - intel
   - conda-forge
+  - nodefaults
 dependencies:
-  - tbb
-  - dpcpp_linux-64
+  - conda-forge::pip
+  - conda-forge::tomli
+  - conda-forge::alembic>=1.11.0
+  - conda-forge::sqlalchemy>=2.0.0
+  - conda-forge::scipy
+  - conda-forge::scikit-learn
+  - conda-forge::pandas
   - numpy
   - numba
-  - cython
-  - cmake>=3.22
-  - ninja
-  - scikit-build
-  - pandas
-  - scipy
-  - scikit-learn
-  - pybind11
-  - tomli
   - dpctl
-  - numba-dpex
   - dpnp
+  - numba-dpex
   - numba-mlir
-  - libgcc-ng>=11.2.0
-  - libstdcxx-ng>=11.2.0
-  - libgomp>=11.2.0
-
-  - pip:
-    - alembic