Skip to content

Commit f4d73ae

Browse files
icfaustdavid-cortes-intel
icfaust
authored and
david-cortes-intel
committed
[CI, enhancement] add pytorch+gpu testing ci (#2494)
* Update _dataframes_support.py * Update conftest.py * Update conftest.py * Update _dataframes_support.py * Update _dataframes_support.py * Update _dataframes_support.py * Update _dataframes_support.py * Update _dataframes_support.py * Update _dataframes_support.py * Update _dataframes_support.py * Update conftest.py * Update conftest.py * Update conftest.py * Update _dataframes_support.py * Update run_test.sh * Update conftest.py * Update run_test.sh * Update run_test.sh * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update run_sklearn_tests.sh * Update ci.yml * Update ci.yml * Update validation.py * Update validation.py * Update validation.py * Update validation.py * Update _device_offload.py * Update _data_conversion.py * Update __init__.py * Update run_sklearn_tests.sh * Update _device_offload.py * Update _data_conversion.py * Update validation.py * Update table.cpp * Update _device_offload.py * Update _device_offload.py * Update deselected_tests.yaml * Update deselected_tests.yaml * Update _device_offload.py * Update deselected_tests.yaml * Update ci.yml * Update ci.yml * Update deselected_tests.yaml * Update ci.yml * Update deselected_tests.yaml * Update validation.py * Update ci.yml * Update __init__.py * Update _data_conversion.py * Update _device_offload.py * Update ci.yml * Update _device_offload.py * Update validation.py
1 parent 9a1049b commit f4d73ae

File tree

7 files changed

+294
-58
lines changed

7 files changed

+294
-58
lines changed

‎.ci/scripts/run_sklearn_tests.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@ export DESELECT_FLAGS="--public ${DESELECT_FLAGS}"
2929
if [ -n "${SKLEARNEX_PREVIEW}" ]; then
3030
export DESELECT_FLAGS="--preview ${DESELECT_FLAGS}"
3131
fi
32-
export DESELECTED_TESTS=$(python ../.circleci/deselect_tests.py ../deselected_tests.yaml ${DESELECT_FLAGS})
32+
if [ "$1" == "gpu" ]; then
33+
export DESELECT_FLAGS="--gpu ${DESELECT_FLAGS}"
34+
fi
3335

36+
export DESELECTED_TESTS=$(python ../.circleci/deselect_tests.py ../deselected_tests.yaml ${DESELECT_FLAGS})
3437
# manual setting of OCL_ICD_FILENAMES is required in
3538
# specific MSYS environment with conda packages downloaded from intel channel
3639
if [[ "$(uname)" =~ "MSYS" ]] && [ -z "${OCL_ICD_FILENAMES}" ] && [ -n "${CONDA_PREFIX}" ]; then

‎.github/workflows/ci.yml

Lines changed: 218 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,35 @@ env:
3333
DPCTL_VERSION: 0.18.1
3434
DPNP_VERSION: 0.16.0
3535
DPCTL_PY_VERSIONS: '3.9\|3.11'
36+
UXL_PYTHONVERSION: "3.12"
37+
UXL_SKLEARNVERSION: "1.4"
38+
ONEDAL_REPO: "uxlfoundation/oneDAL"
3639

3740
jobs:
41+
42+
onedal_nightly:
43+
runs-on: ubuntu-24.04
44+
name: Identify oneDAL nightly
45+
timeout-minutes: 2
46+
47+
steps:
48+
- name: Get run ID of "Nightly-build" workflow
49+
id: get-run-id
50+
run: |
51+
WF_NAME="Nightly-build"
52+
JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId'
53+
RUN_ID=`gh run --repo ${{ env.ONEDAL_REPO }} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"`
54+
echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
55+
echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
56+
env:
57+
GH_TOKEN: ${{ github.token }}
58+
outputs:
59+
run-id: ${{ steps.get-run-id.outputs.run-id }}
60+
uxl-python: ${{ env.UXL_PYTHONVERSION }}
61+
uxl-sklearn: ${{ env.UXL_SKLEARNVERSION }}
62+
3863
sklearn_lnx:
64+
needs: onedal_nightly
3965
strategy:
4066
fail-fast: false
4167
matrix:
@@ -46,7 +72,7 @@ jobs:
4672
SKLEARN_VERSION: "1.2"
4773
- PYTHON_VERSION: "3.11"
4874
SKLEARN_VERSION: "1.3"
49-
name: LinuxNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
75+
name: LinuxNightly/venv Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
5076
runs-on: ubuntu-24.04
5177
timeout-minutes: 120
5278

@@ -57,32 +83,21 @@ jobs:
5783
uses: actions/setup-python@v5
5884
with:
5985
python-version: ${{ matrix.PYTHON_VERSION }}
60-
- name: Get run ID of "Nightly-build" workflow
61-
id: get-run-id
62-
run: |
63-
OTHER_REPO="uxlfoundation/oneDAL"
64-
WF_NAME="Nightly-build"
65-
JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId'
66-
RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"`
67-
echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
68-
echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
69-
env:
70-
GH_TOKEN: ${{ github.token }}
7186
- name: Download oneDAL build artifact
7287
uses: actions/download-artifact@v4
7388
with:
7489
name: __release_lnx
7590
github-token: ${{ github.token }}
76-
repository: uxlfoundation/oneDAL
77-
run-id: ${{ steps.get-run-id.outputs.run-id }}
91+
repository: ${{ env.ONEDAL_REPO }}
92+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
7893
path: ./__release_lnx
7994
- name: Download oneDAL environment artifact
8095
uses: actions/download-artifact@v4
8196
with:
8297
name: oneDAL_env
8398
github-token: ${{ github.token }}
84-
repository: uxlfoundation/oneDAL
85-
run-id: ${{ steps.get-run-id.outputs.run-id }}
99+
repository: ${{ env.ONEDAL_REPO }}
100+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
86101
path: .ci/env
87102
- name: Set Environment Variables
88103
id: set-env
@@ -161,6 +176,7 @@ jobs:
161176
bash .ci/scripts/run_sklearn_tests.sh $CPU
162177
163178
sklearn_win:
179+
needs: onedal_nightly
164180
strategy:
165181
fail-fast: false
166182
matrix:
@@ -171,7 +187,7 @@ jobs:
171187
SKLEARN_VERSION: "1.2"
172188
- PYTHON_VERSION: "3.11"
173189
SKLEARN_VERSION: "1.3"
174-
name: WindowsNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
190+
name: WindowsNightly/venv Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
175191
runs-on: windows-2025
176192
timeout-minutes: 120
177193

@@ -182,33 +198,21 @@ jobs:
182198
uses: actions/setup-python@v5
183199
with:
184200
python-version: ${{ matrix.PYTHON_VERSION }}
185-
- name: Get run ID of "Nightly-build" workflow
186-
id: get-run-id
187-
shell: bash
188-
run: |
189-
OTHER_REPO="uxlfoundation/oneDAL"
190-
WF_NAME="Nightly-build"
191-
JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId'
192-
RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"`
193-
echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
194-
echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
195-
env:
196-
GH_TOKEN: ${{ github.token }}
197201
- name: Download oneDAL build artifact
198202
uses: actions/download-artifact@v4
199203
with:
200204
name: __release_win
201205
github-token: ${{ github.token }}
202-
repository: uxlfoundation/oneDAL
203-
run-id: ${{ steps.get-run-id.outputs.run-id }}
206+
repository: ${{ env.ONEDAL_REPO }}
207+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
204208
path: ./__release_win
205209
- name: Download Intel BaseKit artifact
206210
uses: actions/download-artifact@v4
207211
with:
208212
name: intel_oneapi_basekit
209213
github-token: ${{ github.token }}
210-
repository: uxlfoundation/oneDAL
211-
run-id: ${{ steps.get-run-id.outputs.run-id }}
214+
repository: ${{ env.ONEDAL_REPO }}
215+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
212216
- name: Decompress Intel BaseKit
213217
shell: cmd
214218
run: |
@@ -234,8 +238,8 @@ jobs:
234238
with:
235239
name: opencl_rt_installer
236240
github-token: ${{ github.token }}
237-
repository: uxlfoundation/oneDAL
238-
run-id: ${{ steps.get-run-id.outputs.run-id }}
241+
repository: ${{ env.ONEDAL_REPO }}
242+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
239243
path: .
240244
- name: Install Intel OpenCL CPU Runtime
241245
if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
@@ -313,3 +317,182 @@ jobs:
313317
if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu
314318
set SKLEARNEX_PREVIEW=YES
315319
bash .ci/scripts/run_sklearn_tests.sh %CPU%
320+
321+
build_uxl:
322+
if: github.repository == 'uxlfoundation/scikit-learn-intelex'
323+
needs: onedal_nightly
324+
name: LinuxNightly build Python${{ needs.onedal_nightly.outputs.uxl-python }}
325+
runs-on: uxl-xlarge
326+
timeout-minutes: 30
327+
328+
steps:
329+
- name: Checkout Scikit-learn-intelex
330+
uses: actions/checkout@v4
331+
- name: Install Python
332+
uses: actions/setup-python@v5
333+
with:
334+
python-version: ${{ env.UXL_PYTHONVERSION }}
335+
cache: 'pip'
336+
cache-dependency-path: |
337+
**/dependencies-dev
338+
**/requirements-test.txt
339+
- name: Download oneDAL build artifact
340+
uses: actions/download-artifact@v4
341+
with:
342+
name: __release_lnx
343+
github-token: ${{ github.token }}
344+
repository: ${{ env.ONEDAL_REPO }}
345+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
346+
path: ./__release_lnx
347+
- name: Download oneDAL environment artifact
348+
uses: actions/download-artifact@v4
349+
with:
350+
name: oneDAL_env
351+
github-token: ${{ github.token }}
352+
repository: ${{ env.ONEDAL_REPO }}
353+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
354+
path: .ci/env
355+
- name: Set Environment Variables
356+
id: set-env
357+
run: |
358+
# Disable SPMD testing
359+
echo "NO_DIST=1" >> "$GITHUB_ENV"
360+
# enable coverage report generation
361+
echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"
362+
- name: apt-get
363+
run: sudo apt-get update && sudo apt-get install -y clang-format
364+
- name: dpcpp installation
365+
run: |
366+
# This CI system yields oneAPI dependencies from the oneDAL repository
367+
bash .ci/env/apt.sh dpcpp
368+
- name: describe system
369+
run: |
370+
source /opt/intel/oneapi/setvars.sh
371+
bash .ci/scripts/describe_system.sh
372+
- name: Install develop requirements
373+
id: install-reqs
374+
run: |
375+
pip install -r dependencies-dev
376+
echo "numpy-version=$(python -m pip freeze | grep numpy)" >> "$GITHUB_OUTPUT"
377+
pip list
378+
- name: Build daal4py/sklearnex
379+
run: |
380+
source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }}
381+
python setup.py bdist_wheel
382+
- name: Archive sklearnex build
383+
uses: actions/upload-artifact@v4
384+
with:
385+
name: sklearnex_build_${{ env.UXL_PYTHONVERSION }}
386+
path: |
387+
./dist/*.whl
388+
389+
outputs:
390+
numpy-version: ${{ steps.install-reqs.outputs.numpy-version }}
391+
392+
test_uxl:
393+
strategy:
394+
fail-fast: false
395+
matrix:
396+
include:
397+
- OS: uxl-gpu-xlarge
398+
FRAMEWORKS: "pytorch,numpy"
399+
DEVICE: gpu
400+
- OS: uxl-xlarge
401+
FRAMEWORKS: "pytorch,numpy"
402+
DEVICE: cpu
403+
needs: [onedal_nightly, build_uxl]
404+
name: LinuxNightly [${{ matrix.FRAMEWORKS }}]-${{ matrix.DEVICE }} test Python${{ needs.onedal_nightly.outputs.uxl-python }}_Sklearn${{ needs.onedal_nightly.outputs.uxl-sklearn }}
405+
runs-on: ${{ matrix.OS }}
406+
timeout-minutes: 120
407+
steps:
408+
- name: Checkout Scikit-learn-intelex
409+
uses: actions/checkout@v4
410+
- name: Install Python
411+
uses: actions/setup-python@v5
412+
with:
413+
python-version: ${{ env.UXL_PYTHONVERSION }}
414+
cache-dependency-path: |
415+
**/dependencies-dev
416+
**/requirements-test.txt
417+
- name: Download oneDAL build artifact
418+
uses: actions/download-artifact@v4
419+
with:
420+
name: __release_lnx
421+
github-token: ${{ github.token }}
422+
repository: ${{ env.ONEDAL_REPO }}
423+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
424+
path: ./__release_lnx
425+
- name: Download oneDAL environment artifact
426+
uses: actions/download-artifact@v4
427+
with:
428+
name: oneDAL_env
429+
github-token: ${{ github.token }}
430+
repository: ${{ env.ONEDAL_REPO }}
431+
run-id: ${{ needs.onedal_nightly.outputs.run-id }}
432+
path: .ci/env
433+
- name: Set Environment Variables
434+
id: set-env
435+
run: |
436+
echo "NO_DIST=1" >> "$GITHUB_ENV"
437+
# enable coverage report generation
438+
echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV"
439+
echo "ONEDAL_PYTEST_FRAMEWORKS=${{ matrix.FRAMEWORKS }}" >> "$GITHUB_ENV"
440+
# reduce GPU driver/runner related memory issues
441+
echo "NEOReadDebugKeys=1" >> "$GITHUB_ENV"
442+
echo "EnableRecoverablePageFaults=1" >> "$GITHUB_ENV"
443+
echo "GpuFaultCheckThreshold=0" >> "$GITHUB_ENV"
444+
# set build numpy version for use in generating code coverage
445+
echo "NUMPY_BUILD=${{ needs.build_uxl.outputs.numpy-version }}" >> "$GITHUB_ENV"
446+
- name: apt-get
447+
run: sudo apt-get update
448+
- name: dpcpp installation
449+
run: |
450+
# This CI system yields oneAPI dependencies from the oneDAL repository
451+
bash .ci/env/apt.sh dpcpp
452+
- name: describe system
453+
run: |
454+
source /opt/intel/oneapi/setvars.sh
455+
bash .ci/scripts/describe_system.sh
456+
- name: Install test requirements
457+
run: |
458+
bash .ci/scripts/setup_sklearn.sh ${{ env.UXL_SKLEARNVERSION }}
459+
pip install --upgrade -r requirements-test.txt
460+
pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ env.UXL_SKLEARVERSION }}) pyyaml
461+
pip list
462+
- name: Download sklearnex wheel
463+
uses: actions/download-artifact@v4
464+
with:
465+
name: sklearnex_build_${{ env.UXL_PYTHONVERSION }}
466+
- name: Install PyTorch
467+
if: contains(matrix.FRAMEWORKS, 'pytorch')
468+
run: |
469+
pip install torch --index-url https://download.pytorch.org/whl/xpu
470+
python -c "import torch; _=[print(torch.xpu.get_device_name(i)) for i in range(torch.xpu.device_count())]"
471+
- name: Install daal4py/sklearnex
472+
run: pip install *.whl
473+
- name: Sklearnex testing
474+
run: |
475+
source .github/scripts/activate_components.sh
476+
export COVERAGE_FILE=$(pwd)/.coverage.sklearnex
477+
cd .ci
478+
../conda-recipe/run_test.sh
479+
- name: Sklearn testing
480+
run: |
481+
source .github/scripts/activate_components.sh
482+
export COVERAGE_FILE=$(pwd)/.coverage.sklearn
483+
bash .ci/scripts/run_sklearn_tests.sh ${{ matrix.DEVICE }}
484+
- name: Create coverage report
485+
run: |
486+
source .github/scripts/activate_components.sh
487+
bash .github/scripts/generate_coverage_reports.sh uxl_lnx_${{ matrix.DEVICE }}
488+
- name: Archive coverage report
489+
uses: actions/upload-artifact@v4
490+
with:
491+
name: coverage_uxl_lnx_${{ matrix.DEVICE }}
492+
path: |
493+
*uxl_lnx_${{ matrix.DEVICE }}.info
494+
- name: Sklearn testing [preview]
495+
run: |
496+
source .github/scripts/activate_components.sh
497+
export SKLEARNEX_PREVIEW='YES'
498+
bash .ci/scripts/run_sklearn_tests.sh ${{ matrix.DEVICE }}

‎daal4py/sklearn/utils/validation.py

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -72,21 +72,20 @@ def _assert_all_finite(
7272

7373
# Data with small size has too big relative overhead
7474
# TODO: tune threshold size
75-
if hasattr(X, "size"):
76-
if X.size < 32768:
77-
if sklearn_check_version("1.1"):
78-
_sklearn_assert_all_finite(
79-
X,
80-
allow_nan=allow_nan,
81-
msg_dtype=msg_dtype,
82-
estimator_name=estimator_name,
83-
input_name=input_name,
84-
)
85-
else:
86-
_sklearn_assert_all_finite(X, allow_nan=allow_nan, msg_dtype=msg_dtype)
87-
return
88-
8975
is_df = is_DataFrame(X)
76+
if not (is_df or isinstance(X, np.ndarray)) or X.size < 32768:
77+
if sklearn_check_version("1.1"):
78+
_sklearn_assert_all_finite(
79+
X,
80+
allow_nan=allow_nan,
81+
msg_dtype=msg_dtype,
82+
estimator_name=estimator_name,
83+
input_name=input_name,
84+
)
85+
else:
86+
_sklearn_assert_all_finite(X, allow_nan=allow_nan, msg_dtype=msg_dtype)
87+
return
88+
9089
num_of_types = get_number_of_types(X)
9190

9291
# if X is heterogeneous pandas.DataFrame then

0 commit comments

Comments
 (0)