Merge branch 'oneapi-src:main' into dev/eglaser-clkernel-removal

.ci/env/environment.yml

@@ -1,7 +1,6 @@
 name: ci-env
 channels:
   - conda-forge
-  - intel
   - defaults
 dependencies:
-  - impi-devel=2021.10.0
+  - impi-devel=2021.12.0

.github/workflows/docker-validation-ci.yml

@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Build docker image
         run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest

.github/workflows/docker-validation-nightly.yml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Build docker image
         run: docker build . --file dev/docker/onedal-dev.Dockerfile --tag onedal-dev:latest
       - name: Building oneDAL

.github/workflows/openssf-scorecard.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           persist-credentials: false
 

.github/workflows/renovate-validation.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - name: Validate
         uses: suzuki-shunsuke/github-action-renovate-config-validator@v1.0.1
         with:

CONTRIBUTING.md

@@ -23,6 +23,14 @@ We welcome community contributions to Intel(R) oneAPI Data Analytics Library. Yo
 
 Refer to our guidelines on [pull requests](#pull-requests) and [issues](#issues) before you proceed.
 
+## Contacting maintainers
+You may reach out to Intel project maintainers privately at onedal.maintainers@intel.com.
+[Codeowners](https://github.com/oneapi-src/oneDAL/blob/main/.github/CODEOWNERS) configuration defines specific maintainers for corresponding code sections, however it's currently limited to Intel members. With further migration to UXL we will be changing this, but here are non-Intel contacts: 
+
+For ARM specifics you may contact: [@rakshithgb-fujitsu](https://github.com/rakshithgb-fujitsu/)
+
+For RISC-V specifics you may contact: [@keeranroth](https://github.com/keeranroth/)
+
 ## Issues
 
 Use [GitHub issues](https://github.com/oneapi-src/oneDAL/issues) to:
@@ -35,8 +43,9 @@ Use [GitHub issues](https://github.com/oneapi-src/oneDAL/issues) to:
 
 To contribute your changes directly to the repository, do the following:
 - Make sure you can build the product and run all the examples with your patch.
+- Product uses bazel for validation and your changes should pass tests. Please add new tests as necessary. [Bazel Guide for oneDAL](https://github.com/oneapi-src/oneDAL/tree/main/dev/bazel)
 - Make sure your code is in line with our [coding style](#code-style) as `clang-format` is one of the checks in our public CI.
-- For a larger feature, provide a relevant example.
+- For a larger feature, provide a relevant example, and tests.
 - [Document](#documentation-guidelines) your code.
 - [Submit](https://github.com/oneapi-src/oneDAL/pulls) a pull request into the `main` branch.
 

MODULE.bazel

@@ -1,3 +1,3 @@
 module(name = "onedal")
 
-bazel_dep(name = "bazel_skylib", version = "1.6.1")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")

README.md

@@ -19,7 +19,11 @@
 
 [Installation](#installation)   |   [Documentation](#documentation)   |   [Support](#support)   |   [Examples](#examples)   |   [How to Contribute](CONTRIBUTING.md)   
 
-[![Build Status](https://dev.azure.com/daal/DAAL/_apis/build/status/oneapi-src.oneDAL?branchName=main)](https://dev.azure.com/daal/DAAL/_build/latest?definitionId=5&branchName=main) [![License](https://img.shields.io/github/license/oneapi-src/oneDAL.svg)](https://github.com/oneapi-src/oneDAL/blob/main/LICENSE) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8859/badge)](https://www.bestpractices.dev/projects/8859) [![Join the community on GitHub Discussions](https://badgen.net/badge/join%20the%20discussion/on%20github/black?icon=github)](https://github.com/oneapi-src/oneDAL/discussions)
+[![Build Status](https://dev.azure.com/daal/DAAL/_apis/build/status/oneapi-src.oneDAL?branchName=main)](https://dev.azure.com/daal/DAAL/_build/latest?definitionId=5&branchName=main)
+[![License](https://img.shields.io/github/license/oneapi-src/oneDAL.svg)](https://github.com/oneapi-src/oneDAL/blob/main/LICENSE)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8859/badge)](https://www.bestpractices.dev/projects/8859)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/oneapi-src/oneDAL/badge)](https://securityscorecards.dev/viewer/?uri=github.com/oneapi-src/oneDAL)
+[![Join the community on GitHub Discussions](https://badgen.net/badge/join%20the%20discussion/on%20github/black?icon=github)](https://github.com/oneapi-src/oneDAL/discussions)
 
 oneAPI Data Analytics Library (oneDAL) is a powerful machine learning library that helps you accelerate big data analysis at all stages: **preprocessing**, **transformation**, **analysis**, **modeling**, **validation**, and **decision making**.
 
@@ -38,9 +42,24 @@ Deprecation Notice: The Java interfaces are removed from the oneDAL library.
 
 ## Installation
 
-Check [System Requirements](https://oneapi-src.github.io/oneDAL/system-requirements.html) before installing oneDAL.
+Check the [System Requirements](https://oneapi-src.github.io/oneDAL/system-requirements.html) before installing to ensure compatibility with your system.
+
+There are several options available for installing oneDAL:
+
+- **Binary Distribution**: You can download pre-built binary packages from the following sources:
+    - Intel® oneAPI:
+        - Download as Part of the [Intel® oneAPI Base Toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onedal.html#gs.8xrue2)
+        - Download as the Stand-Alone [Intel® oneAPI Data Analytics Library](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onedal.html#gs.8xrue2)
+    - Anaconda:
+        | Channel | Version |
+        |:-------:|:-------:|
+        | intel | [![Anaconda-Server Intel Badge](https://anaconda.org/intel/dal-devel/badges/version.svg)](https://anaconda.org/intel/dal-devel) |
+        | conda-forge | [![Anaconda-Server Conda-forge Badge](https://anaconda.org/conda-forge/dal-devel/badges/version.svg)](https://anaconda.org/conda-forge/dal-devel) |
+
+    - [NuGet](https://www.nuget.org/packages/inteldal.devel.linux-x64)
+
+- **Source Distribution**: You can build the library from source. To do this, [download the specific version of oneDAL](https://github.com/oneapi-src/oneDAL/releases) from the official GitHub repository and follow the instructions in the [INSTALL.md](INSTALL.md).
 
-You can [download the specific version of oneDAL](https://github.com/oneapi-src/oneDAL/releases) or [install it from sources](INSTALL.md).
 
 ## Examples
 
@@ -117,7 +136,11 @@ To report a vulnerability, refer to [Intel vulnerability reporting policy](https
 
 ### Contribute <!-- omit in toc -->
 
-We welcome community contributions. Check our [contributing guidelines](CONTRIBUTING.md) to learn more.
+We welcome community contributions. Check our [contributing guidelines](CONTRIBUTING.md) to learn more. You can also contact the oneDAL team via [UXL Foundation Slack] using
+[#onedal] channel.
+
+[UXL Foundation Slack]: https://slack-invite.uxlfoundation.org/
+[#onedal]: https://uxlfoundation.slack.com/channels/onedal
 
 ## License <!-- omit in toc -->
 

SECURITY.md

@@ -19,13 +19,13 @@
 
 As an open-source project, we understand the importance of and responsibility
 for security. This Security Policy outlines our guidelines and procedures to
-ensure the highest level of security and trust for Intel(R) oneAPI Data Analytics Library.
+ensure the highest level of security and trust for oneDAL
 
 ## Supported Versions
 
 Security vulnerabilities are fixed in the [latest version][1]
 and delivered as a patch release. We don't guarantee security fixes to be
-back-ported to older Intel(R) oneAPI Data Analytics Library versions.
+back-ported to older oneDAL versions.
 
 ## Report a Vulnerability
 
@@ -52,24 +52,24 @@ Along with the report, provide the following info:
 
 ### When Should I Report a Vulnerability?
 
-* You think you discovered a potential security vulnerability in Intel(R) oneAPI Data Analytics Library.
-* You are unsure how the potential vulnerability affects Intel(R) oneAPI Data Analytics Library.
+* You think you discovered a potential security vulnerability in oneDAL
+* You are unsure how the potential vulnerability affects oneDAL
 * You think you discovered a vulnerability in another project or 3rd party
-component on which Intel(R) oneAPI Data Analytics Library depends. If the issue is not fixed in the 3rd party
+component on which oneDAL depends. If the issue is not fixed in the 3rd party
 component, try to report directly there first.
 
 ### When Should I NOT Report a Vulnerability?
 
 * You got an automated scan hit and are unable to provide details.
-* You need help using Intel(R) oneAPI Data Analytics Library for security.
+* You need help using oneDAL for security.
 * You need help applying security-related updates.
 * Your issue is not security-related.
 
 ## Security Reports Review Process
 
 We aim to respond quickly to your inquiry and coordinate a fix and
 disclosure with you. All confirmed security vulnerabilities will be addressed
-according to severity level and impact on Intel(R) oneAPI Data Analytics Library. Normally, security issues are fixed in the next planned release.
+according to severity level and impact on oneDAL Normally, security issues are fixed in the next planned release.
 
 ## Disclosure Policy
 
@@ -88,4 +88,6 @@ If you have any suggestions on how this Policy could be improved, submit
 an issue or a pull request to this repository. **Do not** report
 potential vulnerabilities or security flaws via a pull request.
 
-[1]: https://github.com/oneapi-src/oneDAL/releases
+[1]: https://github.com/oneapi-src/oneDAL/releases
+[2]: https://github.com/oneapi-src/oneDAL/security/advisories/new
+[3]: https://github.com/oneapi-src/oneDAL/security/advisories

cpp/daal/include/services/env_detect.h

@@ -182,6 +182,10 @@ class DAAL_EXPORT Environment : public Base
     void initNumberOfThreads();
 
     env _env;
+    // Pointer to the oneapi::tbb::task_scheduler_handle class object, global for oneDAL.
+    // The oneapi::tbb::task_scheduler_handle and the oneapi::tbb::finalize function
+    // allow user to wait for completion of worker threads.
+    void * _schedulerHandle;
     void * _globalControl;
 };
 } // namespace interface1

cpp/daal/src/externals/core_threading_win_dll.cpp

@@ -143,7 +143,9 @@ typedef void (*_daal_wait_task_group_t)(void * taskGroupPtr);
 
 typedef bool (*_daal_is_in_parallel_t)();
 typedef void (*_daal_tbb_task_scheduler_free_t)(void *& globalControl);
+typedef void (*_daal_tbb_task_scheduler_handle_free_t)(void *& schedulerHandle);
 typedef size_t (*_setNumberOfThreads_t)(const size_t, void **);
+typedef size_t (*_setSchedulerHandle_t)(void **);
 typedef void * (*_daal_threader_env_t)();
 
 typedef void (*_daal_parallel_sort_int32_t)(int *, int *);
@@ -205,10 +207,12 @@ static _daal_del_task_group_t _daal_del_task_group_ptr   = NULL;
 static _daal_run_task_group_t _daal_run_task_group_ptr   = NULL;
 static _daal_wait_task_group_t _daal_wait_task_group_ptr = NULL;
 
-static _daal_is_in_parallel_t _daal_is_in_parallel_ptr                   = NULL;
-static _daal_tbb_task_scheduler_free_t _daal_tbb_task_scheduler_free_ptr = NULL;
-static _setNumberOfThreads_t _setNumberOfThreads_ptr                     = NULL;
-static _daal_threader_env_t _daal_threader_env_ptr                       = NULL;
+static _daal_is_in_parallel_t _daal_is_in_parallel_ptr                                 = NULL;
+static _daal_tbb_task_scheduler_free_t _daal_tbb_task_scheduler_free_ptr               = NULL;
+static _daal_tbb_task_scheduler_handle_free_t _daal_tbb_task_scheduler_handle_free_ptr = NULL;
+static _setNumberOfThreads_t _setNumberOfThreads_ptr                                   = NULL;
+static _setSchedulerHandle_t _setSchedulerHandle_ptr                                   = NULL;
+static _daal_threader_env_t _daal_threader_env_ptr                                     = NULL;
 
 static _daal_parallel_sort_int32_t _daal_parallel_sort_int32_ptr                         = NULL;
 static _daal_parallel_sort_uint64_t _daal_parallel_sort_uint64_ptr                       = NULL;
@@ -657,6 +661,16 @@ DAAL_EXPORT void _daal_tbb_task_scheduler_free(void *& init)
     return _daal_tbb_task_scheduler_free_ptr(init);
 }
 
+DAAL_EXPORT void _daal_tbb_task_scheduler_handle_free(void *& init)
+{
+    load_daal_thr_dll();
+    if (_daal_tbb_task_scheduler_handle_free_ptr == NULL)
+    {
+        _daal_tbb_task_scheduler_handle_free_ptr = (_daal_tbb_task_scheduler_handle_free_t)load_daal_thr_func("_daal_tbb_task_scheduler_handle_free");
+    }
+    return _daal_tbb_task_scheduler_handle_free_ptr(init);
+}
+
 DAAL_EXPORT size_t _setNumberOfThreads(const size_t numThreads, void ** init)
 {
     load_daal_thr_dll();
@@ -667,6 +681,16 @@ DAAL_EXPORT size_t _setNumberOfThreads(const size_t numThreads, void ** init)
     return _setNumberOfThreads_ptr(numThreads, init);
 }
 
+DAAL_EXPORT size_t _setSchedulerHandle(void ** init)
+{
+    load_daal_thr_dll();
+    if (_setSchedulerHandle_ptr == NULL)
+    {
+        _setSchedulerHandle_ptr = (_setSchedulerHandle_t)load_daal_thr_func("_setSchedulerHandle");
+    }
+    return _setSchedulerHandle_ptr(init);
+}
+
 DAAL_EXPORT void * _daal_threader_env()
 {
     load_daal_thr_dll();

cpp/daal/src/externals/service_stat_ref.h

@@ -1,6 +1,7 @@
 /* file: service_stat_ref.h */
 /*******************************************************************************
 * Copyright 2023 Intel Corporation
+* Copyright contributors to the oneDAL project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -24,8 +25,9 @@
 #ifndef __SERVICE_STAT_REF_H__
 #define __SERVICE_STAT_REF_H__
 
-typedef void (*func_type)(DAAL_INT, DAAL_INT, DAAL_INT, void *);
+#include "src/externals/service_memory.h"
 
+typedef void (*func_type)(DAAL_INT, DAAL_INT, DAAL_INT, void *);
 extern "C"
 {
 #define __DAAL_VSL_SS_MATRIX_STORAGE_COLS           0x00020000
@@ -194,8 +196,37 @@ struct RefStatistics<double, cpu>
 
     static int x2c_mom(const double * data, const __int64 nFeatures, const __int64 nVectors, double * variance, const __int64 method)
     {
-        int errcode = 0;
-
+        // E(x-\mu)^2 = E(x^2) - \mu^2
+        int errcode  = 0;
+        double * sum = (double *)daal::services::internal::service_calloc<double, cpu>(nFeatures, sizeof(double));
+        if (!sum) return -4;
+        daal::services::internal::service_memset<double, cpu>(variance, double(0), nFeatures);
+        DAAL_INT feature_ptr, vec_ptr;
+        double wtInv      = (double)1 / nVectors;
+        double wtInvMinus = (double)1 / (nVectors - 1);
+        double pt         = 0;
+        for (vec_ptr = 0; vec_ptr < nVectors; ++vec_ptr)
+        {
+#pragma omp simd
+            for (feature_ptr = 0; feature_ptr < nFeatures; ++feature_ptr)
+            {
+                pt = data[vec_ptr * nFeatures + feature_ptr];
+                sum[feature_ptr] += pt;
+                variance[feature_ptr] += (pt * pt); // 2RSum
+            }
+        }
+        double sumSqDivN; // S^2/n = n*\mu^2
+#pragma omp simd
+        for (feature_ptr = 0; feature_ptr < nFeatures; ++feature_ptr)
+        {
+            sumSqDivN = sum[feature_ptr];
+            sumSqDivN *= sumSqDivN;
+            sumSqDivN *= wtInv;
+            variance[feature_ptr] -= sumSqDivN; // (2RSum-S^2/n)
+            variance[feature_ptr] *= wtInvMinus;
+        }
+        daal::services::internal::service_free<double, cpu>(sum);
+        sum = NULL;
         return errcode;
     }
 
@@ -276,8 +307,37 @@ struct RefStatistics<float, cpu>
 
     static int x2c_mom(const float * data, const __int64 nFeatures, const __int64 nVectors, float * variance, const __int64 method)
     {
+        // E(x-\mu)^2 = E(x^2) - \mu^2
         int errcode = 0;
-
+        float * sum = (float *)daal::services::internal::service_calloc<float, cpu>(nFeatures, sizeof(float));
+        if (!sum) return -4;
+        daal::services::internal::service_memset<float, cpu>(variance, float(0), nFeatures);
+        DAAL_INT feature_ptr, vec_ptr;
+        float wtInv      = (float)1 / nVectors;
+        float wtInvMinus = (float)1 / (nVectors - 1);
+        float pt         = 0;
+        for (vec_ptr = 0; vec_ptr < nVectors; ++vec_ptr)
+        {
+#pragma omp simd
+            for (feature_ptr = 0; feature_ptr < nFeatures; ++feature_ptr)
+            {
+                pt = data[vec_ptr * nFeatures + feature_ptr];
+                sum[feature_ptr] += pt;
+                variance[feature_ptr] += (pt * pt); // 2RSum
+            }
+        }
+        float sumSqDivN; // S^2/n = n*\mu^2
+#pragma omp simd
+        for (feature_ptr = 0; feature_ptr < nFeatures; ++feature_ptr)
+        {
+            sumSqDivN = sum[feature_ptr];
+            sumSqDivN *= sumSqDivN;
+            sumSqDivN *= wtInv;
+            variance[feature_ptr] -= sumSqDivN; // (2RSum-S^2/n)
+            variance[feature_ptr] *= wtInvMinus;
+        }
+        daal::services::internal::service_free<float, cpu>(sum);
+        sum = NULL;
         return errcode;
     }
 

cpp/daal/src/services/env_detect.cpp

@@ -125,7 +125,7 @@ DAAL_EXPORT void daal::services::Environment::setDynamicLibraryThreadingTypeOnWi
     initNumberOfThreads();
 }
 
-DAAL_EXPORT daal::services::Environment::Environment() : _globalControl {}
+DAAL_EXPORT daal::services::Environment::Environment() : _schedulerHandle {}, _globalControl {}
 {
     _env.cpuid_init_flag = false;
     _env.cpuid           = -1;
@@ -136,7 +136,14 @@ DAAL_EXPORT daal::services::Environment::Environment(const Environment & e) : da
 DAAL_EXPORT void daal::services::Environment::initNumberOfThreads()
 {
     if (isInit) return;
-
+        // Initializes global oneapi::tbb::task_scheduler_handle object in oneDAL to prevent the unexpected
+        // destruction of the calling thread.
+        // When the oneapi::tbb::finalize function is called with an oneapi::tbb::task_scheduler_handle
+        // instance, it blocks the calling thread until the completion of all worker
+        // threads that were implicitly created by the library.
+#if defined(TARGET_X86_64)
+    daal::setSchedulerHandle(&_schedulerHandle);
+#endif
     /* if HT enabled - set _numThreads to physical cores num */
     if (daal::internal::ServiceInst::serv_get_ht())
     {
@@ -155,7 +162,6 @@ DAAL_EXPORT void daal::services::Environment::initNumberOfThreads()
 DAAL_EXPORT daal::services::Environment::~Environment()
 {
     daal::services::daal_free_buffers();
-    _daal_tbb_task_scheduler_free(_globalControl);
 }
 
 void daal::services::Environment::_cpu_detect(int enable)
@@ -170,6 +176,9 @@ void daal::services::Environment::_cpu_detect(int enable)
 DAAL_EXPORT void daal::services::Environment::setNumberOfThreads(const size_t numThreads)
 {
     isInit = true;
+#if defined(TARGET_X86_64)
+    daal::setSchedulerHandle(&_schedulerHandle);
+#endif
     daal::setNumberOfThreads(numThreads, &_globalControl);
 }