Merge pull request #6649 from riga/ml_software

Update ML related software

cmssw-tool-conf.spec

@@ -192,8 +192,8 @@ Requires: oracle-fake-toolfile
 Requires: tensorflow-toolfile
 Requires: xtensor-toolfile
 Requires: xtl-toolfile
+Requires: xgboost-toolfile
 
 %define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug
 
 ## IMPORT scramv1-tool-conf
-

onnxruntime.spec

@@ -1,15 +1,15 @@
-### RPM external onnxruntime 1.3.0
+### RPM external onnxruntime 1.6.0
 ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
-%define tag 9294d790219aa0d1007d918d7d736d7f5d3e82d8
-%define branch cms/v1.3.0
 %define github_user cms-externals
+%define branch cms/v%{realversion}
+%define tag 89a104708d109afc7f41661be33062605b7776a3
 Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz
 
 BuildRequires: cmake ninja
 Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11
 
 %prep
-%setup -n %{n}-%{realversion}
+%setup -q -n %{n}-%{realversion}
 
 %build
 rm -rf ../build; mkdir ../build; cd ../build

pip/cmsml.file

@@ -0,0 +1,3 @@
+Requires: py2-six
+
+%define RelocatePython %{i}/bin/*

pip/fire.file

@@ -0,0 +1 @@
+Requires: py2-six py2-termcolor py2-enum34

pip/flit.file

@@ -1 +1 @@
-Requires: py3-docutils py3-flit-core py3-pytoml py2-requests
+Requires: py2-docutils py3-flit-core py3-pytoml py2-requests

pip/law.file

@@ -0,0 +1,3 @@
+Requires: py2-six py2-luigi py3-luigi py2-scinum
+
+%define RelocatePython %{i}/bin/*

pip/lockfile.file

@@ -0,0 +1 @@
+Requires: py2-wheel

pip/luigi.file

@@ -0,0 +1,5 @@
+Requires: py2-tornado py3-tornado py2-python-daemon py2-python-dateutil py2-enum34
+
+%define PipPostBuildPy2 (cd "%{i}/bin" && for f in *; do cp ${f} ${f}2; done)
+%define PipPostBuildPy3 (cd "%{i}/bin" && for f in *; do mv ${f} ${f}3; done)
+%define RelocatePython %{i}/bin/*

pip/xgboost.file → pip/py2-xgboost.file

@@ -1,4 +1,4 @@
 %ifnarch x86_64
-Patch0: py2-xgboost-arm-and-ppc
+Patch0: xgboost-arm-and-ppc-py-0-82
 %endif
 Requires: py2-scipy

pip/py3-keras2onnx.file

@@ -0,0 +1,2 @@
+Requires:  py3-numpy py2-protobuf py2-requests py2-onnx py3-onnxconverter-common py2-fire
+%define source0 https://github.com/onnx/keras-onnx/archive/v%{realversion}.tar.gz

pip/py3-onnxconverter-common.file

@@ -0,0 +1,2 @@
+Requires: py2-numpy py3-numpy py2-onnx py2-protobuf py2-six
+%define source0 https://github.com/microsoft/onnxconverter-common/archive/v%{realversion}.tar.gz

pip/py3-onnxmltools.file

@@ -0,0 +1,2 @@
+Requires: py3-numpy py2-onnx py2-protobuf py3-onnxconverter-common py3-skl2onnx py3-keras2onnx
+%define source0 https://github.com/onnx/onnxmltools/archive/v%{realversion}.tar.gz

pip/py3-skl2onnx.file

@@ -0,0 +1,2 @@
+Requires: py2-six py3-numpy py3-scipy py2-protobuf py2-onnx py3-scikit-learn py3-onnxconverter-common
+%define source0 https://github.com/onnx/sklearn-onnx/archive/%{realversion}.tar.gz

pip/py3-xgboost.file

@@ -1 +1,8 @@
-Requires: py3-scipy
+Requires: py3-scipy xgboost
+%define patchsrc sed -i -e 's|^        outfiles = super().install()|        return super().install()|' setup.py
+%define PipBuildOptions --install-option="--use-system-libxgboost"
+%define PipPostInstall \
+  mkdir -p "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \
+  ( cd "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \
+    ln -s ../../../../../../../xgboost/${XGBOOST_VERSION}/lib64/libxgboost.so . \
+  )

pip/python-daemon.file

@@ -0,0 +1 @@
+Requires: py2-wheel py2-docutils py2-lockfile

pip/requirements.txt

@@ -53,6 +53,7 @@ cachetools==4.2.0 ; python_version>'3.0'
 cachy==0.3.0 ; python_version>'3.0'
 cleo==0.8.1 ; python_version>'3.0'
 clikit==0.6.2 ; python_version>'3.0'
+cmsml==0.1.1
 crashtest==0.3.1 ; python_version>'3.0'
 certifi==2020.12.5
 cffi==1.14.4
@@ -75,11 +76,12 @@ decorator==4.4.2
 defusedxml==0.6.0
 distlib==0.3.1
 docopt==0.6.2
-docutils==0.16;python_version>'3.0'
+docutils==0.16
 downhill==0.4.0
 entrypoints==0.3
 enum34==1.1.10;python_version<'3.0'
 filelock==3.0.12
+fire==0.4.0
 flit==3.0.0; python_version>'3.0'
 flit-core==3.0.0; python_version>'3.0'
 flake8==3.8.4
@@ -140,13 +142,18 @@ jupyterlab-widgets==1.0.0 ; python_version>'3.0'
 Keras==2.4.3
 keras-applications==1.0.8
 keras-preprocessing==1.1.0
+keras2onnx==1.7.0 ; python_version>'3.0'
 keyring==21.8.0 ; python_version>'3.0'
 kiwisolver==1.1.0
+law==0.1.3
 #can not go to 1.5.x in python2 due to astroid
 lazy-object-proxy==1.4.3
 lizard==1.17.7
 #NO_AUTO_UPDATE: Source is defined in llvmlite.file, changing version here is not going to get new sources
 llvmlite==0.33.0
+lockfile==0.12.2
+luigi==2.8.13 ; python_version<'3.0'
+luigi==3.0.2 ; python_version>'3.0'
 lxml==4.6.2
 lz4==2.2.1 ; python_version<'3.0'
 lz4==3.1.1 ; python_version>'3.0'
@@ -181,7 +188,9 @@ numexpr==2.7.2
 numpy==1.16.6 ; python_version<'3.0'
 #numpy 1.19 needs an updated tensorflow
 numpy==1.17.5 ; python_version>'3.0'
-onnx==1.8.0
+onnx==1.8.1
+onnxmltools==1.7.0 ; python_version>'3.0'
+onnxconverter-common==1.7.0 ; python_version>'3.0'
 opt-einsum==2.3.2 ; python_version<'3.0'
 opt-einsum==3.3.0 ; python_version>'3.0'
 ordereddict==1.1
@@ -215,7 +224,7 @@ psutil==5.8.0
 ptyprocess==0.7.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
-pybind11==2.6.0
+pybind11==2.6.2
 pybrain==0.3.3
 pycodestyle==2.6.0
 pycparser==2.20
@@ -239,6 +248,7 @@ pytest==6.2.1 ; python_version>'3.0'
 pytest-cov==2.10.1
 pytest-runner==5.2
 python-cjson==1.2.2;python_version<'3.0'
+python-daemon==2.2.4
 python-dateutil==2.8.1
 python-ldap==3.3.1
 pytoml==0.1.21;python_version>'3.0'
@@ -258,7 +268,8 @@ rootpy==1.0.1
 scandir==1.10.0
 schema==0.7.3
 scikit-learn==0.20.4 ; python_version<'3.0'
-scikit-learn==0.23.2 ; python_version>'3.0'
+scikit-learn==0.24.1 ; python_version>'3.0'
+scinum==1.1.3
 scipy==1.2.3 ; python_version<'3.0'
 scipy==1.5.4 ; python_version>'3.0'
 threadpoolctl==2.0.0 ; python_version> '3.0'
@@ -269,6 +280,7 @@ shellingham==1.3.2 ; python_version> '3.0'
 simplegeneric==0.8.1
 singledispatch==3.4.0.3
 six==1.15.0
+skl2onnx==1.7.1 ; python_version>'3.0'
 smmap==3.0.4
 smmap2==3.0.1
 soupsieve==1.9.6
@@ -318,8 +330,9 @@ wheel==0.33.6
 widgetsnbextension==3.5.1
 #cannot wrapt update before astroid is
 wrapt==1.11.2
+# xgboost version for python 3 must match the one we install as a library
 xgboost==0.82 ; python_version<'3.0'
-xgboost==0.90 ; python_version>'3.0'
+xgboost==1.3.3 ; python_version>'3.0'
 #bumping this pulls in xrootd - which looks like it needs some understanding
 xrootdpyfs==0.2.1
 yarl==1.6.3 ;python_version>'3.0'

pip/scinum.file

@@ -0,0 +1 @@
+Requires: py2-numpy py3-numpy

python_tools.spec

@@ -2,7 +2,7 @@
 ## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES}
 ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
 Source: none
- 
+
 Requires: root curl python python3 xrootd llvm hdf5 mxnet-predict yoda opencv
 
 Requires: py2-scipy
@@ -12,6 +12,8 @@ Requires: py2-scikit-learn
 #save for the end
 Requires: py2-tensorflow py3-tensorflow
 Requires: py2-googlePackages
+Requires: py2-cmsml
+Requires: py2-law
 
 Requires: py2-cloudpickle
 Requires: py2-tables
@@ -21,9 +23,9 @@ Requires: py2-histogrammar py3-histogrammar
 Requires: py2-pandas
 Requires: py2-root_numpy
 Requires: py2-Bottleneck
-Requires: py2-downhill 
+Requires: py2-downhill
 Requires: py2-theanets
-Requires: py2-xgboost
+Requires: py2-xgboost py3-xgboost
 Requires: py2-llvmlite
 Requires: py3-numba
 Requires: py2-hep_ml
@@ -108,7 +110,7 @@ Requires: py2-schema
 #Requires: py2-Jinja
 Requires: py2-python-dateutil
 Requires: py2-python-cjson
-Requires: py2-enum34 
+Requires: py2-enum34
 Requires: py2-functools32
 Requires: py2-mock
 Requires: py2-pbr
@@ -164,6 +166,7 @@ Requires: py2-atomicwrites
 Requires: py2-attrs
 Requires: py2-nbdime
 Requires: py2-onnx
+Requires: py3-onnxmltools
 Requires: py2-backports
 Requires: py2-backports_abc
 Requires: py2-colorama
@@ -245,4 +248,4 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/python_tools.xml
 <tool name="%{n}" version="%{v}">
 </tool>
 EOF_TOOLFILE
- 
+

py2-xgboost-arm-and-ppc.patch → xgboost-arm-and-ppc-py-0-82.patch

@@ -5,7 +5,7 @@ index 8e13403..9a68e88 100644
 @@ -12,14 +12,7 @@
  #include "./hist_util.h"
  #include "./quantile.h"
- 
+
 -#if defined(XGBOOST_MM_PREFETCH_PRESENT)
 -  #include <xmmintrin.h>
 -  #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0)
@@ -15,6 +15,6 @@ index 8e13403..9a68e88 100644
 -  #define PREFETCH_READ_T0(addr) do {} while (0)
 -#endif  // defined(XGBOOST_MM_PREFETCH_PRESENT)
 +#define PREFETCH_READ_T0(addr) do {} while (0)
- 
+
  namespace xgboost {
  namespace common {

xgboost-arm-and-ppc.patch

@@ -0,0 +1,20 @@
+diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
+index 8e13403..9a68e88 100644
+--- a/src/common/hist_util.cc
++++ b/src/common/hist_util.cc
+@@ -17,14 +17,7 @@
+ #include "quantile.h"
+ #include "./../tree/updater_quantile_hist.h"
+
+-#if defined(XGBOOST_MM_PREFETCH_PRESENT)
+-  #include <xmmintrin.h>
+-  #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0)
+-#elif defined(XGBOOST_BUILTIN_PREFETCH_PRESENT)
+-  #define PREFETCH_READ_T0(addr) __builtin_prefetch(reinterpret_cast<const char*>(addr), 0, 3)
+-#else  // no SW pre-fetching available; PREFETCH_READ_T0 is no-op
+-  #define PREFETCH_READ_T0(addr) do {} while (0)
+-#endif  // defined(XGBOOST_MM_PREFETCH_PRESENT)
++#define PREFETCH_READ_T0(addr) do {} while (0)
+
+ namespace xgboost {
+ namespace common {

xgboost-toolfile.spec

@@ -0,0 +1,24 @@
+### RPM external xgboost-toolfile 1.0
+Requires: xgboost
+
+%prep
+
+%build
+
+%install
+mkdir -p %i/etc/scram.d
+cat << \EOF_TOOLFILE >%i/etc/scram.d/xgboost.xml
+<tool name="xgboost" version="@TOOL_VERSION@">
+  <lib name="xgboost"/>
+  <client>
+    <environment name="XGBOOST_BASE" default="@TOOL_ROOT@"/>
+    <environment name="BINDIR" default="$XGBOOST_BASE/bin"/>
+    <environment name="LIBDIR" default="$XGBOOST_BASE/lib64"/>
+    <environment name="INCLUDE" default="$XGBOOST_BASE/include"/>
+  </client>
+  <runtime name="PATH" value="$BINDIR" type="path"/>
+  <runtime name="ROOT_INCLUDE_PATH" value="$INCLUDE" type="path"/>
+</tool>
+EOF_TOOLFILE
+
+## IMPORT scram-tools-post

xgboost.spec

@@ -0,0 +1,23 @@
+### RPM external xgboost 1.3.3
+
+BuildRequires: cmake
+Source: git+https://github.com/dmlc/xgboost.git?obj=master/v%{realversion}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz
+Patch0: xgboost-arm-and-ppc
+
+%prep
+%setup -q -n %{n}-%{realversion}
+%ifnarch x86_64
+%patch0 -p1
+%endif
+
+%build
+rm -rf ../build; mkdir ../build; cd ../build
+cmake ../%{n}-%{realversion} \
+  -DCMAKE_INSTALL_PREFIX=%{i} \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DUSE_CUDA=OFF
+
+%install
+cd ../build
+make %{makeprocesses}
+make install