diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec
index fe19e7b46eb..d3576e5a7f6 100644
--- a/cmssw-tool-conf.spec
+++ b/cmssw-tool-conf.spec
@@ -192,8 +192,8 @@ Requires: oracle-fake-toolfile
Requires: tensorflow-toolfile
Requires: xtensor-toolfile
Requires: xtl-toolfile
+Requires: xgboost-toolfile
%define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug
## IMPORT scramv1-tool-conf
-
diff --git a/onnxruntime.spec b/onnxruntime.spec
index 95e4b906bc4..fdd639ccd11 100644
--- a/onnxruntime.spec
+++ b/onnxruntime.spec
@@ -1,15 +1,15 @@
-### RPM external onnxruntime 1.3.0
+### RPM external onnxruntime 1.6.0
## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
-%define tag 9294d790219aa0d1007d918d7d736d7f5d3e82d8
-%define branch cms/v1.3.0
%define github_user cms-externals
+%define branch cms/v%{realversion}
+%define tag 89a104708d109afc7f41661be33062605b7776a3
Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz
BuildRequires: cmake ninja
Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11
%prep
-%setup -n %{n}-%{realversion}
+%setup -q -n %{n}-%{realversion}
%build
rm -rf ../build; mkdir ../build; cd ../build
diff --git a/pip/cmsml.file b/pip/cmsml.file
new file mode 100644
index 00000000000..60df17043b3
--- /dev/null
+++ b/pip/cmsml.file
@@ -0,0 +1,3 @@
+Requires: py2-six
+
+%define RelocatePython %{i}/bin/*
diff --git a/pip/fire.file b/pip/fire.file
new file mode 100644
index 00000000000..e86e438798a
--- /dev/null
+++ b/pip/fire.file
@@ -0,0 +1 @@
+Requires: py2-six py2-termcolor py2-enum34
diff --git a/pip/flit.file b/pip/flit.file
index cc3264b5174..2fb5707265c 100644
--- a/pip/flit.file
+++ b/pip/flit.file
@@ -1 +1 @@
-Requires: py3-docutils py3-flit-core py3-pytoml py2-requests
+Requires: py2-docutils py3-flit-core py3-pytoml py2-requests
diff --git a/pip/law.file b/pip/law.file
new file mode 100644
index 00000000000..8d82d495e94
--- /dev/null
+++ b/pip/law.file
@@ -0,0 +1,3 @@
+Requires: py2-six py2-luigi py3-luigi py2-scinum
+
+%define RelocatePython %{i}/bin/*
diff --git a/pip/lockfile.file b/pip/lockfile.file
new file mode 100644
index 00000000000..04935344e87
--- /dev/null
+++ b/pip/lockfile.file
@@ -0,0 +1 @@
+Requires: py2-wheel
diff --git a/pip/luigi.file b/pip/luigi.file
new file mode 100644
index 00000000000..23d3f71b3ef
--- /dev/null
+++ b/pip/luigi.file
@@ -0,0 +1,5 @@
+Requires: py2-tornado py3-tornado py2-python-daemon py2-python-dateutil py2-enum34
+
+%define PipPostBuildPy2 (cd "%{i}/bin" && for f in *; do cp ${f} ${f}2; done)
+%define PipPostBuildPy3 (cd "%{i}/bin" && for f in *; do mv ${f} ${f}3; done)
+%define RelocatePython %{i}/bin/*
diff --git a/pip/xgboost.file b/pip/py2-xgboost.file
similarity index 54%
rename from pip/xgboost.file
rename to pip/py2-xgboost.file
index 52a4a4fbfc3..422b00365e2 100644
--- a/pip/xgboost.file
+++ b/pip/py2-xgboost.file
@@ -1,4 +1,4 @@
%ifnarch x86_64
-Patch0: py2-xgboost-arm-and-ppc
+Patch0: xgboost-arm-and-ppc-py-0-82
%endif
Requires: py2-scipy
diff --git a/pip/py3-keras2onnx.file b/pip/py3-keras2onnx.file
new file mode 100644
index 00000000000..1ac130bca0e
--- /dev/null
+++ b/pip/py3-keras2onnx.file
@@ -0,0 +1,2 @@
+Requires: py3-numpy py2-protobuf py2-requests py2-onnx py3-onnxconverter-common py2-fire
+%define source0 https://github.com/onnx/keras-onnx/archive/v%{realversion}.tar.gz
diff --git a/pip/py3-onnxconverter-common.file b/pip/py3-onnxconverter-common.file
new file mode 100644
index 00000000000..8722ab0f1ae
--- /dev/null
+++ b/pip/py3-onnxconverter-common.file
@@ -0,0 +1,2 @@
+Requires: py2-numpy py3-numpy py2-onnx py2-protobuf py2-six
+%define source0 https://github.com/microsoft/onnxconverter-common/archive/v%{realversion}.tar.gz
diff --git a/pip/py3-onnxmltools.file b/pip/py3-onnxmltools.file
new file mode 100644
index 00000000000..8a0127b4a1d
--- /dev/null
+++ b/pip/py3-onnxmltools.file
@@ -0,0 +1,2 @@
+Requires: py3-numpy py2-onnx py2-protobuf py3-onnxconverter-common py3-skl2onnx py3-keras2onnx
+%define source0 https://github.com/onnx/onnxmltools/archive/v%{realversion}.tar.gz
diff --git a/pip/py3-skl2onnx.file b/pip/py3-skl2onnx.file
new file mode 100644
index 00000000000..2b9ee31de77
--- /dev/null
+++ b/pip/py3-skl2onnx.file
@@ -0,0 +1,2 @@
+Requires: py2-six py3-numpy py3-scipy py2-protobuf py2-onnx py3-scikit-learn py3-onnxconverter-common
+%define source0 https://github.com/onnx/sklearn-onnx/archive/%{realversion}.tar.gz
diff --git a/pip/py3-xgboost.file b/pip/py3-xgboost.file
index d0e1c7be8d1..f3d784df50c 100644
--- a/pip/py3-xgboost.file
+++ b/pip/py3-xgboost.file
@@ -1 +1,8 @@
-Requires: py3-scipy
+Requires: py3-scipy xgboost
+%define patchsrc sed -i -e 's|^ outfiles = super().install()| return super().install()|' setup.py
+%define PipBuildOptions --install-option="--use-system-libxgboost"
+%define PipPostInstall \
+ mkdir -p "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \
+ ( cd "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \
+ ln -s ../../../../../../../xgboost/${XGBOOST_VERSION}/lib64/libxgboost.so . \
+ )
diff --git a/pip/python-daemon.file b/pip/python-daemon.file
new file mode 100644
index 00000000000..05506c5a36b
--- /dev/null
+++ b/pip/python-daemon.file
@@ -0,0 +1 @@
+Requires: py2-wheel py2-docutils py2-lockfile
diff --git a/pip/requirements.txt b/pip/requirements.txt
index 0a799fc5e28..5f4c47b382b 100644
--- a/pip/requirements.txt
+++ b/pip/requirements.txt
@@ -53,6 +53,7 @@ cachetools==4.2.0 ; python_version>'3.0'
cachy==0.3.0 ; python_version>'3.0'
cleo==0.8.1 ; python_version>'3.0'
clikit==0.6.2 ; python_version>'3.0'
+cmsml==0.1.1
crashtest==0.3.1 ; python_version>'3.0'
certifi==2020.12.5
cffi==1.14.4
@@ -75,11 +76,12 @@ decorator==4.4.2
defusedxml==0.6.0
distlib==0.3.1
docopt==0.6.2
-docutils==0.16;python_version>'3.0'
+docutils==0.16
downhill==0.4.0
entrypoints==0.3
enum34==1.1.10;python_version<'3.0'
filelock==3.0.12
+fire==0.4.0
flit==3.0.0; python_version>'3.0'
flit-core==3.0.0; python_version>'3.0'
flake8==3.8.4
@@ -140,13 +142,18 @@ jupyterlab-widgets==1.0.0 ; python_version>'3.0'
Keras==2.4.3
keras-applications==1.0.8
keras-preprocessing==1.1.0
+keras2onnx==1.7.0 ; python_version>'3.0'
keyring==21.8.0 ; python_version>'3.0'
kiwisolver==1.1.0
+law==0.1.3
#can not go to 1.5.x in python2 due to astroid
lazy-object-proxy==1.4.3
lizard==1.17.7
#NO_AUTO_UPDATE: Source is defined in llvmlite.file, changing version here is not going to get new sources
llvmlite==0.33.0
+lockfile==0.12.2
+luigi==2.8.13 ; python_version<'3.0'
+luigi==3.0.2 ; python_version>'3.0'
lxml==4.6.2
lz4==2.2.1 ; python_version<'3.0'
lz4==3.1.1 ; python_version>'3.0'
@@ -181,7 +188,9 @@ numexpr==2.7.2
numpy==1.16.6 ; python_version<'3.0'
#numpy 1.19 needs an updated tensorflow
numpy==1.17.5 ; python_version>'3.0'
-onnx==1.8.0
+onnx==1.8.1
+onnxmltools==1.7.0 ; python_version>'3.0'
+onnxconverter-common==1.7.0 ; python_version>'3.0'
opt-einsum==2.3.2 ; python_version<'3.0'
opt-einsum==3.3.0 ; python_version>'3.0'
ordereddict==1.1
@@ -215,7 +224,7 @@ psutil==5.8.0
ptyprocess==0.7.0
pyasn1-modules==0.2.8
pyasn1==0.4.8
-pybind11==2.6.0
+pybind11==2.6.2
pybrain==0.3.3
pycodestyle==2.6.0
pycparser==2.20
@@ -239,6 +248,7 @@ pytest==6.2.1 ; python_version>'3.0'
pytest-cov==2.10.1
pytest-runner==5.2
python-cjson==1.2.2;python_version<'3.0'
+python-daemon==2.2.4
python-dateutil==2.8.1
python-ldap==3.3.1
pytoml==0.1.21;python_version>'3.0'
@@ -258,7 +268,8 @@ rootpy==1.0.1
scandir==1.10.0
schema==0.7.3
scikit-learn==0.20.4 ; python_version<'3.0'
-scikit-learn==0.23.2 ; python_version>'3.0'
+scikit-learn==0.24.1 ; python_version>'3.0'
+scinum==1.1.3
scipy==1.2.3 ; python_version<'3.0'
scipy==1.5.4 ; python_version>'3.0'
threadpoolctl==2.0.0 ; python_version> '3.0'
@@ -269,6 +280,7 @@ shellingham==1.3.2 ; python_version> '3.0'
simplegeneric==0.8.1
singledispatch==3.4.0.3
six==1.15.0
+skl2onnx==1.7.1 ; python_version>'3.0'
smmap==3.0.4
smmap2==3.0.1
soupsieve==1.9.6
@@ -318,8 +330,9 @@ wheel==0.33.6
widgetsnbextension==3.5.1
#cannot wrapt update before astroid is
wrapt==1.11.2
+# xgboost version for python 3 must match the one we install as a library
xgboost==0.82 ; python_version<'3.0'
-xgboost==0.90 ; python_version>'3.0'
+xgboost==1.3.3 ; python_version>'3.0'
#bumping this pulls in xrootd - which looks like it needs some understanding
xrootdpyfs==0.2.1
yarl==1.6.3 ;python_version>'3.0'
diff --git a/pip/scinum.file b/pip/scinum.file
new file mode 100644
index 00000000000..f442253c785
--- /dev/null
+++ b/pip/scinum.file
@@ -0,0 +1 @@
+Requires: py2-numpy py3-numpy
diff --git a/python_tools.spec b/python_tools.spec
index 161c0c82b7b..feff88ef7cb 100644
--- a/python_tools.spec
+++ b/python_tools.spec
@@ -2,7 +2,7 @@
## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES}
## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
Source: none
-
+
Requires: root curl python python3 xrootd llvm hdf5 mxnet-predict yoda opencv
Requires: py2-scipy
@@ -12,6 +12,8 @@ Requires: py2-scikit-learn
#save for the end
Requires: py2-tensorflow py3-tensorflow
Requires: py2-googlePackages
+Requires: py2-cmsml
+Requires: py2-law
Requires: py2-cloudpickle
Requires: py2-tables
@@ -21,9 +23,9 @@ Requires: py2-histogrammar py3-histogrammar
Requires: py2-pandas
Requires: py2-root_numpy
Requires: py2-Bottleneck
-Requires: py2-downhill
+Requires: py2-downhill
Requires: py2-theanets
-Requires: py2-xgboost
+Requires: py2-xgboost py3-xgboost
Requires: py2-llvmlite
Requires: py3-numba
Requires: py2-hep_ml
@@ -108,7 +110,7 @@ Requires: py2-schema
#Requires: py2-Jinja
Requires: py2-python-dateutil
Requires: py2-python-cjson
-Requires: py2-enum34
+Requires: py2-enum34
Requires: py2-functools32
Requires: py2-mock
Requires: py2-pbr
@@ -164,6 +166,7 @@ Requires: py2-atomicwrites
Requires: py2-attrs
Requires: py2-nbdime
Requires: py2-onnx
+Requires: py3-onnxmltools
Requires: py2-backports
Requires: py2-backports_abc
Requires: py2-colorama
@@ -245,4 +248,4 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/python_tools.xml
EOF_TOOLFILE
-
+
diff --git a/py2-xgboost-arm-and-ppc.patch b/xgboost-arm-and-ppc-py-0-82.patch
similarity index 99%
rename from py2-xgboost-arm-and-ppc.patch
rename to xgboost-arm-and-ppc-py-0-82.patch
index 877e1d95301..a4df6651dcf 100644
--- a/py2-xgboost-arm-and-ppc.patch
+++ b/xgboost-arm-and-ppc-py-0-82.patch
@@ -5,7 +5,7 @@ index 8e13403..9a68e88 100644
@@ -12,14 +12,7 @@
#include "./hist_util.h"
#include "./quantile.h"
-
+
-#if defined(XGBOOST_MM_PREFETCH_PRESENT)
- #include
- #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast(addr), _MM_HINT_T0)
@@ -15,6 +15,6 @@ index 8e13403..9a68e88 100644
- #define PREFETCH_READ_T0(addr) do {} while (0)
-#endif // defined(XGBOOST_MM_PREFETCH_PRESENT)
+#define PREFETCH_READ_T0(addr) do {} while (0)
-
+
namespace xgboost {
namespace common {
diff --git a/xgboost-arm-and-ppc.patch b/xgboost-arm-and-ppc.patch
new file mode 100644
index 00000000000..81c4565554c
--- /dev/null
+++ b/xgboost-arm-and-ppc.patch
@@ -0,0 +1,20 @@
+diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc
+index 8e13403..9a68e88 100644
+--- a/src/common/hist_util.cc
++++ b/src/common/hist_util.cc
+@@ -17,14 +17,7 @@
+ #include "quantile.h"
+ #include "./../tree/updater_quantile_hist.h"
+
+-#if defined(XGBOOST_MM_PREFETCH_PRESENT)
+- #include
+- #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast(addr), _MM_HINT_T0)
+-#elif defined(XGBOOST_BUILTIN_PREFETCH_PRESENT)
+- #define PREFETCH_READ_T0(addr) __builtin_prefetch(reinterpret_cast(addr), 0, 3)
+-#else // no SW pre-fetching available; PREFETCH_READ_T0 is no-op
+- #define PREFETCH_READ_T0(addr) do {} while (0)
+-#endif // defined(XGBOOST_MM_PREFETCH_PRESENT)
++#define PREFETCH_READ_T0(addr) do {} while (0)
+
+ namespace xgboost {
+ namespace common {
diff --git a/xgboost-toolfile.spec b/xgboost-toolfile.spec
new file mode 100644
index 00000000000..cfdf1299961
--- /dev/null
+++ b/xgboost-toolfile.spec
@@ -0,0 +1,24 @@
+### RPM external xgboost-toolfile 1.0
+Requires: xgboost
+
+%prep
+
+%build
+
+%install
+mkdir -p %i/etc/scram.d
+cat << \EOF_TOOLFILE >%i/etc/scram.d/xgboost.xml
+
+
+
+
+
+
+
+
+
+
+
+EOF_TOOLFILE
+
+## IMPORT scram-tools-post
diff --git a/xgboost.spec b/xgboost.spec
new file mode 100644
index 00000000000..46c36d6c0b0
--- /dev/null
+++ b/xgboost.spec
@@ -0,0 +1,23 @@
+### RPM external xgboost 1.3.3
+
+BuildRequires: cmake
+Source: git+https://github.com/dmlc/xgboost.git?obj=master/v%{realversion}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz
+Patch0: xgboost-arm-and-ppc
+
+%prep
+%setup -q -n %{n}-%{realversion}
+%ifnarch x86_64
+%patch0 -p1
+%endif
+
+%build
+rm -rf ../build; mkdir ../build; cd ../build
+cmake ../%{n}-%{realversion} \
+ -DCMAKE_INSTALL_PREFIX=%{i} \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DUSE_CUDA=OFF
+
+%install
+cd ../build
+make %{makeprocesses}
+make install