diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index fe19e7b46eb..d3576e5a7f6 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -192,8 +192,8 @@ Requires: oracle-fake-toolfile Requires: tensorflow-toolfile Requires: xtensor-toolfile Requires: xtl-toolfile +Requires: xgboost-toolfile %define skipreqtools jcompiler icc-cxxcompiler icc-ccompiler icc-f77compiler rivet2 opencl opencl-cpp nvidia-drivers intel-vtune jemalloc-debug ## IMPORT scramv1-tool-conf - diff --git a/onnxruntime.spec b/onnxruntime.spec index 95e4b906bc4..fdd639ccd11 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -1,15 +1,15 @@ -### RPM external onnxruntime 1.3.0 +### RPM external onnxruntime 1.6.0 ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES} -%define tag 9294d790219aa0d1007d918d7d736d7f5d3e82d8 -%define branch cms/v1.3.0 %define github_user cms-externals +%define branch cms/v%{realversion} +%define tag 89a104708d109afc7f41661be33062605b7776a3 Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz BuildRequires: cmake ninja Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11 %prep -%setup -n %{n}-%{realversion} +%setup -q -n %{n}-%{realversion} %build rm -rf ../build; mkdir ../build; cd ../build diff --git a/pip/cmsml.file b/pip/cmsml.file new file mode 100644 index 00000000000..60df17043b3 --- /dev/null +++ b/pip/cmsml.file @@ -0,0 +1,3 @@ +Requires: py2-six + +%define RelocatePython %{i}/bin/* diff --git a/pip/fire.file b/pip/fire.file new file mode 100644 index 00000000000..e86e438798a --- /dev/null +++ b/pip/fire.file @@ -0,0 +1 @@ +Requires: py2-six py2-termcolor py2-enum34 diff --git a/pip/flit.file b/pip/flit.file index cc3264b5174..2fb5707265c 100644 --- a/pip/flit.file +++ b/pip/flit.file @@ -1 +1 @@ -Requires: py3-docutils py3-flit-core py3-pytoml py2-requests +Requires: py2-docutils py3-flit-core py3-pytoml py2-requests diff --git a/pip/law.file b/pip/law.file new file mode 100644 index 00000000000..8d82d495e94 --- /dev/null +++ b/pip/law.file @@ -0,0 +1,3 @@ +Requires: py2-six py2-luigi py3-luigi py2-scinum + +%define RelocatePython %{i}/bin/* diff --git a/pip/lockfile.file b/pip/lockfile.file new file mode 100644 index 00000000000..04935344e87 --- /dev/null +++ b/pip/lockfile.file @@ -0,0 +1 @@ +Requires: py2-wheel diff --git a/pip/luigi.file b/pip/luigi.file new file mode 100644 index 00000000000..23d3f71b3ef --- /dev/null +++ b/pip/luigi.file @@ -0,0 +1,5 @@ +Requires: py2-tornado py3-tornado py2-python-daemon py2-python-dateutil py2-enum34 + +%define PipPostBuildPy2 (cd "%{i}/bin" && for f in *; do cp ${f} ${f}2; done) +%define PipPostBuildPy3 (cd "%{i}/bin" && for f in *; do mv ${f} ${f}3; done) +%define RelocatePython %{i}/bin/* diff --git a/pip/xgboost.file b/pip/py2-xgboost.file similarity index 54% rename from pip/xgboost.file rename to pip/py2-xgboost.file index 52a4a4fbfc3..422b00365e2 100644 --- a/pip/xgboost.file +++ b/pip/py2-xgboost.file @@ -1,4 +1,4 @@ %ifnarch x86_64 -Patch0: py2-xgboost-arm-and-ppc +Patch0: xgboost-arm-and-ppc-py-0-82 %endif Requires: py2-scipy diff --git a/pip/py3-keras2onnx.file b/pip/py3-keras2onnx.file new file mode 100644 index 00000000000..1ac130bca0e --- /dev/null +++ b/pip/py3-keras2onnx.file @@ -0,0 +1,2 @@ +Requires: py3-numpy py2-protobuf py2-requests py2-onnx py3-onnxconverter-common py2-fire +%define source0 https://github.com/onnx/keras-onnx/archive/v%{realversion}.tar.gz diff --git a/pip/py3-onnxconverter-common.file b/pip/py3-onnxconverter-common.file new file mode 100644 index 00000000000..8722ab0f1ae --- /dev/null +++ b/pip/py3-onnxconverter-common.file @@ -0,0 +1,2 @@ +Requires: py2-numpy py3-numpy py2-onnx py2-protobuf py2-six +%define source0 https://github.com/microsoft/onnxconverter-common/archive/v%{realversion}.tar.gz diff --git a/pip/py3-onnxmltools.file b/pip/py3-onnxmltools.file new file mode 100644 index 00000000000..8a0127b4a1d --- /dev/null +++ b/pip/py3-onnxmltools.file @@ -0,0 +1,2 @@ +Requires: py3-numpy py2-onnx py2-protobuf py3-onnxconverter-common py3-skl2onnx py3-keras2onnx +%define source0 https://github.com/onnx/onnxmltools/archive/v%{realversion}.tar.gz diff --git a/pip/py3-skl2onnx.file b/pip/py3-skl2onnx.file new file mode 100644 index 00000000000..2b9ee31de77 --- /dev/null +++ b/pip/py3-skl2onnx.file @@ -0,0 +1,2 @@ +Requires: py2-six py3-numpy py3-scipy py2-protobuf py2-onnx py3-scikit-learn py3-onnxconverter-common +%define source0 https://github.com/onnx/sklearn-onnx/archive/%{realversion}.tar.gz diff --git a/pip/py3-xgboost.file b/pip/py3-xgboost.file index d0e1c7be8d1..f3d784df50c 100644 --- a/pip/py3-xgboost.file +++ b/pip/py3-xgboost.file @@ -1 +1,8 @@ -Requires: py3-scipy +Requires: py3-scipy xgboost +%define patchsrc sed -i -e 's|^ outfiles = super().install()| return super().install()|' setup.py +%define PipBuildOptions --install-option="--use-system-libxgboost" +%define PipPostInstall \ + mkdir -p "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \ + ( cd "%{i}/${PYTHON3_LIB_SITE_PACKAGES}/xgboost/lib" && \ + ln -s ../../../../../../../xgboost/${XGBOOST_VERSION}/lib64/libxgboost.so . \ + ) diff --git a/pip/python-daemon.file b/pip/python-daemon.file new file mode 100644 index 00000000000..05506c5a36b --- /dev/null +++ b/pip/python-daemon.file @@ -0,0 +1 @@ +Requires: py2-wheel py2-docutils py2-lockfile diff --git a/pip/requirements.txt b/pip/requirements.txt index 0a799fc5e28..5f4c47b382b 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -53,6 +53,7 @@ cachetools==4.2.0 ; python_version>'3.0' cachy==0.3.0 ; python_version>'3.0' cleo==0.8.1 ; python_version>'3.0' clikit==0.6.2 ; python_version>'3.0' +cmsml==0.1.1 crashtest==0.3.1 ; python_version>'3.0' certifi==2020.12.5 cffi==1.14.4 @@ -75,11 +76,12 @@ decorator==4.4.2 defusedxml==0.6.0 distlib==0.3.1 docopt==0.6.2 -docutils==0.16;python_version>'3.0' +docutils==0.16 downhill==0.4.0 entrypoints==0.3 enum34==1.1.10;python_version<'3.0' filelock==3.0.12 +fire==0.4.0 flit==3.0.0; python_version>'3.0' flit-core==3.0.0; python_version>'3.0' flake8==3.8.4 @@ -140,13 +142,18 @@ jupyterlab-widgets==1.0.0 ; python_version>'3.0' Keras==2.4.3 keras-applications==1.0.8 keras-preprocessing==1.1.0 +keras2onnx==1.7.0 ; python_version>'3.0' keyring==21.8.0 ; python_version>'3.0' kiwisolver==1.1.0 +law==0.1.3 #can not go to 1.5.x in python2 due to astroid lazy-object-proxy==1.4.3 lizard==1.17.7 #NO_AUTO_UPDATE: Source is defined in llvmlite.file, changing version here is not going to get new sources llvmlite==0.33.0 +lockfile==0.12.2 +luigi==2.8.13 ; python_version<'3.0' +luigi==3.0.2 ; python_version>'3.0' lxml==4.6.2 lz4==2.2.1 ; python_version<'3.0' lz4==3.1.1 ; python_version>'3.0' @@ -181,7 +188,9 @@ numexpr==2.7.2 numpy==1.16.6 ; python_version<'3.0' #numpy 1.19 needs an updated tensorflow numpy==1.17.5 ; python_version>'3.0' -onnx==1.8.0 +onnx==1.8.1 +onnxmltools==1.7.0 ; python_version>'3.0' +onnxconverter-common==1.7.0 ; python_version>'3.0' opt-einsum==2.3.2 ; python_version<'3.0' opt-einsum==3.3.0 ; python_version>'3.0' ordereddict==1.1 @@ -215,7 +224,7 @@ psutil==5.8.0 ptyprocess==0.7.0 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pybind11==2.6.0 +pybind11==2.6.2 pybrain==0.3.3 pycodestyle==2.6.0 pycparser==2.20 @@ -239,6 +248,7 @@ pytest==6.2.1 ; python_version>'3.0' pytest-cov==2.10.1 pytest-runner==5.2 python-cjson==1.2.2;python_version<'3.0' +python-daemon==2.2.4 python-dateutil==2.8.1 python-ldap==3.3.1 pytoml==0.1.21;python_version>'3.0' @@ -258,7 +268,8 @@ rootpy==1.0.1 scandir==1.10.0 schema==0.7.3 scikit-learn==0.20.4 ; python_version<'3.0' -scikit-learn==0.23.2 ; python_version>'3.0' +scikit-learn==0.24.1 ; python_version>'3.0' +scinum==1.1.3 scipy==1.2.3 ; python_version<'3.0' scipy==1.5.4 ; python_version>'3.0' threadpoolctl==2.0.0 ; python_version> '3.0' @@ -269,6 +280,7 @@ shellingham==1.3.2 ; python_version> '3.0' simplegeneric==0.8.1 singledispatch==3.4.0.3 six==1.15.0 +skl2onnx==1.7.1 ; python_version>'3.0' smmap==3.0.4 smmap2==3.0.1 soupsieve==1.9.6 @@ -318,8 +330,9 @@ wheel==0.33.6 widgetsnbextension==3.5.1 #cannot wrapt update before astroid is wrapt==1.11.2 +# xgboost version for python 3 must match the one we install as a library xgboost==0.82 ; python_version<'3.0' -xgboost==0.90 ; python_version>'3.0' +xgboost==1.3.3 ; python_version>'3.0' #bumping this pulls in xrootd - which looks like it needs some understanding xrootdpyfs==0.2.1 yarl==1.6.3 ;python_version>'3.0' diff --git a/pip/scinum.file b/pip/scinum.file new file mode 100644 index 00000000000..f442253c785 --- /dev/null +++ b/pip/scinum.file @@ -0,0 +1 @@ +Requires: py2-numpy py3-numpy diff --git a/python_tools.spec b/python_tools.spec index 161c0c82b7b..feff88ef7cb 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -2,7 +2,7 @@ ## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES} ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES} Source: none - + Requires: root curl python python3 xrootd llvm hdf5 mxnet-predict yoda opencv Requires: py2-scipy @@ -12,6 +12,8 @@ Requires: py2-scikit-learn #save for the end Requires: py2-tensorflow py3-tensorflow Requires: py2-googlePackages +Requires: py2-cmsml +Requires: py2-law Requires: py2-cloudpickle Requires: py2-tables @@ -21,9 +23,9 @@ Requires: py2-histogrammar py3-histogrammar Requires: py2-pandas Requires: py2-root_numpy Requires: py2-Bottleneck -Requires: py2-downhill +Requires: py2-downhill Requires: py2-theanets -Requires: py2-xgboost +Requires: py2-xgboost py3-xgboost Requires: py2-llvmlite Requires: py3-numba Requires: py2-hep_ml @@ -108,7 +110,7 @@ Requires: py2-schema #Requires: py2-Jinja Requires: py2-python-dateutil Requires: py2-python-cjson -Requires: py2-enum34 +Requires: py2-enum34 Requires: py2-functools32 Requires: py2-mock Requires: py2-pbr @@ -164,6 +166,7 @@ Requires: py2-atomicwrites Requires: py2-attrs Requires: py2-nbdime Requires: py2-onnx +Requires: py3-onnxmltools Requires: py2-backports Requires: py2-backports_abc Requires: py2-colorama @@ -245,4 +248,4 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/python_tools.xml EOF_TOOLFILE - + diff --git a/py2-xgboost-arm-and-ppc.patch b/xgboost-arm-and-ppc-py-0-82.patch similarity index 99% rename from py2-xgboost-arm-and-ppc.patch rename to xgboost-arm-and-ppc-py-0-82.patch index 877e1d95301..a4df6651dcf 100644 --- a/py2-xgboost-arm-and-ppc.patch +++ b/xgboost-arm-and-ppc-py-0-82.patch @@ -5,7 +5,7 @@ index 8e13403..9a68e88 100644 @@ -12,14 +12,7 @@ #include "./hist_util.h" #include "./quantile.h" - + -#if defined(XGBOOST_MM_PREFETCH_PRESENT) - #include - #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast(addr), _MM_HINT_T0) @@ -15,6 +15,6 @@ index 8e13403..9a68e88 100644 - #define PREFETCH_READ_T0(addr) do {} while (0) -#endif // defined(XGBOOST_MM_PREFETCH_PRESENT) +#define PREFETCH_READ_T0(addr) do {} while (0) - + namespace xgboost { namespace common { diff --git a/xgboost-arm-and-ppc.patch b/xgboost-arm-and-ppc.patch new file mode 100644 index 00000000000..81c4565554c --- /dev/null +++ b/xgboost-arm-and-ppc.patch @@ -0,0 +1,20 @@ +diff --git a/src/common/hist_util.cc b/src/common/hist_util.cc +index 8e13403..9a68e88 100644 +--- a/src/common/hist_util.cc ++++ b/src/common/hist_util.cc +@@ -17,14 +17,7 @@ + #include "quantile.h" + #include "./../tree/updater_quantile_hist.h" + +-#if defined(XGBOOST_MM_PREFETCH_PRESENT) +- #include +- #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast(addr), _MM_HINT_T0) +-#elif defined(XGBOOST_BUILTIN_PREFETCH_PRESENT) +- #define PREFETCH_READ_T0(addr) __builtin_prefetch(reinterpret_cast(addr), 0, 3) +-#else // no SW pre-fetching available; PREFETCH_READ_T0 is no-op +- #define PREFETCH_READ_T0(addr) do {} while (0) +-#endif // defined(XGBOOST_MM_PREFETCH_PRESENT) ++#define PREFETCH_READ_T0(addr) do {} while (0) + + namespace xgboost { + namespace common { diff --git a/xgboost-toolfile.spec b/xgboost-toolfile.spec new file mode 100644 index 00000000000..cfdf1299961 --- /dev/null +++ b/xgboost-toolfile.spec @@ -0,0 +1,24 @@ +### RPM external xgboost-toolfile 1.0 +Requires: xgboost + +%prep + +%build + +%install +mkdir -p %i/etc/scram.d +cat << \EOF_TOOLFILE >%i/etc/scram.d/xgboost.xml + + + + + + + + + + + +EOF_TOOLFILE + +## IMPORT scram-tools-post diff --git a/xgboost.spec b/xgboost.spec new file mode 100644 index 00000000000..46c36d6c0b0 --- /dev/null +++ b/xgboost.spec @@ -0,0 +1,23 @@ +### RPM external xgboost 1.3.3 + +BuildRequires: cmake +Source: git+https://github.com/dmlc/xgboost.git?obj=master/v%{realversion}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz +Patch0: xgboost-arm-and-ppc + +%prep +%setup -q -n %{n}-%{realversion} +%ifnarch x86_64 +%patch0 -p1 +%endif + +%build +rm -rf ../build; mkdir ../build; cd ../build +cmake ../%{n}-%{realversion} \ + -DCMAKE_INSTALL_PREFIX=%{i} \ + -DCMAKE_BUILD_TYPE=Release \ + -DUSE_CUDA=OFF + +%install +cd ../build +make %{makeprocesses} +make install