Merge pull request #6674 from cms-sw/tf-241

Update Tensorflow 2.4.1

cmssw-tool-conf.spec

@@ -27,6 +27,7 @@ Requires: evtgen-toolfile
 Requires: expat-toolfile
 Requires: fakesystem
 Requires: fastjet-toolfile
+Requires: flatbuffers-toolfile
 Requires: fmt-toolfile
 Requires: gbl-toolfile
 Requires: gcc-toolfile

eigen.spec

@@ -1,21 +1,14 @@
-### RPM external eigen d812f411c3f9
+### RPM external eigen 011e0db31d1bed8b7f73662be6d57d9f30fa457a
 ## INITENV +PATH PKG_CONFIG_PATH %{i}/share/pkgconfig
-## INITENV SETV EIGEN_SOURCE %{source0}
-## INITENV SETV EIGEN_STRIP_PREFIX %{source_prefix}
 ## NOCOMPILER
-%define tag b20a61c3a0dc9a79790cd258130a99b574662272
-
-# These are needed by Tensorflow sources
-# NOTE: Never apply any patch in the spec file, this way tensorflow gets the exact same sources
-%define source0 https://github.com/cms-externals/eigen-git-mirror/archive/%{tag}.tar.gz
-%define source_prefix eigen-git-mirror-%{tag}
-Source: %{source0}
+%define tag fd0c254c0a9adc47c094257ce6256ab4d94b956e
+%define branch cms/master/%{realversion}
+%define github_user cms-externals
+Source: git+https://github.com/%{github_user}/eigen-git-mirror.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz
 BuildRequires: cmake
-Patch0: eigen-altivec-fix
 
 %prep
-%setup -n %{source_prefix}
-%patch0 -p1
+%setup -n %{n}-%{realversion}
 
 %build
 mkdir build

flatbuffers-toolfile.spec

@@ -0,0 +1,22 @@
+### RPM external flatbuffers-toolfile 1.0
+Requires: flatbuffers
+%prep
+
+%build
+
+%install
+
+mkdir -p %i/etc/scram.d
+cat << \EOF_TOOLFILE >%i/etc/scram.d/flatbuffers.xml
+<tool name="flatbuffers" version="@TOOL_VERSION@">
+  <lib name="flatbuffers"/>
+  <client>
+    <environment name="FLATBUFFERS_BASE" default="@TOOL_ROOT@"/>
+    <environment name="INCLUDE"          default="$FLATBUFFERS_BASE/include"/>
+    <environment name="LIBDIR"           default="$FLATBUFFERS_BASE/lib64"/>
+  </client>
+  <runtime name="PATH" value="$FLATBUFFERS_BASE/bin" type="path"/>
+</tool>
+EOF_TOOLFILE
+
+## IMPORT scram-tools-post

flatbuffers.spec

@@ -0,0 +1,31 @@
+### RPM external flatbuffers 1.12.0
+## INITENV +PATH LD_LIBRARY_PATH %{i}/lib64
+
+%define tag v%{realversion}
+%define branch master
+%define github_user google
+Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz
+
+BuildRequires: cmake gmake
+
+%prep
+%setup -n %{n}-%{realversion}
+
+%build
+rm -rf ../build
+mkdir ../build
+cd ../build
+
+cmake ../%{n}-%{realversion} \
+   -DCMAKE_BUILD_TYPE=Release \
+  -DFLATBUFFERS_BUILD_CPP17=ON \
+  -DFLATBUFFERS_BUILD_SHAREDLIB=ON \
+  -DFLATBUFFERS_BUILD_TESTS=OFF \
+  -DCMAKE_INSTALL_PREFIX="%{i}"
+
+make -v %{makeprocesses}
+
+%install
+cd ../build
+
+make %{makeprocesses} install

opencv.spec

@@ -1,4 +1,4 @@
-### RPM external opencv 4.3.0
+### RPM external opencv 4.5.1
 ## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES}
 ## INITENV +PATH PYTHON3PATH %{i}/${PYTHON3_LIB_SITE_PACKAGES}
 %define tag %{realversion}
@@ -21,6 +21,9 @@ cmake ../%{n}-%{realversion} \
     -GNinja \
     -DCMAKE_INSTALL_PREFIX="%{i}" \
     -DCMAKE_INSTALL_LIBDIR=lib \
+%ifarch ppc64le
+    -DWITH_EIGEN=OFF \
+%endif
     -DPYTHON2_EXECUTABLE:FILEPATH="${PYTHON_ROOT}/bin/python" \
     -DPYTHON2_INCLUDE_DIR:PATH="${PYTHON_ROOT}/include/python2.7" \
     -DPYTHON2_LIBRARY:FILEPATH="${PYTHON_ROOT}/lib/libpython2.7.so" \

pip/flatbuffers.file

@@ -0,0 +1 @@
+%define PipDownloadSourceType none

pip/google-auth-oauthlib.file

@@ -0,0 +1 @@
+Requires: py3-google-auth py3-requests-oauthlib

pip/google-auth.file

@@ -0,0 +1 @@
+Requires: py3-cachetools py3-pyasn1-modules py3-rsa py2-six

pip/py3-tensorboard.file

@@ -1,2 +1,3 @@
 Requires: py3-numpy py2-absl-py py2-grpcio
+Requires: py3-google-auth-oauthlib py3-tensorboard-plugin-wit
 %define PipPostBuild mv %{i}/bin/tensorboard %{i}/bin/tensorboard3

pip/py3-tensorflow.file

@@ -1,4 +1,4 @@
 Requires: py3-tensorboard py3-opt-einsum py2-tensorflow-estimator py2-wrapt py2-google-pasta py3-scipy
-BuildRequires: tensorflow-python3-sources
-%define PipPreBuildPy3 PIPFILE=${TENSORFLOW_PYTHON3_SOURCES_ROOT}/tensorflow-%{realversion}-cp38-cp38-linux_%{_arch}.whl
+BuildRequires: tensorflow-sources
+%define PipPreBuildPy3 PIPFILE=${TENSORFLOW_SOURCES_ROOT}/tensorflow-%{realversion}-cp38-cp38-linux_%{_arch}.whl
 %define PipPostBuild rm -f %{i}/bin/tensorboard* ; ls %{i}/bin/* | xargs -i mv '{}' '{}3'

pip/requests-oauthlib.file

@@ -0,0 +1 @@
+Requires: py3-oauthlib py2-requests

pip/requirements.txt

@@ -86,6 +86,7 @@ fire==0.4.0
 flit==3.0.0; python_version>'3.0'
 flit-core==3.0.0; python_version>'3.0'
 flake8==3.8.4
+flatbuffers==1.12.0 ; python_version>'3.0'
 flawfinder==2.0.15
 fs==2.4.12
 funcsigs==1.0.2
@@ -97,6 +98,8 @@ gitdb2==2.0.6 ; python_version<'3.0'
 gitdb==4.0.5 ; python_version>'3.0'
 GitPython==2.1.15 ; python_version<'3.0'
 GitPython==3.1.12 ; python_version>'3.0'
+google-auth==1.27.0 ; python_version>'3.0'
+google-auth-oauthlib==0.4.2 ; python_version>'3.0'
 google-common==0.0.1
 google-pasta==0.2.0
 grpcio==1.34.1
@@ -142,7 +145,7 @@ jupyter==1.0.0
 jupyterlab-widgets==1.0.0 ; python_version>'3.0'
 Keras==2.4.3
 keras-applications==1.0.8
-keras-preprocessing==1.1.0
+keras-preprocessing==1.1.2
 keras2onnx==1.7.0 ; python_version>'3.0'
 keyring==21.8.0 ; python_version>'3.0'
 kiwisolver==1.1.0
@@ -192,6 +195,7 @@ numpy==1.17.5 ; python_version>'3.0'
 onnx==1.8.1
 onnxmltools==1.7.0 ; python_version>'3.0'
 onnxconverter-common==1.7.0 ; python_version>'3.0'
+oauthlib==3.1.0 ; python_version>'3.0'
 opt-einsum==2.3.2 ; python_version<'3.0'
 opt-einsum==3.3.0 ; python_version>'3.0'
 ordereddict==1.1
@@ -225,6 +229,7 @@ psutil==5.8.0
 ptyprocess==0.7.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
+pyasn1-modules==0.2.8 ; python_version>'3.0'
 pybind11==2.6.2
 pybrain==0.3.3
 pycodestyle==2.6.0
@@ -265,9 +270,11 @@ repoze-lru==0.7
 rep==0.6.6
 requests==2.25.1
 requests-toolbelt==0.9.1 ; python_version> '3.0'
+requests-oauthlib==1.3.0 ; python_version> '3.0'
 root_numpy==4.8.0
 root_pandas==0.7.0
 rootpy==1.0.1
+rsa==4.7.2 ; python_version>'3.0'
 scandir==1.10.0
 schema==0.7.3
 scikit-learn==0.20.4 ; python_version<'3.0'
@@ -295,13 +302,11 @@ sympy==1.5.1 ; python_version<'3.0'
 sympy==1.7.1 ; python_version>'3.0'
 tables==3.5.2 ; python_version<'3.0'
 tables==3.6.1 ; python_version>'3.0'
-#NO_AUTO_UPDATE:4: Force to use tensorflow 1.6.0; this should match the version in tensorflow-sources.spec
-#tensorboard: Need separate packages for py2 and py3 to properly get the correct sources
-tensorboard==2.0.0 ; python_version<'3.0'
-tensorboard==2.0.0 ; python_version>'3.0'
-tensorflow==2.3.1 ; python_version<'3.0'
-tensorflow==2.3.1 ; python_version>'3.0'
-tensorflow-estimator==2.1.0
+#NO_AUTO_UPDATE:2: Force to use tensorflow 2.4.1; this should match the version in tensorflow-sources.spec
+tensorboard==2.4.1 ; python_version>'3.0'
+tensorboard-plugin-wit==1.8.0 ; python_version>'3.0'
+tensorflow==2.4.1 ; python_version>'3.0'
+tensorflow-estimator==2.4.0
 termcolor==1.1.0
 terminado==0.8.3
 testpath==0.4.4 ; python_version<'3.0'
@@ -333,7 +338,6 @@ wheel==0.33.6
 widgetsnbextension==3.5.1
 #cannot wrapt update before astroid is
 wrapt==1.11.2
-# xgboost version for python 3 must match the one we install as a library
 xgboost==0.82 ; python_version<'3.0'
 xgboost==1.3.3 ; python_version>'3.0'
 #bumping this pulls in xrootd - which looks like it needs some understanding

pip/rsa.file

@@ -0,0 +1 @@
+Requires: py2-pyasn1

pip/tensorboard-plugin-wit.file

@@ -0,0 +1 @@
+%define PipDownloadSourceType none

python_tools.spec

@@ -10,7 +10,7 @@ Requires: py2-Keras
 Requires: py2-Theano
 Requires: py2-scikit-learn
 #save for the end
-Requires: py2-tensorflow py3-tensorflow
+Requires: py3-tensorflow
 Requires: py2-googlePackages
 Requires: py2-cmsml
 Requires: py2-law

tensorflow-requires.file

@@ -1,5 +1,5 @@
-Requires: python python3 py2-numpy py2-enum34 py2-mock py2-wheel
+Requires: python python3 py2-numpy py2-enum34 py2-mock py2-wheel py2-typing py2-typing_extensions
 Requires: py2-keras-applications py2-keras-preprocessing py2-setuptools py2-future py2-wrapt py2-gast py3-setuptools
 Requires: py2-cython py2-googlePackages py2-functools32 py2-astor py2-six py2-termcolor py2-absl-py
-Requires: py2-backports py2-opt-einsum py3-opt-einsum
-Requires: eigen protobuf zlib libpng libjpeg-turbo curl pcre giflib sqlite swig
+Requires: py2-backports py2-opt-einsum py3-opt-einsum py3-flatbuffers
+Requires: eigen protobuf zlib libpng libjpeg-turbo curl pcre giflib sqlite grpc flatbuffers

tensorflow-sources.file

@@ -9,13 +9,13 @@
 #   and update eigen-toolfile.spec accordingly
 ############################################
 
-BuildRequires: bazel swig java-env git
+BuildRequires: bazel java-env git
 #Keep all requires in separate file, so that can be included in py-tensorflow too
 ## INCLUDE tensorflow-requires
 ## INCLUDE compilation_flags
 
-%define tag         77e1c215529a442d2c9268c62f010dcf30d90cc4
-%define branch      cms/v2.3.1
+%define tag         050756d3ddbf0692b74451f6ba3d75d54d345d9e
+%define branch      cms/v%{realversion}
 %define github_user cms-externals
 Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag}&export=tensorflow-%{realversion}&output=/tensorflow-%{realversion}.tgz
 
@@ -37,16 +37,21 @@ Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag
 export PYTHON_BIN_PATH="$(which %{python_cmd})"
 export USE_DEFAULT_PYTHON_LIB_PATH=1
 export GCC_HOST_COMPILER_PATH="$(which gcc)"
+export CC_OPT_FLAGS="-Wno-sign-compare"
+
+BAZEL_OPTS="--batch --output_user_root ../build build -s --verbose_failures --distinct_host_configuration=false"
 %ifarch x86_64
-export CC_OPT_FLAGS="%{vectorize_flag}"
+BAZEL_OPTS="$BAZEL_OPTS --copt=%{vectorize_flag}"
 %else
 %ifarch ppc64le
-export CC_OPT_FLAGS="-mcpu=native -mtune=native --param=l1-cache-size=64 --param=l1-cache-line-size=128 --param=l2-cache-size=512"
+BAZEL_OPTS="$BAZEL_OPTS --copt=-mcpu=native --copt=-mtune=native"
+BAZEL_OPTS="$BAZEL_OPTS --copt=--param=l1-cache-size=64 --copt=--param=l1-cache-line-size=128 --copt=--param=l2-cache-size=512"
 %else
-export CC_OPT_FLAGS="-mcpu=native -mtune=native"
+BAZEL_OPTS="$BAZEL_OPTS --copt=-mcpu=native --copt=-mtune=native"
 %endif
 %endif
-export CXX_OPT_FLAGS="-std=c++17"
+BAZEL_OPTS="$BAZEL_OPTS --config=%{build_type} --cxxopt=-std=c++17 %{makeprocesses}"
+BAZEL_OPTS="$BAZEL_OPTS --config=noaws --config=nogcp --config=nohdfs --config=nonccl"
 
 export TF_NEED_JEMALLOC=0
 export TF_NEED_HDFS=0
@@ -73,19 +78,22 @@ export TF_CMS_EXTERNALS="%{_builddir}/cms_externals.txt"
 echo "png:${LIBPNG_ROOT}"                   >  ${TF_CMS_EXTERNALS}
 echo "libjpeg_turbo:${LIBJPEG_TURBO_ROOT}"  >> ${TF_CMS_EXTERNALS}
 echo "zlib:${ZLIB_ROOT}"                    >> ${TF_CMS_EXTERNALS}
+echo "eigen_archive:${EIGEN_ROOT}"          >> ${TF_CMS_EXTERNALS}
 echo "curl:${CURL_ROOT}"                    >> ${TF_CMS_EXTERNALS}
 echo "com_google_protobuf:${PROTOBUF_ROOT}" >> ${TF_CMS_EXTERNALS}
+echo "com_github_grpc_grpc:${GRPC_ROOT}"    >> ${TF_CMS_EXTERNALS}
 echo "pcre:${PCRE_ROOT}"                    >> ${TF_CMS_EXTERNALS}
 echo "gif:${GIFLIB_ROOT}"                   >> ${TF_CMS_EXTERNALS}
 echo "org_sqlite:${SQLITE_ROOT}"            >> ${TF_CMS_EXTERNALS}
-echo "swig:"                                >> ${TF_CMS_EXTERNALS}
 echo "cython:"                              >> ${TF_CMS_EXTERNALS}
+echo "flatbuffers:${FLATBUFFERS_ROOT}"      >> ${TF_CMS_EXTERNALS}
 echo "functools32_archive:"                 >> ${TF_CMS_EXTERNALS}
 echo "enum34_archive:"                      >> ${TF_CMS_EXTERNALS}
 echo "astor_archive:"                       >> ${TF_CMS_EXTERNALS}
 echo "six_archive:"                         >> ${TF_CMS_EXTERNALS}
 echo "absl_py:"                             >> ${TF_CMS_EXTERNALS}
 echo "termcolor_archive:"                   >> ${TF_CMS_EXTERNALS}
+echo "typing_extensions_archive:"           >> ${TF_CMS_EXTERNALS}
 echo "pasta:"                               >> ${TF_CMS_EXTERNALS}
 echo "wrapt:"                               >> ${TF_CMS_EXTERNALS}
 echo "gast_archive:"                        >> ${TF_CMS_EXTERNALS}
@@ -116,12 +124,6 @@ else
 fi
 export PATH=%{_builddir}/cms-pytool:$PATH
 
-# define bazel options
-BAZEL_OPTS="--batch --output_user_root ../build build -s --verbose_failures --config=opt --cxxopt=$CXX_OPT_FLAGS %{makeprocesses}"
-BAZEL_OPTS="$BAZEL_OPTS --config=noaws --config=nogcp --config=nohdfs --config=nonccl"
-BAZEL_OPTS="$BAZEL_OPTS --action_env=SWIG_LIB=${SWIG_LIB}"
-BAZEL_OPTS="$BAZEL_OPTS --distinct_host_configuration=false"
-
 # build tensorflow python targets
 bazel $BAZEL_OPTS //tensorflow/tools/pip_package:build_pip_package
 
@@ -141,7 +143,7 @@ bazel $BAZEL_OPTS //tensorflow:install_headers
 chmod -R a+rwX $PWD/bazel-bin/tensorflow/include
 for f in $(find tensorflow -name "*.proto")
 do
-    protoc --cpp_out=$PWD/bazel-bin/tensorflow/include $f
+  protoc --cpp_out=$PWD/bazel-bin/tensorflow/include $f
 done
 %endif
 

tensorflow-sources.spec

@@ -1,9 +1,8 @@
-### RPM external tensorflow-sources 2.3.1
-%define python_cmd python
-%define python_env PYTHON27PATH
+### RPM external tensorflow-sources 2.4.1
+%define python_cmd python3
+%define python_env PYTHON3PATH
 %define build_type opt
 %define pythonOnly no
 %define vectorize_flag -msse3
-Requires: py2-futures
 ## INCLUDE tensorflow-sources
 

tensorflow.spec

@@ -1,8 +1,9 @@
-### RPM external tensorflow 2.3.1
+### RPM external tensorflow 2.4.1
 %define source_package tensorflow-sources
 %if "%{?vectorized_package:set}" != "set"
 BuildRequires: %{source_package}
 %define tf_root %(echo %{source_package}_ROOT | tr '[a-z-]' '[A-Z_]')
+%define tf_version %(echo %{source_package}_VERSION | tr '[a-z-]' '[A-Z_]')
 %else
 BuildRequires: %{source_package}_%{vectorized_package}
 %define tf_root %(echo %{source_package}_%{vectorized_package}_ROOT | tr '[a-z-]' '[A-Z_]')
@@ -11,6 +12,13 @@ Provides: libtensorflow_cc.so(tensorflow)(64bit)
 Source: none
 
 %prep
+case ${%{tf_version}} in
+  %{realversion}|%{realversion}-*) ;;
+  * ) echo "ERROR: Mismatch %{n} (%{realversion}) and %{source_package} (${%{tf_version}}) versions."
+      echo "Please update %{n}.spec to use ${%{tf_version}} verison."
+      exit 1
+      ;;
+esac
 
 %build