[TF] Update TF to version 2.1 using externals from cms distribution

bazel-0.29.1-patches.patch

@@ -0,0 +1,55 @@
+diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
+index 4426ad4c5a..7c64f42381 100644
+--- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
++++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
+@@ -112,7 +112,7 @@ public class LocalSpawnRunner implements SpawnRunner {
+     this.localExecutionOptions = Preconditions.checkNotNull(localExecutionOptions);
+     this.hostName = NetUtil.getCachedShortHostName();
+     this.resourceManager = resourceManager;
+-    this.useProcessWrapper = useProcessWrapper;
++    this.useProcessWrapper = false;
+     this.localEnvProvider = localEnvProvider;
+     this.binTools = binTools;
+   }
+diff --git a/tools/cpp/unix_cc_configure.bzl b/tools/cpp/unix_cc_configure.bzl
+index ccb18a431a..1858574756 100644
+--- a/tools/cpp/unix_cc_configure.bzl
++++ b/tools/cpp/unix_cc_configure.bzl
+@@ -200,6 +200,8 @@ def _find_gold_linker_path(repository_ctx, cc):
+         for flag in line.split(" "):
+             if flag.find("gold") == -1:
+                 continue
++            if flag.find("enable-gold") > -1 or flag.find("with-plugin-ld") > -1:
++                continue
+
+             # flag is '-fuse-ld=gold' for GCC or "/usr/lib/ld.gold" for Clang
+             # strip space, single quote, and double quotes
+diff --git a/src/conditions/BUILD b/src/conditions/BUILD
+index 2b28e28..faa41a4 100644
+--- a/src/conditions/BUILD
++++ b/src/conditions/BUILD
+@@ -11,6 +11,12 @@ filegroup(
+ )
+
+ config_setting(
++    name = "linux_ppc",
++    values = {"cpu": "ppc"},
++    visibility = ["//visibility:public"],
++)
++
++config_setting(
+     name = "linux_x86_64",
+     values = {"cpu": "k8"},
+     visibility = ["//visibility:public"],
+diff --git a/third_party/BUILD b/third_party/BUILD
+index 159006d..b2c0163 100644
+--- a/third_party/BUILD
++++ b/third_party/BUILD
+@@ -526,6 +526,7 @@ UNNECESSARY_DYNAMIC_LIBRARIES = select({
+     # The .so file is an x86 one, so we can just remove it if the CPU is not x86
+     "//src/conditions:arm": "*.so *.jnilib *.dll",
+     "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll",
++    "//src/conditions:linux_ppc": "*.so *.jnilib *.dll",
+     # Play it safe -- better have a big binary than a slow binary
+     # zip -d does require an argument. Supply something bogus.
+     "//conditions:default": "*.bogusextension",

bazel.spec

@@ -1,19 +1,36 @@
-### RPM external bazel 0.11.1
+### RPM external bazel 0.29.1
 
 Source: https://github.com/bazelbuild/bazel/releases/download/%{realversion}/bazel-%{realversion}-dist.zip
-BuildRequires: java-env
+
+BuildRequires: java-env python3
+
+# For some build steps, bazel uses a process-wrapper that is executed in an empty environment.
+# Therefore, the wrapper is linked to the system library /lib64/libstdc++.so.6, and complains about
+# a missing GLIBCXX_3.4.21 version when (e.g.) used during the compilation of tensorflow python
+# modules invoked via swig. This mechanism of bazel is actually only useful in combination with its
+# remote compilation features. When disabling the process-wrapper, the local environment is taken
+# into account which is the desired behavior for us. For example, see:
+#   - https://github.com/bazelbuild/bazel/issues/4137
+#   - https://github.com/bazelbuild/bazel/issues/4510
+#   - https://github.com/tensorflow/tensorboard/issues/1611
+
+# configuration issue
+# https://github.com/bazelbuild/bazel/issues/9392
+Patch0: bazel-0.29.1-patches
+
 %prep
 
 %define __unzip unzip -d bazel-%{realversion}
 
 %setup -q -n bazel-%{realversion}
-sed -i -e 's|^## Release 0.11.1|## Release 0.9.0|' CHANGELOG.md
+
+%patch0 -p1
 
 %build
+
+export EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk"
 bash ./compile.sh
 
 %install
 mkdir %{i}/bin
 cp output/bazel %{i}/bin/.
-
-

libjpeg-turbo.spec

@@ -1,6 +1,5 @@
 ### RPM external libjpeg-turbo 2.0.2
-## INITENV SETV LIBJPEG_TURBO_SOURCE %{source0}
-## INITENV SETV LIBJPEG_TURBO_STRIP_PREFIX %{source_prefix}
+## INITENV +PATH LD_LIBRARY_PATH %{i}/lib64
 
 %define source0 https://github.com/libjpeg-turbo/libjpeg-turbo/archive/%{realversion}.tar.gz
 %define source_prefix %{n}-%{realversion}

onnxruntime.spec

@@ -1,6 +1,6 @@
 ### RPM external onnxruntime 1.0.0
-%define tag eb635bfb66dae6cf414abfd1eb94c93842f1e66e
-%define branch cms/v1.0.0
+%define tag 0f048da7774428d5fb2c4c808fc5820809ab24b8
+%define branch cms/v1.0.0_pb380
 %define github_user cms-externals
 Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz
 

pip/google-pasta.file

@@ -0,0 +1 @@
+Requires: py2-six

pip/opt-einsum.file

@@ -0,0 +1 @@
+%define source0 https://github.com/dgasmith/opt_einsum/archive/v%{realversion}.tar.gz

pip/py2-tensorboard.file

@@ -1,2 +1,2 @@
-Requires: py2-numpy
+Requires: py2-numpy py2-grpcio py2-absl-py
 %define PipPostBuild cp %{i}/bin/tensorboard %{i}/bin/tensorboard2

pip/py2-tensorflow.file

@@ -1,4 +1,4 @@
-Requires: py2-enum34 py2-tensorboard
+Requires: py2-enum34 py2-tensorboard py2-wrapt py2-functools32 py2-google-pasta py2-opt-einsum py2-tensorflow-estimator py2-scipy
 BuildRequires: tensorflow-sources
 %define PipPreBuildPy2 PIPFILE=${TENSORFLOW_SOURCES_ROOT}/tensorflow-%{realversion}-cp27-cp27mu-linux_%{_arch}.whl
 %define PipPostBuild rm -f %{i}/bin/tensorboard* ; ls %{i}/bin/* | xargs -i cp '{}' '{}2'

pip/py3-tensorboard.file

@@ -1,2 +1,2 @@
-Requires: py3-numpy
+Requires: py3-numpy py2-absl-py py2-grpcio
 %define PipPostBuild mv %{i}/bin/tensorboard %{i}/bin/tensorboard3

pip/py3-tensorflow.file

@@ -1,4 +1,4 @@
+Requires: py3-tensorboard py3-opt-einsum py2-tensorflow-estimator py2-wrapt py2-google-pasta py3-scipy
 BuildRequires: tensorflow-python3-sources
-Requires: py3-tensorboard
 %define PipPreBuildPy3 PIPFILE=${TENSORFLOW_PYTHON3_SOURCES_ROOT}/tensorflow-%{realversion}-cp36-cp36m-linux_%{_arch}.whl
 %define PipPostBuild rm -f %{i}/bin/tensorboard* ; ls %{i}/bin/* | xargs -i mv '{}' '{}3'

pip/requirements.txt

@@ -74,6 +74,7 @@ gitdb2==2.0.6
 GitPython==2.1.14 ; python_version<'3.0'
 GitPython==3.0.4 ; python_version>'3.0'
 google-common==0.0.1
+google-pasta==0.1.8
 grpcio==1.25.0
 h5py-cache==1.0
 h5py==2.10.0
@@ -104,9 +105,7 @@ jupyter_console==5.2.0 ; python_version<'3.0'
 jupyter_console==6.0.0 ; python_version>'3.0'
 jupyter_core==4.6.1
 jupyter==1.0.0
-#NO_AUTO_UPDATE:1: looks like Keras version 2.1.5 works with tensorflow 1.6.0.
-#Update it when tensorflow is updated.
-Keras==2.1.5
+Keras==2.3.1
 keras-applications==1.0.8
 keras-preprocessing==1.1.0
 kiwisolver==1.1.0
@@ -146,6 +145,8 @@ numpy==1.15.1 ; python_version<'3.0' ; platform_machine=='aarch64'
 numpy==1.15.1 ; python_version>'3.0' ; platform_machine=='aarch64'
 oamap==0.12.4
 onnx==1.6.0
+opt-einsum==2.3.2 ; python_version<'3.0'
+opt-einsum==3.1.0 ; python_version>'3.0'
 ordereddict==1.1
 packaging==19.2
 pandas==0.24.2 ; python_version<'3.0'
@@ -227,10 +228,11 @@ tables==3.5.2 ; python_version<'3.0'
 tables==3.6.1 ; python_version>'3.0'
 #NO_AUTO_UPDATE:4: Force to use tensorflow 1.6.0; this should match the version in tensorflow-sources.spec
 #tensorboard: Need separate packages for py2 and py3 to properly get the correct sources
-tensorboard==1.6.0 ; python_version<'3.0'
-tensorboard==1.6.0 ; python_version>'3.0'
-tensorflow==1.6.0 ; python_version<'3.0'
-tensorflow==1.6.0 ; python_version>'3.0'
+tensorboard==2.0.0 ; python_version<'3.0'
+tensorboard==2.0.0 ; python_version>'3.0'
+tensorflow==2.1.0 ; python_version<'3.0'
+tensorflow==2.1.0 ; python_version>'3.0'
+tensorflow-estimator==2.1.0
 termcolor==1.1.0
 terminado==0.8.3
 testpath==0.4.4

pip/tensorflow-estimator.file

@@ -0,0 +1 @@
+%define PipDownloadSourceType none

pip/tensorflow.file

@@ -1,4 +1,5 @@
-Requires: py2-funcsigs py2-protobuf py2-pbr py2-six py2-packaging py2-appdirs py2-setuptools py2-pyparsing py2-mock py2-Werkzeug
-Requires: py2-astor py2-gast py2-absl-py py2-backports py2-termcolor py2-grpcio
+## INCLUDE tensorflow-requires
+Requires: py2-funcsigs py2-pbr py2-packaging py2-appdirs py2-setuptools py2-pyparsing py2-mock py2-Werkzeug
+Requires: py2-grpcio
 %define source0 none
 %define source_file none

protobuf.spec

@@ -1,4 +1,4 @@
-### RPM external protobuf 3.5.2
+### RPM external protobuf 3.8.0
 ## INITENV SETV PROTOBUF_SOURCE %{source0}
 ## INITENV SETV PROTOBUF_STRIP_PREFIX %{source_prefix}
 #============= IMPORTANT NOTE ========================#
@@ -24,22 +24,7 @@ BuildRequires: autotools
 
 %build
 ./autogen.sh
-# Update to detect aarch64 and ppc64le
-rm -f ./config.{sub,guess}
-curl -L -k -s -o ./config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'
-curl -L -k -s -o ./config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'
-chmod +x ./config.{sub,guess}
 
-
-rm -f ./gmock/gtest/build-aux/config.{sub,guess}
-curl -L -k -s -o ./gmock/gtest/build-aux/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'
-curl -L -k -s -o ./gmock/gtest/build-aux/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'
-chmod +x ./gmock/gtest/build-aux/config.{sub,guess}
-
-rm -f ./gmock/build-aux/config.{sub,guess}
-curl -L -k -s -o ./gmock/build-aux/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'
-curl -L -k -s -o ./gmock/build-aux/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'
-chmod +x ./gmock/build-aux/config.{sub,guess}
 ./configure --prefix %{i} \
     --disable-static \
     --disable-dependency-tracking \

tensorflow-python3-sources.spec

@@ -1,159 +1,9 @@
-### RPM external tensorflow-python3-sources 1.6.0
-#Source: https://github.com/tensorflow/tensorflow/archive/v%{realversion}.tar.gz
-# NOTE: whenever the version of tensorflow changes, update it also in tensorflow-c tensorflow-cc and py2-tensorflow
-%define isslc6amd64 %(case %{cmsplatf} in (slc6_amd64_*) echo 1 ;; (*) echo 0 ;; esac)
-%define tag 6eea62c87173ad98c71f10ff2f796f6654f5b604
-%define branch cms/v%{realversion}
-%define github_user cms-externals
-Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag}&export=tensorflow-%{realversion}&output=/tensorflow-%{realversion}-%{tag}.tgz
-Patch1: tensorflow-1.6.0-eigen-backports
-Patch2: tensorflow-1.6.0-eigen-update-gemm_pack_lhs
-Patch3: tensorflow-1.6.0-eigen-rename-sigmoid
-
-BuildRequires: bazel eigen protobuf gcc
-BuildRequires: py2-setuptools java-env
+### RPM external tensorflow-python3-sources 2.1.0
+%define python_cmd python3
+%define python_env PYTHON3PATH
+%define build_type opt
+%define pythonOnly yes
+#Just to make sure that only one tensorflow-source package built at a time
 BuildRequires: tensorflow-sources
-Requires: py3-numpy python3 py2-wheel
-
-%prep
-
-%setup -q -n tensorflow-%{realversion}
-%patch1 -p1
-%patch2 -p1
-%patch3 -p1
-
-%build
-export PYTHON_BIN_PATH=`which python3`
-export TF_NEED_JEMALLOC=0
-export TF_NEED_HDFS=0
-export CC_OPT_FLAGS=-march=core2
-export CXX_OPT_FLAGS=-std=c++11
-export TF_NEED_GCP=0
-export TF_ENABLE_XLA=0
-export TF_NEED_OPENCL=0
-export TF_NEED_CUDA=0
-export TF_NEED_VERBS=0
-export TF_NEED_MKL=0
-export TF_NEED_MPI=0
-export USE_DEFAULT_PYTHON_LIB_PATH=1
-export TF_NEED_S3=0
-export TF_NEED_GDR=0
-export TF_NEED_OPENCL_SYCL=0
-export TF_SET_ANDROID_WORKSPACE=false
-export TF_NEED_KAFKA=false
-
-#temp directory
-export TEST_TMPDIR=%{_builddir}/cache/baze
-rm -rf ../build $TEST_TMPDIR
-
-./configure
-
-sed -i -e "s|@EIGEN_SOURCE@|${EIGEN_SOURCE}|;s|@EIGEN_STRIP_PREFIX@|${EIGEN_STRIP_PREFIX}|" tensorflow/workspace.bzl tensorflow/contrib/makefile/download_dependencies.sh
-sed -i -e "s|@PROTOBUF_SOURCE@|${PROTOBUF_SOURCE}|;s|@PROTOBUF_STRIP_PREFIX@|${PROTOBUF_STRIP_PREFIX}|" tensorflow/workspace.bzl tensorflow/contrib/makefile/download_dependencies.sh
-bazel --output_user_root ../build fetch "tensorflow:libtensorflow_cc.so"
-
-#This is needed on SLC6 because the version of glibc is old
-%if %isslc6amd64
-sed -i -e 's| linkopts=\[\],| linkopts=["-lrt"],|' ../build/*/external/org_tensorflow/tensorflow/tensorflow.bzl
-sed -i -e 's|"-z defs",|"-z defs","-lrt",|' ../build/*/external/org_tensorflow/tensorflow/BUILD
-%endif
-
-sed -i -e 's|executable=ctx.executable._swig,|env=ctx.configuration.default_shell_env, executable=ctx.executable._swig,|' ../build/*/external/org_tensorflow/tensorflow/tensorflow.bzl
-sed -i -e 's|mnemonic="ProtoCompile",|env=ctx.configuration.default_shell_env, mnemonic="ProtoCompile",|' ../build/*/external/protobuf_archive/protobuf.bzl
-
-bazel --output_user_root ../build build -s --verbose_failures -c opt --cxxopt=$CXX_OPT_FLAGS tensorflow:libtensorflow_cc.so
-bazel --output_user_root ../build build -s --verbose_failures -c opt --cxxopt=$CXX_OPT_FLAGS //tensorflow/tools/pip_package:build_pip_package
-bazel --output_user_root ../build build -s --verbose_failures -c opt --cxxopt=$CXX_OPT_FLAGS //tensorflow/tools/lib_package:libtensorflow
-bazel --output_user_root ../build build -s --verbose_failures -c opt --cxxopt=$CXX_OPT_FLAGS //tensorflow/python/tools:tools_pip
-bazel --output_user_root ../build build -s --verbose_failures -c opt --cxxopt=$CXX_OPT_FLAGS //tensorflow/tools/graph_transforms:transform_graph
-
-bazel shutdown
-
-#Copying out what was built by bazel
-incdir="$PWD/tensorflow_cc/include"
-libdir="$PWD/tensorflow_cc/lib"
-
-# Make directory and clean it
-mkdir -p $incdir
-mkdir -p $libdir
-
-rm -rf $incdir/*
-rm -rf $libdir/*
-
-cp -v $PWD/bazel-bin/tensorflow/libtensorflow_cc.so $libdir
-cp -v $PWD/bazel-bin/tensorflow/libtensorflow_framework.so $libdir
-
-#Download depencies used by tensorflow and copy to include dir
-tensorflow/contrib/makefile/download_dependencies.sh
-tdir=$PWD
-dwnldir=$PWD/tensorflow/contrib/makefile/downloads
-gendir=$PWD/bazel-genfiles
-
-# tensorflow headers
-cd ${tdir}
-header_list=`find tensorflow -type f -name "*.h" | grep -v contrib`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# generated headers
-cd ${gendir}
-header_list=`find tensorflow -type f -name "*.h"`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# third party headers
-cd ${tdir}
-header_list=`find third_party -type f -name "*.h" | grep -v contrib`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# third party eigen headers
-header_list=`find third_party/eigen3 -type f | grep -v contrib`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# downloaded headers
-cd ${dwnldir}
-header_list=`find gemmlowp googletest re2 -type f -name "*.h"`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# downloaded eigen headers
-header_list=`find eigen/Eigen eigen/unsupported -type f`
-for my_header in ${header_list}
-do
-  my_header_dir=$(dirname "${my_header}")
-  mkdir -p ${incdir}/${my_header_dir}
-  cp -p ${my_header} ${incdir}/${my_header_dir}
-done
-# downloaded nsync headers
-header_list=`find nsync/public -name '*.h' -type f`
-for my_header in ${header_list}
-do
-  cp -p ${my_header} ${incdir}/
-done
-# eigen signature file
-cp -p eigen/signature_of_eigen3_matrix_library ${incdir}/eigen/ || exit 1
-
-%install
-
-bazel-bin/tensorflow/tools/pip_package/build_pip_package %{i}
-
-cp $PWD/bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz %{i}
+## INCLUDE tensorflow-sources
 
-tar cfz %{i}/libtensorflow_cc.tar.gz tensorflow_cc/.

tensorflow-requires.file

@@ -0,0 +1,5 @@
+Requires: python python3 py2-numpy py2-enum34 py2-mock py2-wheel
+Requires: py2-keras-applications py2-keras-preprocessing py2-setuptools py2-future py2-wrapt py2-gast
+Requires: py2-cython py2-googlePackages py2-functools32 py2-astor py2-six py2-termcolor py2-absl-py
+Requires: py2-backports py2-opt-einsum py3-opt-einsum
+Requires: eigen protobuf zlib libpng libjpeg-turbo curl pcre giflib sqlite swig