sarkars/Upgrade TF

CMakeLists.txt

@@ -100,7 +100,7 @@ endif()
 #       libngraph_bridge.so
 #       ...
 #     /tensorflow
-#       libtensorflow_framework.so
+#       libtensorflow_framework.so.1
 #       python/
 #           _pywrap....so
 # Therefore we are setting two entries in the RPATH:
@@ -119,12 +119,7 @@ endif()
 # Find TensorFlow
 find_package(TensorFlow REQUIRED)
 
-add_library(tensorflow_framework_lib SHARED IMPORTED)
-set_target_properties(
-    tensorflow_framework_lib
-    PROPERTIES IMPORTED_LOCATION
-    ${TensorFlow_DIR}/libtensorflow_framework.so
-)
+
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(NGRAPH_TF_CXX11_ABI "${TensorFlow_CXX_ABI}")

bazel/tf_configure/BUILD.tpl

@@ -9,8 +9,7 @@ cc_library(
 
 cc_library(
     name = "libtensorflow_framework",
-    srcs = [":libtensorflow_framework.so"],
-    #data = ["lib/libtensorflow_framework.so"],
+    srcs = [":libtensorflow_framework.so.1"],
     visibility = ["//visibility:public"],
 )
 

bazel/tf_configure/tf_configure.bzl

@@ -182,14 +182,14 @@ def _tf_pip_impl(repository_ctx):
     )
 
     tf_shared_library_dir = repository_ctx.os.environ[_TF_SHARED_LIBRARY_DIR]
-    tf_shared_library_path = "%s/libtensorflow_framework.so" % tf_shared_library_dir
+    tf_shared_library_path = "%s/libtensorflow_framework.so.1" % tf_shared_library_dir
     tf_shared_library_rule = _symlink_genrule_for_dir(
         repository_ctx,
         None,
         "",
-        "libtensorflow_framework.so",
+        "libtensorflow_framework.so.1",
         [tf_shared_library_path],
-        ["libtensorflow_framework.so"],
+        ["libtensorflow_framework.so.1"],
     )
 
     _tpl(repository_ctx, "BUILD", {

build_ngtf.py

@@ -25,7 +25,7 @@ def main():
 
     # Component versions
     ngraph_version = "v0.20.0-rc.0"
-    tf_version = "v1.13.1"
+    tf_version = "v1.14.0-rc0"
 
     # Command line parser options
     parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter)

cmake/FindTensorFlow.cmake

@@ -101,9 +101,17 @@ endif()
 message(STATUS "TensorFlow_VERSION: " ${TensorFlow_VERSION})
 
 # Make sure that the TF library exists
+if ( APPLE )
+    set(TF_LIB_NAME libtensorflow_framework.dylib)
+else()
+    set(TF_LIB_NAME libtensorflow_framework.so.1)
+endif()
+
+message(STATUS "TF_LIB: " ${TF_LIB_NAME})
+
 find_library(
   TensorFlow_FRAMEWORK_LIBRARY
-  NAME tensorflow_framework
+  NAME ${TF_LIB_NAME}
   PATHS ${TensorFlow_DIR}
   NO_DEFAULT_PATH
 )

configure_bazel.sh

@@ -25,7 +25,7 @@ rm -f .bazelrc
 if python -c "import tensorflow" &> /dev/null; then
     echo 'using installed tensorflow'
 else
-    pip install tensorflow==1.13.1
+    pip install tensorflow==v1.14.0-rc0
     pip install tensorflow_estimator
 fi
 

logging/CMakeLists.txt

@@ -31,7 +31,7 @@ set(SRC
 add_library(${LIB_NAME} ${SRC})
 target_link_libraries( 
     ${LIB_NAME} 
-    tensorflow_framework_lib 
+    ${TensorFlow_FRAMEWORK_LIBRARY} 
 )
 
 target_include_directories(${LIB_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")

src/CMakeLists.txt

@@ -91,7 +91,7 @@ message(STATUS "NGRAPH_TF_USE_GRAPPLER_OPTIMIZER: ${NGRAPH_TF_USE_GRAPPLER_OPTIM
 add_library(${LIB_NAME} SHARED ${SRC})
 
 target_link_libraries( ${LIB_NAME} ngraph_logger) 
-target_link_libraries( ${LIB_NAME} tensorflow_framework_lib )
+target_link_libraries( ${LIB_NAME} ${TensorFlow_FRAMEWORK_LIBRARY})
 target_link_libraries( ${LIB_NAME} ngraph_lib )
 target_link_libraries( ${LIB_NAME} absl_algorithm )
 target_link_libraries( ${LIB_NAME} absl_container )

src/ngraph_encapsulate_op.cc

@@ -106,7 +106,7 @@ class NGraphEncapsulateOp : public OpKernel {
           errors::Internal("Did not find graphdef for encapsulate ", flib_key,
                            " in NGraphClusterManager or function library"));
       // TODO: how to convert from functiondef to graphdef. Anything easier?
-      FunctionBody* fnbody;
+      std::unique_ptr<FunctionBody> fnbody;
       const auto get_func_sig = [&flib](const string& op, const OpDef** sig) {
         return flib.LookUpOpDef(op, sig);
       };

src/ngraph_freshness_tracker.h

@@ -89,7 +89,7 @@ class NGraphFreshnessTracker : public ResourceBase {
   NGraphFreshnessTracker(const NGraphFreshnessTracker&) = delete;
   NGraphFreshnessTracker& operator=(const NGraphFreshnessTracker&) = delete;
 
-  std::string DebugString() override { return "FreshnessTracker"; }
+  std::string DebugString() const override { return "FreshnessTracker"; }
 
   // If freshness_map_ has the base_pointer, then inserts the user function into
   // its set of user functions

src/ngraph_tracked_variable.cc

@@ -53,7 +53,7 @@ class NGraphVar : public ResourceBase {
   mutex* mu() { return &mu_; }
   Tensor* tensor() { return &tensor_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(DataTypeString(tensor_.dtype()), "/",
                            tensor_.shape().DebugString());
   }

test/CMakeLists.txt

@@ -36,7 +36,7 @@ add_library(tensorflow_cc_lib SHARED IMPORTED)
 set_target_properties(
     tensorflow_cc_lib
     PROPERTIES IMPORTED_LOCATION
-    ${TensorFlow_SRC_DIR}/bazel-bin/tensorflow/libtensorflow_cc.so
+    ${TensorFlow_SRC_DIR}/bazel-bin/tensorflow/libtensorflow_cc.so.1
 )
 
 set(SRC
@@ -94,7 +94,7 @@ target_link_libraries(
     ngraph_lib
     libgtest
     pthread
-    tensorflow_framework_lib
+    ${TensorFlow_FRAMEWORK_LIBRARY}
     tensorflow_cc_lib
 )
 

test/ci/buildkite/ngtf-cpu_centos-grappler.yaml

@@ -27,7 +27,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
         --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_bazel
 
@@ -52,7 +52,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
 
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_centos.yaml

@@ -40,7 +40,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
         --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_bazel
 
@@ -65,7 +65,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
 
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu.yaml

@@ -40,7 +40,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
         --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_bazel
 
@@ -65,7 +65,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
 
     label: ":gear: Install"

test/ci/buildkite/ngtf-cpu_ubuntu_18_04.yaml

@@ -0,0 +1,105 @@
+  - command: |
+      rm -rf /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID
+      virtualenv -p /usr/bin/python3 /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv 
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install -U yapf==0.26.0
+
+    label: ":gear: Setup"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+    parallelism: 1
+  - wait
+
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      export PATH=/opt/llvm-3.9.0/bin/:$$PATH 
+      maint/check-code-format.sh
+
+    label: ":pencil: Code Format ?"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+    parallelism: 1
+  - wait
+
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      python3 build_ngtf.py --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID
+
+    label: ":hammer_and_wrench: Build"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+    parallelism: 1
+  - wait
+
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install -U \
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
+      PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
+        --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_bazel
+
+    label: ":bazel: Bazel Build"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - wait
+
+  - command: |
+      PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
+        --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_cpp
+
+    label: ":chrome: C++ Unit Test"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - wait 
+
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install psutil && pip install -U \
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
+      pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
+
+    label: ":gear: Install"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - wait
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install pytest
+      PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
+        --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_python
+
+    label: ":python: nGraph Pytest"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install pytest
+      PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
+        --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_tf_python
+
+    label: ":python: TensorFlow Pytest"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \
+        --artifacts /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID --test_resnet
+    label: ":bar_chart: ResNet50"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu_tf_1.14"
+  - wait
+  - command: |
+      rm -rf /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID
+    label: ":wastebasket: Cleanup"
+    agents:
+    - "queue=cpu_tf_1.14"
+

test/ci/buildkite/ngtf-gpu_ubuntu.yaml

@@ -40,7 +40,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       pip install psutil && pip install -U \
-        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.13.1-cp??-cp??m-linux_x86_64.whl
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp??-cp??m-linux_x86_64.whl
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
 
     label: ":gear: Install"

test/ci/docker/dockerfiles/Dockerfile.ngraph-tf-ci-py2

@@ -77,8 +77,9 @@ RUN pip install --upgrade pytest
 RUN apt-get update && apt-get install -y openjdk-8-jdk
 #
 # This bazel version works with current TF
-RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb
-RUN dpkg -i bazel_0.21.0-linux-x86_64.deb || true
+ARG BAZEL_VERSION=0.24.1
+RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb
+RUN dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb || true
 
 # Copy in the run-as-user.sh script
 # This will allow the builds, which are done in a mounted directory, to

test/ci/docker/dockerfiles/Dockerfile.ngraph-tf-ci-py3

@@ -79,8 +79,9 @@ RUN pip3 install --upgrade pytest
 RUN apt-get update && apt-get install -y openjdk-8-jdk
 #
 # This bazel version works with current TF
-RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb
-RUN dpkg -i bazel_0.21.0-linux-x86_64.deb || true
+ARG BAZEL_VERSION=0.24.1
+RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb
+RUN dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb || true
 
 # Copy in the run-as-user.sh script
 # This will allow the builds, which are done in a mounted directory, to

test/ci/docker/dockerfiles/Dockerfile.ngraph_tf.build_ngtf_centos76_py36

@@ -70,7 +70,7 @@ RUN pip3 install --upgrade pytest
 
 # This bazel version works with current TF
 # Install the most recent bazel release.
-ARG BAZEL_VERSION=0.21.0
+ARG BAZEL_VERSION=0.24.1
 RUN mkdir /bazel && \
     cd /bazel && \
     wget --quiet https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh && \

test/ci/docker/dockerfiles/Dockerfile.ngraph_tf.build_ngtf_ubuntu1604_gcc48_py35

@@ -84,8 +84,9 @@ RUN pip3 install --upgrade pytest
 RUN apt-get update && apt-get install -y openjdk-8-jdk
 #
 # This bazel version works with current TF
-RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb
-RUN dpkg -i bazel_0.21.0-linux-x86_64.deb || true
+ARG BAZEL_VERSION=0.24.1
+RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb
+RUN dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb || true
 
 # Copy in the run-as-user.sh script
 # This will allow the builds, which are done in a mounted directory, to

test/ci/docker/dockerfiles/Dockerfile.ngraph_tf.build_ngtf_ubuntu1604_py35

@@ -66,8 +66,9 @@ RUN pip3 install --upgrade pytest
 RUN apt-get update && apt-get install -y openjdk-8-jdk
 #
 # This bazel version works with current TF
-RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb
-RUN dpkg -i bazel_0.21.0-linux-x86_64.deb || true
+ARG BAZEL_VERSION=0.24.1
+RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb
+RUN dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb || true
 
 # Copy in the run-as-user.sh script
 # This will allow the builds, which are done in a mounted directory, to

test/ci/docker/dockerfiles/Dockerfile.ngraph_tf.build_ngtf_ubuntu1804_py36

@@ -63,8 +63,9 @@ RUN pip3 install --upgrade pytest
 RUN apt-get update && apt-get install -y openjdk-8-jdk
 
 # This bazel version works with current TF
-RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb
-RUN dpkg -i bazel_0.21.0-linux-x86_64.deb || true
+ARG BAZEL_VERSION=0.24.1
+RUN wget --no-verbose -c https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb
+RUN dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb || true
 
 # Copy in the run-as-user.sh script
 # This will allow the builds, which are done in a mounted directory, to