Fixed the build_tf.py which needed to be added the TF C++ library bui…

…ld recipe

CMakeLists.txt

@@ -234,7 +234,7 @@ if (NOT USE_PRE_BUILT_NGRAPH)
     ExternalProject_Add(
         ext_ngraph
         GIT_REPOSITORY https://github.com/NervanaSystems/ngraph
-        GIT_TAG v0.20.0-rc.1
+        GIT_TAG v0.22.0
         CMAKE_ARGS 
             -DNGRAPH_DISTRIBUTED_ENABLE=${NGRAPH_DISTRIBUTED_ENABLE}
             -DNGRAPH_INSTALL_PREFIX=${NGRAPH_ARTIFACTS_DIR}

bazel/WORKSPACE

@@ -25,11 +25,11 @@ tf_configure(
 http_archive(
     name = "ngraph",
     build_file = "//:bazel/ngraph.BUILD",
-    sha256 = "e3b448fb1329b8cf544b3eee2f82656e5b9d60f94fbdaa3448afd20475786a87",
-    strip_prefix = "ngraph-0.20.0-rc.0",
+    sha256 = "4e3c9a4ce1e5a58877ba0014ca627a1f0c32cb2dd7d142113b65771652e5f4a7",
+    strip_prefix = "ngraph-0.22.0",
     urls = [
-        "https://mirror.bazel.build/github.com/NervanaSystems/ngraph/archive/v0.20.0-rc.0-rc.0.tar.gz",
-        "https://github.com/NervanaSystems/ngraph/archive/v0.20.0-rc.0.tar.gz"
+        "https://mirror.bazel.build/github.com/NervanaSystems/ngraph/archive/v0.22.0.tar.gz",
+        "https://github.com/NervanaSystems/ngraph/archive/v0.22.0.tar.gz"
     ],
 )
 

build_ngtf.py

@@ -53,7 +53,7 @@ def main():
     '''
 
     # Component versions
-    ngraph_version = "v0.20.0-rc.1"
+    ngraph_version = "v0.22.0"
     tf_version = "v1.14.0"
 
     # Command line parser options

build_tf.py

@@ -45,11 +45,14 @@ def main():
 
     assert not os.path.isdir(
         arguments.output_dir), arguments.output_dir + " already exists"
-    os.mkdir(arguments.output_dir)
+    os.makedirs(arguments.output_dir)
     os.chdir(arguments.output_dir)
     assert not is_venv(
     ), "Please deactivate virtual environment before running this script"
 
+    assert not is_venv(
+    ), "Please deactivate virtual environment before running this script"
+
     venv_dir = './venv3/'
 
     install_virtual_env(venv_dir)
@@ -63,11 +66,21 @@ def main():
     # Build TensorFlow
     build_tensorflow(venv_dir, "tensorflow", 'artifacts', arguments.target_arch,
                      False)
-    shutil.copytree('./tensorflow/tensorflow/python',
-                    './artifacts/tensorflow/python')
-    print('To build ngraph-bridge using this prebuilt tensorflow, use:')
-    print('./build_ngtf.py --use_tensorflow_from_location ' +
-          os.path.abspath(arguments.output_dir))
+
+    # Build TensorFlow C++ Library
+    build_tensorflow_cc("tensorflow", 'artifacts', arguments.target_arch, False)
+
+    pwd = os.getcwd()
+    artifacts_dir = os.path.join(pwd, 'artifacts/tensorflow')
+    os.chdir("tensorflow")
+
+    copy_tf_to_artifacts(artifacts_dir, None)
+
+    print('\033[1;35mTensorFlow Build finished\033[0m')
+
+    print('When building ngraph-bridge using this prebuilt tensorflow, use:')
+    print('\033[3;34mpython3 build_ngtf.py --use_tensorflow_from_location ' +
+          os.path.abspath(arguments.output_dir) + '\033[1;0m')
 
 
 if __name__ == '__main__':

python/setup.in.py

@@ -59,7 +59,7 @@ def get_tag(self):
 
 setup( 
     name='ngraph_tensorflow_bridge',
-    version='0.14.0',
+    version='0.16.0rc-master',
     description='Intel nGraph compiler and runtime for TensorFlow',
     long_description=long_description,
     long_description_content_type="text/markdown",

src/ngraph_rewrite_pass.cc

@@ -242,7 +242,10 @@ class NGraphEncapsulationPass : public NGraphRewritePass {
         backend_name);  // SplitBackendConfig
     backend_name = config_map.at("ngraph_backend");
     config_map.erase("ngraph_backend");
-    NGRAPH_VLOG(0) << "NGraph using backend: " << backend_name;
+
+    if ((std::getenv("NGRAPH_TF_LOG_0_DISABLED") == nullptr)) {
+      NGRAPH_VLOG(0) << "NGraph using backend: " << backend_name;
+    }
 
     // Now Process the Graph
 

src/version.cc

@@ -22,15 +22,15 @@
 // nGraph-TensorFlow bridge uses semantic versioning: see http://semver.org/
 
 #define NG_TF_MAJOR_VERSION 0
-#define NG_TF_MINOR_VERSION 14
+#define NG_TF_MINOR_VERSION 16
 #define NG_TF_PATCH_VERSION 0
 
 // The version suffix is used for pre-release version numbers
 // For example before v0.7.0 we may do a pre-release i.e., a release
 // candidate such as v0.7.0-rc0
 // The code in master will always have the last released version number
 // with a suffix of '-master'
-#define NG_TF_VERSION_SUFFIX ""
+#define NG_TF_VERSION_SUFFIX "-master"
 
 #define VERSION_STR_HELPER(x) #x
 #define VERSION_STR(x) VERSION_STR_HELPER(x)

test/ci/buildkite/ngtf-models.yaml → test/ci/buildkite/ngtf-cpu_model-test.yaml

@@ -23,11 +23,24 @@
   - wait
 
   - command: |
-       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
-       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate
-       cd test/model_level_tests; python test_main.py --models benchmarks,MLP,ngraph-models,nmt,tfmodels,unet --run_basic_tests; cd ../..
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
+      pip install psutil && pip install -U \
+        /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-1.14.0rc0-cp36-cp36m-linux_x86_64.whl
+      pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
+      
+    label: ":gear: Install"
+    timeout_in_minutes: 30
+    agents:
+    - "queue=cpu"
+
+  - wait
+
+  - command: |
+      source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate
+      cd test/model_level_tests
+      python test_main.py --models benchmarks,MLP,ngraph-models,nmt,tfmodels,unet --run_basic_tests; cd ../..
 
-    label: ":goal_net: Network tests"
+    label: ":kafka: Model tests" 
     timeout_in_minutes: 40
     agents:
     - "queue=cpu"

test/ci/buildkite/test_runner.py

@@ -101,12 +101,14 @@ def main():
                     "NNOps.Qu*:NNOps.SoftmaxZeroDimTest*:"
                     "NNOps.SparseSoftmaxCrossEntropyWithLogits:"
                     "ArrayOps.GatherNd*")
+        os.environ['NGRAPH_TF_LOG_0_DISABLED'] = '1'
         run_ngtf_cpp_gtests(arguments.artifacts_dir, './', test_filter)
     elif (arguments.test_python):
         run_ngtf_pytests_from_artifacts(arguments.artifacts_dir)
     elif (arguments.test_bazel_build):
         run_bazel_build()
     elif (arguments.test_tf_python):
+        os.environ['NGRAPH_TF_LOG_0_DISABLED'] = '1'
         run_tensorflow_pytests_from_artifacts(
             arguments.backend, './',
             arguments.artifacts_dir + '/tensorflow/python', False)

test/python/test_depthwiseconv2d.py

@@ -21,6 +21,7 @@
 from __future__ import print_function
 
 import pytest
+import platform
 
 import tensorflow as tf
 from tensorflow.python.framework import constant_op
@@ -31,6 +32,7 @@
 
 class TestDepthwiseConv2dOperations(NgraphTest):
 
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     @pytest.mark.parametrize("padding", ("VALID", "SAME"))
     def test_depthwise_conv2d(self, padding):
         tensor_in_sizes = [1, 2, 3, 2]

test/python/test_flib.py

@@ -20,6 +20,7 @@
 from __future__ import print_function
 
 import pytest
+import platform
 
 import tensorflow as tf
 import numpy as np
@@ -28,6 +29,7 @@
 
 class TestFlibOperations(NgraphTest):
 
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_flib_1(self):
         graph = self.import_pbtxt('flib_graph_1.pbtxt')
         with graph.as_default() as g:

test/python/test_fusedConv2D.py

@@ -21,6 +21,7 @@
 from __future__ import print_function
 
 import pytest
+import platform
 
 import tensorflow as tf
 from tensorflow.python.framework import constant_op
@@ -50,6 +51,7 @@ def get_relu_op(self, relutype):
         ('relu6',),
         ('',),
     ))
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_fusedconv2d_bias_relu(self, relutype):
         inp_values = np.random.rand(*self.INPUT_SIZES)
         filt_values = np.random.rand(*self.FILTER_SIZES)
@@ -79,6 +81,7 @@ def run_test(sess):
         ('relu6',),
         ('',),
     ))
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_fusedconv2d_batchnorm(self, relutype):
         inp_values = np.random.rand(*self.INPUT_SIZES)
         filt_values = np.random.rand(*self.FILTER_SIZES)
@@ -119,6 +122,7 @@ def run_test(sess):
             rtol=0,
             atol=5e-5)
 
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_fusedconv2d_squeeze_bias(self):
         inp_values = np.random.rand(*self.INPUT_SIZES)
         filt_values = np.random.rand(*self.FILTER_SIZES)

test/python/test_fusedMatMul.py

@@ -20,6 +20,7 @@
 from __future__ import print_function
 
 import pytest
+import platform
 
 import tensorflow as tf
 from tensorflow.python.framework import constant_op
@@ -42,6 +43,7 @@ class TestFusedMatMul(NgraphTest):
             ('fusedmatmul_2.pbtxt',),  # No activation
         ))
     @pytest.mark.parametrize(("dim1", "dim2", "dim3"), ((3, 2, 2), (3, 4, 5)))
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_fusedmatmul_bias_pbtxt(self, filename, dim1, dim2, dim3):
         graph = self.import_pbtxt(filename)
         with graph.as_default() as g:

test/python/test_modeltester.py

@@ -22,6 +22,7 @@
 from __future__ import print_function
 
 import pytest
+import platform
 import os
 
 from common import NgraphTest
@@ -30,6 +31,7 @@
 
 class TestModelTester(NgraphTest):
 
+    @pytest.mark.skipif(platform.system() == 'Darwin', reason='Only for Linux')
     def test_MLP(self):
         cwd = os.getcwd()
         os.chdir('../model_level_tests/')

test_ngtf.py

@@ -92,6 +92,7 @@ def main():
                       "https://github.com/tensorflow/tensorflow.git", "v1.14.0")
 
     # Next run the TensorFlow python tests
+    os.environ['NGRAPH_TF_LOG_0_DISABLED'] = '1'
     run_tensorflow_pytests(venv_dir, build_dir, './', tf_src_dir)
 
     # Finally run Resnet50 based training and inferences

tools/build_utils.py

@@ -44,7 +44,7 @@ def is_venv():
             (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix))
 
 
-def command_executor(cmd, verbose=False, msg=None, stdout=None):
+def command_executor(cmd, verbose=False, msg=None, stdout=None, stderr=None):
     '''
     Executes the command.
     Example: 
@@ -56,7 +56,7 @@ def command_executor(cmd, verbose=False, msg=None, stdout=None):
     if verbose:
         tag = 'Running COMMAND: ' if msg is None else msg
         print(tag + cmd)
-    if (call(shlex.split(cmd), stdout=stdout) != 0):
+    if (call(shlex.split(cmd), stdout=stdout, stderr=stderr) != 0):
         raise Exception("Error running command: " + cmd)
 
 
@@ -346,6 +346,7 @@ def copy_tf_to_artifacts(artifacts_dir, tf_prebuilt):
     else:
         tf_cc_lib_file = os.path.abspath(tf_prebuilt + '/libtensorflow_cc.so.1')
         tf_cc_fmwk_file = os.path.abspath(tf_prebuilt + '/' + tf_fmwk_lib_name)
+    print("PWD: ", os.getcwd())
     print("Copying %s to %s" % (tf_cc_lib_file, artifacts_dir))
     shutil.copy(tf_cc_lib_file, artifacts_dir)
 

tools/test_utils.py

@@ -69,7 +69,7 @@ def run_ngtf_gtests(build_dir, filters):
     else:
         cmd = ['./gtest_ngtf']
 
-    command_executor(cmd, verbose=True)
+    command_executor(cmd, verbose=True, stderr=subprocess.DEVNULL)
 
     os.chdir(root_pwd)
 
@@ -97,7 +97,7 @@ def run_ngtf_cpp_gtests(artifacts_dir, log_dir, filters):
     else:
         cmd = ['./gtest_ngtf']
 
-    command_executor(cmd, verbose=True)
+    command_executor(cmd, verbose=True, stderr=subprocess.DEVNULL)
     os.chdir(root_pwd)
 
 
@@ -202,11 +202,17 @@ def run_tensorflow_pytests(venv_dir, build_dir, ngraph_tf_src_dir, tf_src_dir):
     os.environ['OMP_NUM_THREADS'] = str(num_cores)
     os.environ['NGRAPH_TF_DISABLE_DEASSIGN_CLUSTERS'] = '1'
 
+    # command_executor([
+    #     "python", test_script, "--tensorflow_path", tf_src_dir,
+    #     "--run_tests_from_file", test_manifest_file, "--xml_report",
+    #     test_xml_report
+    # ], verbose=True)
+
     command_executor([
         "python", test_script, "--tensorflow_path", tf_src_dir,
-        "--run_tests_from_file", test_manifest_file, "--xml_report",
-        test_xml_report
-    ])
+        "--run_tests_from_file", test_manifest_file
+    ],
+                     verbose=True)
 
     os.chdir(root_pwd)