Add plugin and compat tests for tensorboard-notf

.travis.yml

@@ -65,6 +65,9 @@ before_install:
   - chmod +x "${bazel_binary}"
   - sudo mv "${bazel_binary}" /usr/local/bin/bazel
 
+  # Fix Boto and Travis issue https://github.com/travis-ci/travis-ci/issues/7940
+  - sudo rm -f /etc/boto.cfg
+
   # Storing build artifacts in this directory helps Travis cache them. This
   # will sometimes cut latency in half, when we're lucky.
   - echo "startup --output_base=${HOME}/.bazel-output-base" >>~/.bazelrc
@@ -94,10 +97,12 @@ before_install:
   - echo "test --action_env=PATH" >>~/.bazelrc
 
 install:
+  - pip install boto3==1.9.86
   - pip install flake8==3.5.0
   - pip install futures==3.1.1
   - pip install grpcio==1.6.3
   - pip install mock==2.0.0
+  - pip install moto==1.3.7
   - pip install yamllint==1.5.0
   - pip install -I "${TF_VERSION_ID}"
 
@@ -121,6 +126,8 @@ script:
   - bazel fetch //tensorboard/...
   - bazel build //tensorboard/...
   - bazel test //tensorboard/...
+  # Run manual S3 test
+  - bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test
   - bazel run //tensorboard/pip_package:build_pip_package -- --tf-version "${TF_VERSION_ID}" --smoke
 
 after_script:

tensorboard/plugins/audio/BUILD

@@ -45,6 +45,28 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "audio_plugin_notf_test",
+    size = "small",
+    srcs = ["audio_plugin_test.py"],
+    main = "audio_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":audio_plugin",
+        ":summary",
+        "//tensorboard:expect_numpy_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "summary",
     srcs = ["summary.py"],

tensorboard/plugins/custom_scalar/BUILD

@@ -87,6 +87,28 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "custom_scalars_plugin_notf_test",
+    size = "small",
+    srcs = ["custom_scalars_plugin_test.py"],
+    main = "custom_scalars_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":custom_scalars_plugin",
+        ":protos_all_py_pb2",
+        ":summary",
+        "//tensorboard:expect_numpy_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/plugins/scalar:scalars_plugin",
+        "//tensorboard/plugins/scalar:summary",
+        "//tensorboard/util:test_util",
+    ],
+)
+
 tb_proto_library(
     name = "protos_all",
     srcs = ["layout.proto"],

tensorboard/plugins/distribution/BUILD

@@ -43,6 +43,29 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "distributions_plugin_notf_test",
+    size = "small",
+    srcs = ["distributions_plugin_test.py"],
+    main = "distributions_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":compressor",
+        ":distributions_plugin",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_accumulator",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/plugins/histogram:summary",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "compressor",
     srcs = ["compressor.py"],

tensorboard/plugins/graph/BUILD

@@ -63,6 +63,27 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "graphs_plugin_notf_test",
+    size = "small",
+    srcs = ["graphs_plugin_test.py"],
+    main = "graphs_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":graphs_plugin",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/compat/proto:protos_all_py_pb2",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "keras_util",
     srcs = ["keras_util.py"],

tensorboard/plugins/histogram/BUILD

@@ -47,6 +47,28 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "histograms_plugin_notf_test",
+    size = "small",
+    srcs = ["histograms_plugin_test.py"],
+    main = "histograms_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":histograms_plugin",
+        ":summary",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_accumulator",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "metadata",
     srcs = ["metadata.py"],

tensorboard/plugins/image/BUILD

@@ -59,6 +59,28 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "images_plugin_notf_test",
+    size = "small",
+    srcs = ["images_plugin_test.py"],
+    main = "images_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":images_plugin",
+        ":summary",
+        "//tensorboard:expect_numpy_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "summary",
     srcs = ["summary.py"],

tensorboard/plugins/pr_curve/BUILD

@@ -56,6 +56,27 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "pr_curves_plugin_notf_test",
+    size = "medium",  # tf integration test
+    srcs = ["pr_curves_plugin_test.py"],
+    main = "pr_curves_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":metadata",
+        ":pr_curve_demo_lib",
+        ":pr_curves_plugin",
+        "//tensorboard:expect_numpy_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_library(
     name = "summary",
     srcs = ["summary.py"],

tensorboard/plugins/projector/BUILD

@@ -68,6 +68,26 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "projector_plugin_notf_test",
+    size = "small",
+    srcs = ["projector_plugin_test.py"],
+    main = "projector_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":projector_plugin",
+        "//tensorboard:expect_numpy_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_multiplexer",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+    ],
+)
+
 tb_proto_library(
     name = "protos_all",
     srcs = glob(["*.proto"]),

tensorboard/plugins/projector/projector_plugin_test.py

@@ -25,6 +25,7 @@
 import os
 import numpy as np
 import tensorflow as tf
+import unittest
 
 from werkzeug import test as werkzeug_test
 from werkzeug import wrappers
@@ -35,13 +36,16 @@
 from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer  # pylint: disable=line-too-long
 from tensorboard.compat.proto import event_pb2
 from tensorboard.compat.proto import summary_pb2
+from tensorboard.compat import tf as tf_compat
 from tensorboard.plugins import base_plugin
 from tensorboard.plugins.projector import projector_config_pb2
 from tensorboard.plugins.projector import projector_plugin
 from tensorboard.util import test_util
 
 tf.compat.v1.disable_v2_behavior()
 
+USING_REAL_TF = tf_compat.__version__ != 'stub'
+
 
 class ProjectorAppTest(tf.test.TestCase):
 
@@ -58,7 +62,10 @@ def testRunsWithValidCheckpoint(self):
     self._GenerateProjectorTestData()
     self._SetupWSGIApp()
     run_json = self._GetJson('/data/plugin/projector/runs')
-    self.assertTrue(run_json)
+    if USING_REAL_TF:
+        self.assertTrue(run_json)
+    else:
+        self.assertFalse(run_json)
 
   def testRunsWithNoCheckpoint(self):
     self._SetupWSGIApp()
@@ -76,6 +83,8 @@ def testRunsWithInvalidModelCheckpointPath(self):
     run_json = self._GetJson('/data/plugin/projector/runs')
     self.assertEqual(run_json, [])
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testRunsWithInvalidModelCheckpointPathInConfig(self):
     config_path = os.path.join(self.log_dir, 'projector_config.pbtxt')
     config = projector_config_pb2.ProjectorConfig()
@@ -89,6 +98,8 @@ def testRunsWithInvalidModelCheckpointPathInConfig(self):
     run_json = self._GetJson('/data/plugin/projector/runs')
     self.assertEqual(run_json, [])
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testInfoWithValidCheckpointNoEventsData(self):
     self._GenerateProjectorTestData()
     self._SetupWSGIApp()
@@ -106,6 +117,8 @@ def testInfoWithValidCheckpointNoEventsData(self):
         'tensorName': 'var3'
     }])
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testInfoWithValidCheckpointAndEventsData(self):
     self._GenerateProjectorTestData()
     self._GenerateEventsData()
@@ -127,6 +140,8 @@ def testInfoWithValidCheckpointAndEventsData(self):
         'tensorName': 'var3'
     }])
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testTensorWithValidCheckpoint(self):
     self._GenerateProjectorTestData()
     self._SetupWSGIApp()
@@ -171,6 +186,8 @@ def testBookmarksUnknownName(self):
     url = '/data/plugin/projector/bookmarks?run=.&name=unknown'
     self.assertEqual(self._Get(url).status_code, 400)
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testBookmarks(self):
     self._GenerateProjectorTestData()
     self._SetupWSGIApp()
@@ -194,6 +211,8 @@ def _AssertTensorResponse(self, tensor_bytes, expected_tensor):
                         expected_tensor.shape)
     self.assertTrue(np.array_equal(tensor, expected_tensor))
 
+  # TODO(#2007): Cleanly separate out projector tests that require real TF
+  @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF')
   def testPluginIsActive(self):
     self._GenerateProjectorTestData()
     self._SetupWSGIApp()

tensorboard/plugins/scalar/BUILD

@@ -47,6 +47,27 @@ py_test(
     ],
 )
 
+# TODO(#2007): Remove this after pruning unnecessary TensorFlow deps in main test
+py_test(
+    name = "scalars_plugin_notf_test",
+    size = "small",
+    srcs = ["scalars_plugin_test.py"],
+    main = "scalars_plugin_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":scalars_plugin",
+        ":summary",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/backend:application",
+        "//tensorboard/backend/event_processing:event_accumulator",
+        "//tensorboard/compat:no_tensorflow",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/util:test_util",
+        "@org_pocoo_werkzeug",
+        "@org_pythonhosted_six",
+    ],
+)
+
 py_binary(
     name = "scalars_demo",
     srcs = ["scalars_demo.py"],