From c07266fb3d835ad762e17260d2beebded39bb1f1 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Mon, 9 Dec 2019 10:07:35 -0800 Subject: [PATCH 1/3] images: remove unused width and height on frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: The images plugin backend sends down images’ widths and heights, but the frontend does not actually need or use this data. (A comment claiming otherwise was in error.) This commit removes that behavior. Test Plan: The dashboard behavior appears unchanged, with both fixed-width and actual-size display behaviors at both the per-image and dashboard-global levels. All references to `width` or `height` in modified files are now irrelevant. No network responses now contain `width` or `height`. wchargin-branch: images-no-dims --- tensorboard/plugins/image/http_api.md | 4 ---- tensorboard/plugins/image/images_plugin.py | 15 ++++----------- .../image/tf_image_dashboard/tf-image-loader.html | 4 ---- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/tensorboard/plugins/image/http_api.md b/tensorboard/plugins/image/http_api.md index f51f3d52d3..a56d776465 100644 --- a/tensorboard/plugins/image/http_api.md +++ b/tensorboard/plugins/image/http_api.md @@ -52,8 +52,6 @@ which is an object with the following items: - `"wall_time"`: floating-point number of seconds since epoch. - `"step"`: integer step counter. - - `"width"`: integer width of the image, in pixels. - - `"height"`: integer height of the image, in pixels. - `"query"`: query string that can be given to the `individualImage` route (below) to serve the actual image content. This string must be treated as opaque: clients must not inspect or modify its value. @@ -61,8 +59,6 @@ which is an object with the following items: Here is an example response: [{ - "width": 28, - "height": 28, "wall_time": 1440210599.246, "step": 63702821, "query": "index=0&sample=0&tagname=input%2Fimage%2F2&run=train" diff --git a/tensorboard/plugins/image/images_plugin.py b/tensorboard/plugins/image/images_plugin.py index dfb0c795bf..2221f11a21 100644 --- a/tensorboard/plugins/image/images_plugin.py +++ b/tensorboard/plugins/image/images_plugin.py @@ -184,8 +184,8 @@ def _image_response_for_run(self, run, tag, sample): fewer than three images will be omitted from the results. Returns: - A list of dictionaries containing the wall time, step, URL, width, and - height for each image. + A list of dictionaries containing the wall time, step, and URL for + each image. """ if self._db_connection_provider: db = self._db_connection_provider() @@ -193,9 +193,7 @@ def _image_response_for_run(self, run, tag, sample): ''' SELECT computed_time, - step, - CAST (T0.data AS INT) AS width, - CAST (T1.data AS INT) AS height + step FROM Tensors JOIN TensorStrings AS T0 ON Tensors.rowid = T0.tensor_rowid @@ -219,23 +217,18 @@ def _image_response_for_run(self, run, tag, sample): return [{ 'wall_time': computed_time, 'step': step, - 'width': width, - 'height': height, 'query': self._query_for_individual_image(run, tag, sample, index) - } for index, (computed_time, step, width, height) in enumerate(cursor)] + } for index, (computed_time, step) in enumerate(cursor)] response = [] index = 0 tensor_events = self._multiplexer.Tensors(run, tag) filtered_events = self._filter_by_sample(tensor_events, sample) for (index, tensor_event) in enumerate(filtered_events): - (width, height) = tensor_event.tensor_proto.string_val[:2] response.append({ 'wall_time': tensor_event.wall_time, 'step': tensor_event.step, # We include the size so that the frontend can add that to the # tag so that the page layout doesn't change when the image loads. - 'width': int(width), - 'height': int(height), 'query': self._query_for_individual_image(run, tag, sample, index) }) return response diff --git a/tensorboard/plugins/image/tf_image_dashboard/tf-image-loader.html b/tensorboard/plugins/image/tf_image_dashboard/tf-image-loader.html index 0b25ff5fea..edbe9d873c 100644 --- a/tensorboard/plugins/image/tf_image_dashboard/tf-image-loader.html +++ b/tensorboard/plugins/image/tf_image_dashboard/tf-image-loader.html @@ -234,8 +234,6 @@ }, // steps: { - // width: number, - // height: number, // wall_time: Date, // step: number, // url: string, @@ -363,8 +361,6 @@ url += '&' + imageMetadata.query; return { - width: imageMetadata.width, - height: imageMetadata.height, // The wall time within the metadata is in seconds. The Date // constructor accepts a time in milliseconds, so we multiply by 1000. wall_time: new Date(imageMetadata.wall_time * 1000), From 12ac101c3f19fe7df036e2fd494d517c93736e35 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Mon, 9 Dec 2019 10:56:51 -0800 Subject: [PATCH 2/3] [update patch] wchargin-branch: images-no-dims wchargin-source: 107104491a31e9b6dd5a03fdeaf195dc20513631 --- tensorboard/plugins/image/images_plugin_test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tensorboard/plugins/image/images_plugin_test.py b/tensorboard/plugins/image/images_plugin_test.py index 09db36aaa8..de7aafc036 100644 --- a/tensorboard/plugins/image/images_plugin_test.py +++ b/tensorboard/plugins/image/images_plugin_test.py @@ -126,8 +126,6 @@ def testOldStyleImagesRoute(self): # Verify that the 1st entry is correct. entry = entries[0] - self.assertEqual(42, entry["width"]) - self.assertEqual(16, entry["height"]) self.assertEqual(0, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["foo"], parsed_query["run"]) @@ -137,8 +135,6 @@ def testOldStyleImagesRoute(self): # Verify that the 2nd entry is correct. entry = entries[1] - self.assertEqual(42, entry["width"]) - self.assertEqual(16, entry["height"]) self.assertEqual(1, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["foo"], parsed_query["run"]) @@ -158,8 +154,6 @@ def testNewStyleImagesRoute(self): # Verify that the 1st entry is correct. entry = entries[0] - self.assertEqual(6, entry["width"]) - self.assertEqual(8, entry["height"]) self.assertEqual(0, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["bar"], parsed_query["run"]) @@ -169,8 +163,6 @@ def testNewStyleImagesRoute(self): # Verify that the 2nd entry is correct. entry = entries[1] - self.assertEqual(6, entry["width"]) - self.assertEqual(8, entry["height"]) self.assertEqual(1, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["bar"], parsed_query["run"]) From 119e209480a081ee27556440de0b26b1ca838400 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 2 Jan 2020 10:28:57 -0800 Subject: [PATCH 3/3] [update patch] wchargin-branch: images-no-dims wchargin-source: e63d04fabba6b1015d24280ef65ffbbf6a0ed1b2 --- tensorboard/plugins/image/images_plugin.py | 223 +----------------- .../plugins/image/images_plugin_test.py | 172 -------------- 2 files changed, 4 insertions(+), 391 deletions(-) diff --git a/tensorboard/plugins/image/images_plugin.py b/tensorboard/plugins/image/images_plugin.py index 812305d53c..121a7cb8b8 100644 --- a/tensorboard/plugins/image/images_plugin.py +++ b/tensorboard/plugins/image/images_plugin.py @@ -202,8 +202,8 @@ def _image_response_for_run(self, run, tag, sample): fewer than three images will be omitted from the results. Returns: - A list of dictionaries containing the wall time, step, URL, width, and - height for each image. + A list of dictionaries containing the wall time, step, and URL + for each image. """ if self._db_connection_provider: db = self._db_connection_provider() @@ -211,9 +211,7 @@ def _image_response_for_run(self, run, tag, sample): """ SELECT computed_time, - step, - CAST (T0.data AS INT) AS width, - CAST (T1.data AS INT) AS height + step FROM Tensors JOIN TensorStrings AS T0 ON Tensors.rowid = T0.tensor_rowid @@ -239,233 +237,21 @@ def _image_response_for_run(self, run, tag, sample): { "wall_time": computed_time, "step": step, - "width": width, - "height": height, "query": self._query_for_individual_image( run, tag, sample, index ), } - for index, (computed_time, step, width, height) in enumerate( - cursor - ) + for index, (computed_time, step) in enumerate(cursor) ] response = [] index = 0 tensor_events = self._multiplexer.Tensors(run, tag) -<<<<<<< HEAD - samples = max([len(event.tensor_proto.string_val[2:]) # width, height - for event in tensor_events] + [0]) - result[run][tag] = {'displayName': summary_metadata.display_name, - 'description': plugin_util.markdown_to_safe_html( - summary_metadata.summary_description), - 'samples': samples} - return result - - @wrappers.Request.application - def _serve_image_metadata(self, request): - """Given a tag and list of runs, serve a list of metadata for images. - - Note that the images themselves are not sent; instead, we respond with URLs - to the images. The frontend should treat these URLs as opaque and should not - try to parse information about them or generate them itself, as the format - may change. - - Args: - request: A werkzeug.wrappers.Request object. - - Returns: - A werkzeug.Response application. - """ - tag = request.args.get('tag') - run = request.args.get('run') - sample = int(request.args.get('sample', 0)) - try: - response = self._image_response_for_run(run, tag, sample) - except KeyError: - return http_util.Respond( - request, 'Invalid run or tag', 'text/plain', code=400 - ) - return http_util.Respond(request, response, 'application/json') - - def _image_response_for_run(self, run, tag, sample): - """Builds a JSON-serializable object with information about images. - - Args: - run: The name of the run. - tag: The name of the tag the images all belong to. - sample: The zero-indexed sample of the image for which to retrieve - information. For instance, setting `sample` to `2` will fetch - information about only the third image of each batch. Steps with - fewer than three images will be omitted from the results. - - Returns: - A list of dictionaries containing the wall time, step, and URL for - each image. - """ - if self._db_connection_provider: - db = self._db_connection_provider() - cursor = db.execute( - ''' - SELECT - computed_time, - step - FROM Tensors - JOIN TensorStrings AS T0 - ON Tensors.rowid = T0.tensor_rowid - JOIN TensorStrings AS T1 - ON Tensors.rowid = T1.tensor_rowid - WHERE - series = ( - SELECT tag_id - FROM Runs - CROSS JOIN Tags USING (run_id) - WHERE Runs.run_name = :run AND Tags.tag_name = :tag) - AND step IS NOT NULL - AND dtype = :dtype - /* Should be n-vector, n >= 3: [width, height, samples...] */ - AND (NOT INSTR(shape, ',') AND CAST (shape AS INT) >= 3) - AND T0.idx = 0 - AND T1.idx = 1 - ORDER BY step - ''', - {'run': run, 'tag': tag, 'dtype': tf.string.as_datatype_enum}) - return [{ - 'wall_time': computed_time, - 'step': step, - 'query': self._query_for_individual_image(run, tag, sample, index) - } for index, (computed_time, step) in enumerate(cursor)] - response = [] - index = 0 - tensor_events = self._multiplexer.Tensors(run, tag) - filtered_events = self._filter_by_sample(tensor_events, sample) - for (index, tensor_event) in enumerate(filtered_events): - response.append({ - 'wall_time': tensor_event.wall_time, - 'step': tensor_event.step, - # We include the size so that the frontend can add that to the - # tag so that the page layout doesn't change when the image loads. - 'query': self._query_for_individual_image(run, tag, sample, index) - }) - return response - - def _filter_by_sample(self, tensor_events, sample): - return [tensor_event for tensor_event in tensor_events - if (len(tensor_event.tensor_proto.string_val) - 2 # width, height - > sample)] - - def _query_for_individual_image(self, run, tag, sample, index): - """Builds a URL for accessing the specified image. - - This should be kept in sync with _serve_image_metadata. Note that the URL is - *not* guaranteed to always return the same image, since images may be - unloaded from the reservoir as new images come in. - - Args: - run: The name of the run. - tag: The tag. - sample: The relevant sample index, zero-indexed. See documentation - on `_image_response_for_run` for more details. - index: The index of the image. Negative values are OK. - - Returns: - A string representation of a URL that will load the index-th sampled image - in the given run with the given tag. - """ - query_string = urllib.parse.urlencode({ - 'run': run, - 'tag': tag, - 'sample': sample, - 'index': index, - }) - return query_string - - def _get_individual_image(self, run, tag, index, sample): - """ - Returns the actual image bytes for a given image. - - Args: - run: The name of the run the image belongs to. - tag: The name of the tag the images belongs to. - index: The index of the image in the current reservoir. - sample: The zero-indexed sample of the image to retrieve (for example, - setting `sample` to `2` will fetch the third image sample at `step`). - - Returns: - A bytestring of the raw image bytes. - """ - if self._db_connection_provider: - db = self._db_connection_provider() - cursor = db.execute( - ''' - SELECT data - FROM TensorStrings - WHERE - /* Skip first 2 elements which are width and height. */ - idx = 2 + :sample - AND tensor_rowid = ( - SELECT rowid - FROM Tensors - WHERE - series = ( - SELECT tag_id - FROM Runs - CROSS JOIN Tags USING (run_id) - WHERE - Runs.run_name = :run - AND Tags.tag_name = :tag) - AND step IS NOT NULL - AND dtype = :dtype - /* Should be n-vector, n >= 3: [width, height, samples...] */ - AND (NOT INSTR(shape, ',') AND CAST (shape AS INT) >= 3) - ORDER BY step - LIMIT 1 - OFFSET :index) - ''', - {'run': run, - 'tag': tag, - 'sample': sample, - 'index': index, - 'dtype': tf.string.as_datatype_enum}) - (data,) = cursor.fetchone() - return six.binary_type(data) - - events = self._filter_by_sample(self._multiplexer.Tensors(run, tag), sample) - images = events[index].tensor_proto.string_val[2:] # skip width, height - return images[sample] - - @wrappers.Request.application - def _serve_individual_image(self, request): - """Serves an individual image.""" - run = request.args.get('run') - tag = request.args.get('tag') - index = int(request.args.get('index', '0')) - sample = int(request.args.get('sample', '0')) - try: - data = self._get_individual_image(run, tag, index, sample) - except (KeyError, IndexError): - return http_util.Respond( - request, 'Invalid run, tag, index, or sample', 'text/plain', code=400 - ) - image_type = imghdr.what(None, data) - content_type = _IMGHDR_TO_MIMETYPE.get(image_type, _DEFAULT_IMAGE_MIMETYPE) - return http_util.Respond(request, data, content_type) - - @wrappers.Request.application - def _serve_tags(self, request): - index = self._index_impl() - return http_util.Respond(request, index, 'application/json') -======= filtered_events = self._filter_by_sample(tensor_events, sample) for (index, tensor_event) in enumerate(filtered_events): - (width, height) = tensor_event.tensor_proto.string_val[:2] response.append( { "wall_time": tensor_event.wall_time, "step": tensor_event.step, - # We include the size so that the frontend can add that to the - # tag so that the page layout doesn't change when the image loads. - "width": int(width), - "height": int(height), "query": self._query_for_individual_image( run, tag, sample, index ), @@ -590,4 +376,3 @@ def _serve_individual_image(self, request): def _serve_tags(self, request): index = self._index_impl() return http_util.Respond(request, index, "application/json") ->>>>>>> 1d374693fd975c5039d97743d28401ad156946a3 diff --git a/tensorboard/plugins/image/images_plugin_test.py b/tensorboard/plugins/image/images_plugin_test.py index 89714e028d..d6cb6fb848 100644 --- a/tensorboard/plugins/image/images_plugin_test.py +++ b/tensorboard/plugins/image/images_plugin_test.py @@ -44,169 +44,6 @@ class ImagesPluginTest(tf.test.TestCase): -<<<<<<< HEAD - - def setUp(self): - self.log_dir = tempfile.mkdtemp() - - # We use numpy.random to generate images. We seed to avoid non-determinism - # in this test. - numpy.random.seed(42) - - # Create old-style image summaries for run "foo". - tf.compat.v1.reset_default_graph() - sess = tf.compat.v1.Session() - placeholder = tf.compat.v1.placeholder(tf.uint8) - tf.compat.v1.summary.image(name="baz", tensor=placeholder) - merged_summary_op = tf.compat.v1.summary.merge_all() - foo_directory = os.path.join(self.log_dir, "foo") - with test_util.FileWriterCache.get(foo_directory) as writer: - writer.add_graph(sess.graph) - for step in xrange(2): - writer.add_summary(sess.run(merged_summary_op, feed_dict={ - placeholder: (numpy.random.rand(1, 16, 42, 3) * 255).astype( - numpy.uint8) - }), global_step=step) - - # Create new-style image summaries for run bar. - tf.compat.v1.reset_default_graph() - sess = tf.compat.v1.Session() - placeholder = tf.compat.v1.placeholder(tf.uint8) - summary.op(name="quux", images=placeholder, - description="how do you pronounce that, anyway?") - merged_summary_op = tf.compat.v1.summary.merge_all() - bar_directory = os.path.join(self.log_dir, "bar") - with test_util.FileWriterCache.get(bar_directory) as writer: - writer.add_graph(sess.graph) - for step in xrange(2): - writer.add_summary(sess.run(merged_summary_op, feed_dict={ - placeholder: (numpy.random.rand(1, 8, 6, 3) * 255).astype( - numpy.uint8) - }), global_step=step) - - # Start a server with the plugin. - multiplexer = event_multiplexer.EventMultiplexer({ - "foo": foo_directory, - "bar": bar_directory, - }) - multiplexer.Reload() - context = base_plugin.TBContext( - logdir=self.log_dir, multiplexer=multiplexer) - plugin = images_plugin.ImagesPlugin(context) - wsgi_app = application.TensorBoardWSGI([plugin]) - self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) - self.routes = plugin.get_plugin_apps() - - def tearDown(self): - shutil.rmtree(self.log_dir, ignore_errors=True) - - def _DeserializeResponse(self, byte_content): - """Deserializes byte content that is a JSON encoding. - - Args: - byte_content: The byte content of a response. - - Returns: - The deserialized python object decoded from JSON. - """ - return json.loads(byte_content.decode("utf-8")) - - def testRoutesProvided(self): - """Tests that the plugin offers the correct routes.""" - self.assertIsInstance(self.routes["/images"], collections.Callable) - self.assertIsInstance(self.routes["/individualImage"], collections.Callable) - self.assertIsInstance(self.routes["/tags"], collections.Callable) - - def testOldStyleImagesRoute(self): - """Tests that the /images routes returns correct old-style data.""" - response = self.server.get( - "/data/plugin/images/images?run=foo&tag=baz/image/0&sample=0") - self.assertEqual(200, response.status_code) - - # Verify that the correct entries are returned. - entries = self._DeserializeResponse(response.get_data()) - self.assertEqual(2, len(entries)) - - # Verify that the 1st entry is correct. - entry = entries[0] - self.assertEqual(0, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["foo"], parsed_query["run"]) - self.assertListEqual(["baz/image/0"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["0"], parsed_query["index"]) - - # Verify that the 2nd entry is correct. - entry = entries[1] - self.assertEqual(1, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["foo"], parsed_query["run"]) - self.assertListEqual(["baz/image/0"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["1"], parsed_query["index"]) - - def testNewStyleImagesRoute(self): - """Tests that the /images routes returns correct new-style data.""" - response = self.server.get( - "/data/plugin/images/images?run=bar&tag=quux/image_summary&sample=0") - self.assertEqual(200, response.status_code) - - # Verify that the correct entries are returned. - entries = self._DeserializeResponse(response.get_data()) - self.assertEqual(2, len(entries)) - - # Verify that the 1st entry is correct. - entry = entries[0] - self.assertEqual(0, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["bar"], parsed_query["run"]) - self.assertListEqual(["quux/image_summary"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["0"], parsed_query["index"]) - - # Verify that the 2nd entry is correct. - entry = entries[1] - self.assertEqual(1, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["bar"], parsed_query["run"]) - self.assertListEqual(["quux/image_summary"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["1"], parsed_query["index"]) - - def testOldStyleIndividualImageRoute(self): - """Tests fetching an individual image from an old-style summary.""" - response = self.server.get( - "/data/plugin/images/individualImage" - "?run=foo&tag=baz/image/0&sample=0&index=0") - self.assertEqual(200, response.status_code) - self.assertEqual("image/png", response.headers.get("content-type")) - - def testNewStyleIndividualImageRoute(self): - """Tests fetching an individual image from a new-style summary.""" - response = self.server.get( - "/data/plugin/images/individualImage" - "?run=bar&tag=quux/image_summary&sample=0&index=0") - self.assertEqual(200, response.status_code) - self.assertEqual("image/png", response.headers.get("content-type")) - - def testRunsRoute(self): - """Tests that the /runs route offers the correct run to tag mapping.""" - response = self.server.get("/data/plugin/images/tags") - self.assertEqual(200, response.status_code) - self.assertDictEqual({ - "foo": { - "baz/image/0": { - "displayName": "baz/image/0", - "description": "", - "samples": 1, - }, - }, - "bar": { - "quux/image_summary": { - "displayName": "quux", - "description": "

how do you pronounce that, anyway?

", - "samples": 1, -======= def setUp(self): self.log_dir = tempfile.mkdtemp() @@ -310,8 +147,6 @@ def testOldStyleImagesRoute(self): # Verify that the 1st entry is correct. entry = entries[0] - self.assertEqual(42, entry["width"]) - self.assertEqual(16, entry["height"]) self.assertEqual(0, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["foo"], parsed_query["run"]) @@ -321,8 +156,6 @@ def testOldStyleImagesRoute(self): # Verify that the 2nd entry is correct. entry = entries[1] - self.assertEqual(42, entry["width"]) - self.assertEqual(16, entry["height"]) self.assertEqual(1, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["foo"], parsed_query["run"]) @@ -343,8 +176,6 @@ def testNewStyleImagesRoute(self): # Verify that the 1st entry is correct. entry = entries[0] - self.assertEqual(6, entry["width"]) - self.assertEqual(8, entry["height"]) self.assertEqual(0, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["bar"], parsed_query["run"]) @@ -354,8 +185,6 @@ def testNewStyleImagesRoute(self): # Verify that the 2nd entry is correct. entry = entries[1] - self.assertEqual(6, entry["width"]) - self.assertEqual(8, entry["height"]) self.assertEqual(1, entry["step"]) parsed_query = urllib.parse.parse_qs(entry["query"]) self.assertListEqual(["bar"], parsed_query["run"]) @@ -401,7 +230,6 @@ def testRunsRoute(self): "samples": 1, }, }, ->>>>>>> 1d374693fd975c5039d97743d28401ad156946a3 }, self._DeserializeResponse(response.get_data()), )