Skip to content
Permalink
 
 
Cannot retrieve contributors at this time
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""tfdbg CLI as SessionRunHook."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.debug.lib import stepper
from tensorflow.python.debug.wrappers import dumping_wrapper
from tensorflow.python.debug.wrappers import framework
from tensorflow.python.debug.wrappers import grpc_wrapper
from tensorflow.python.debug.wrappers import local_cli_wrapper
from tensorflow.python.training import session_run_hook
class LocalCLIDebugHook(session_run_hook.SessionRunHook):
"""Command-line-interface debugger hook.
Can be used as a monitor/hook for `tf.train.MonitoredSession`s and
`tf.contrib.learn`'s `Estimator`s and `Experiment`s.
"""
def __init__(self, ui_type="curses", dump_root=None, thread_name_filter=None):
"""Create a local debugger command-line interface (CLI) hook.
Args:
ui_type: (str) user-interface type.
dump_root: (`str`) optional path to the dump root directory. Must be a
directory that does not exist or an empty directory. If the directory
does not exist, it will be created by the debugger core during debug
`run()` calls and removed afterwards.
thread_name_filter: Regular-expression white list for threads on which the
wrapper session will be active. See doc of `BaseDebugWrapperSession` for
more details.
"""
self._ui_type = ui_type
self._dump_root = dump_root
self._thread_name_filter = thread_name_filter
self._session_wrapper = None
self._pending_tensor_filters = {}
def add_tensor_filter(self, filter_name, tensor_filter):
"""Add a tensor filter.
See doc of `LocalCLIDebugWrapperSession.add_tensor_filter()` for details.
Override default behavior to accommodate the possibility of this method
being
called prior to the initialization of the underlying
`LocalCLIDebugWrapperSession` object.
Args:
filter_name: See doc of `LocalCLIDebugWrapperSession.add_tensor_filter()`
for details.
tensor_filter: See doc of
`LocalCLIDebugWrapperSession.add_tensor_filter()` for details.
"""
if self._session_wrapper:
self._session_wrapper.add_tensor_filter(filter_name, tensor_filter)
else:
self._pending_tensor_filters[filter_name] = tensor_filter
def begin(self):
pass
def before_run(self, run_context):
if not self._session_wrapper:
self._session_wrapper = local_cli_wrapper.LocalCLIDebugWrapperSession(
run_context.session,
ui_type=self._ui_type,
dump_root=self._dump_root,
thread_name_filter=self._thread_name_filter)
# Actually register tensor filters registered prior to the construction
# of the underlying LocalCLIDebugWrapperSession object.
for filter_name in self._pending_tensor_filters:
self._session_wrapper.add_tensor_filter(
filter_name, self._pending_tensor_filters[filter_name])
# Increment run call counter.
self._session_wrapper.increment_run_call_count()
# Adapt run_context to an instance of OnRunStartRequest for invoking
# superclass on_run_start().
on_run_start_request = framework.OnRunStartRequest(
run_context.original_args.fetches, run_context.original_args.feed_dict,
None, None, self._session_wrapper.run_call_count)
on_run_start_response = self._session_wrapper.on_run_start(
on_run_start_request)
self._performed_action = on_run_start_response.action
run_args = session_run_hook.SessionRunArgs(
None, feed_dict=None, options=config_pb2.RunOptions())
if self._performed_action == framework.OnRunStartAction.DEBUG_RUN:
# pylint: disable=protected-access
self._session_wrapper._decorate_run_options_for_debug(
run_args.options,
on_run_start_response.debug_urls,
debug_ops=on_run_start_response.debug_ops,
node_name_regex_whitelist=(
on_run_start_response.node_name_regex_whitelist),
op_type_regex_whitelist=(
on_run_start_response.op_type_regex_whitelist),
tensor_dtype_regex_whitelist=(
on_run_start_response.tensor_dtype_regex_whitelist),
tolerate_debug_op_creation_failures=(
on_run_start_response.tolerate_debug_op_creation_failures))
# pylint: enable=protected-access
elif self._performed_action == framework.OnRunStartAction.PROFILE_RUN:
# pylint: disable=protected-access
self._session_wrapper._decorate_run_options_for_profile(run_args.options)
# pylint: enable=protected-access
elif self._performed_action == framework.OnRunStartAction.INVOKE_STEPPER:
# The _finalized property must be set to False so that the NodeStepper
# can insert ops for retrieving TensorHandles.
# pylint: disable=protected-access
run_context.session.graph._finalized = False
# pylint: enable=protected-access
with stepper.NodeStepper(
run_context.session, run_context.original_args.fetches,
run_context.original_args.feed_dict) as node_stepper:
self._session_wrapper.invoke_node_stepper(
node_stepper, restore_variable_values_on_exit=True)
return run_args
def after_run(self, run_context, run_values):
# Adapt run_context and run_values to OnRunEndRequest and invoke superclass
# on_run_end()
on_run_end_request = framework.OnRunEndRequest(self._performed_action,
run_values.run_metadata)
self._session_wrapper.on_run_end(on_run_end_request)
class DumpingDebugHook(session_run_hook.SessionRunHook):
"""A debugger hook that dumps debug data to filesystem.
Can be used as a monitor/hook for `tf.train.MonitoredSession`s and
`tf.contrib.learn`'s `Estimator`s and `Experiment`s.
"""
def __init__(self,
session_root,
watch_fn=None,
thread_name_filter=None,
log_usage=True):
"""Create a local debugger command-line interface (CLI) hook.
Args:
session_root: See doc of
`dumping_wrapper.DumpingDebugWrapperSession.__init__`.
watch_fn: See doc of
`dumping_wrapper.DumpingDebugWrapperSession.__init__`.
thread_name_filter: Regular-expression white list for threads on which the
wrapper session will be active. See doc of `BaseDebugWrapperSession` for
more details.
log_usage: (bool) Whether usage is to be logged.
"""
self._session_root = session_root
self._watch_fn = watch_fn
self._thread_name_filter = thread_name_filter
self._log_usage = log_usage
self._session_wrapper = None
def begin(self):
pass
def before_run(self, run_context):
if not self._session_wrapper:
self._session_wrapper = dumping_wrapper.DumpingDebugWrapperSession(
run_context.session,
self._session_root,
watch_fn=self._watch_fn,
thread_name_filter=self._thread_name_filter,
log_usage=self._log_usage)
self._session_wrapper.increment_run_call_count()
# pylint: disable=protected-access
debug_urls, watch_options = self._session_wrapper._prepare_run_watch_config(
run_context.original_args.fetches, run_context.original_args.feed_dict)
# pylint: enable=protected-access
run_options = config_pb2.RunOptions()
debug_utils.watch_graph(
run_options,
run_context.session.graph,
debug_urls=debug_urls,
debug_ops=watch_options.debug_ops,
node_name_regex_whitelist=watch_options.node_name_regex_whitelist,
op_type_regex_whitelist=watch_options.op_type_regex_whitelist,
tensor_dtype_regex_whitelist=watch_options.tensor_dtype_regex_whitelist,
tolerate_debug_op_creation_failures=(
watch_options.tolerate_debug_op_creation_failures))
run_args = session_run_hook.SessionRunArgs(
None, feed_dict=None, options=run_options)
return run_args
def after_run(self, run_context, run_values):
pass
class GrpcDebugHook(session_run_hook.SessionRunHook):
"""A hook that streams debugger-related events to any grpc_debug_server.
For example, the debugger data server is a grpc_debug_server. The debugger
data server writes debugger-related events it receives via GRPC to logdir.
This enables debugging features in Tensorboard such as health pills.
When the arguments of debug_utils.watch_graph changes, strongly consider
changing arguments here too so that features are available to tflearn users.
Can be used as a monitor/hook for `tf.train.MonitoredSession`s and
`tf.contrib.learn`'s `Estimator`s and `Experiment`s.
"""
def __init__(self,
grpc_debug_server_addresses,
watch_fn=None,
thread_name_filter=None,
log_usage=True):
"""Constructs a GrpcDebugHook.
Args:
grpc_debug_server_addresses: (`list` of `str`) A list of the gRPC debug
server addresses, in the format of <host:port>, with or without the
"grpc://" prefix. For example: ["localhost:7000", "192.168.0.2:8000"]
watch_fn: A function that allows for customizing which ops to watch at
which specific steps. See doc of
`dumping_wrapper.DumpingDebugWrapperSession.__init__` for details.
thread_name_filter: Regular-expression white list for threads on which the
wrapper session will be active. See doc of `BaseDebugWrapperSession` for
more details.
log_usage: (bool) Whether usage is to be logged.
"""
self._grpc_debug_wrapper_session = None
self._thread_name_filter = thread_name_filter
self._grpc_debug_server_addresses = (
grpc_debug_server_addresses
if isinstance(grpc_debug_server_addresses, list) else
[grpc_debug_server_addresses])
self._watch_fn = watch_fn
self._log_usage = log_usage
def before_run(self, run_context):
"""Called right before a session is run.
Args:
run_context: A session_run_hook.SessionRunContext. Encapsulates
information on the run.
Returns:
A session_run_hook.SessionRunArgs object.
"""
if not self._grpc_debug_wrapper_session:
self._grpc_debug_wrapper_session = grpc_wrapper.GrpcDebugWrapperSession(
run_context.session,
self._grpc_debug_server_addresses,
watch_fn=self._watch_fn,
thread_name_filter=self._thread_name_filter,
log_usage=self._log_usage)
fetches = run_context.original_args.fetches
feed_dict = run_context.original_args.feed_dict
watch_options = self._watch_fn(fetches, feed_dict)
run_options = config_pb2.RunOptions()
debug_utils.watch_graph(
run_options,
run_context.session.graph,
debug_urls=self._grpc_debug_wrapper_session.prepare_run_debug_urls(
fetches, feed_dict),
debug_ops=watch_options.debug_ops,
node_name_regex_whitelist=watch_options.node_name_regex_whitelist,
op_type_regex_whitelist=watch_options.op_type_regex_whitelist,
tensor_dtype_regex_whitelist=watch_options.tensor_dtype_regex_whitelist,
tolerate_debug_op_creation_failures=(
watch_options.tolerate_debug_op_creation_failures))
return session_run_hook.SessionRunArgs(
None, feed_dict=None, options=run_options)
class TensorBoardDebugHook(GrpcDebugHook):
"""A tfdbg hook that can be used with TensorBoard Debugger Plugin.
This hook is the same as `GrpcDebugHook`, except that it uses a predefined
`watch_fn` that
1) uses `DebugIdentity` debug ops with the `gated_grpc` attribute set to
`True`, to allow the interactive enabling and disabling of tensor
breakpoints.
2) watches all tensors in the graph.
This saves the need for the user to define a `watch_fn`.
"""
def __init__(self,
grpc_debug_server_addresses,
thread_name_filter=None,
send_traceback_and_source_code=True,
log_usage=True):
"""Constructor of TensorBoardDebugHook.
Args:
grpc_debug_server_addresses: gRPC address(es) of debug server(s), as a
`str` or a `list` of `str`s. E.g., "localhost:2333",
"grpc://localhost:2333", ["192.168.0.7:2333", "192.168.0.8:2333"].
thread_name_filter: Optional filter for thread names.
send_traceback_and_source_code: Whether traceback of graph elements and
the source code are to be sent to the debug server(s).
log_usage: Whether the usage of this class is to be logged (if
applicable).
"""
def _gated_grpc_watch_fn(fetches, feeds):
del fetches, feeds # Unused.
return framework.WatchOptions(
debug_ops=["DebugIdentity(gated_grpc=true)"])
super(TensorBoardDebugHook, self).__init__(
grpc_debug_server_addresses,
watch_fn=_gated_grpc_watch_fn,
thread_name_filter=thread_name_filter,
log_usage=log_usage)
self._grpc_debug_server_addresses = grpc_debug_server_addresses
self._send_traceback_and_source_code = send_traceback_and_source_code
self._sent_graph_version = -1
grpc_wrapper.register_signal_handler()
def before_run(self, run_context):
if self._send_traceback_and_source_code:
self._sent_graph_version = grpc_wrapper.publish_traceback(
self._grpc_debug_server_addresses, run_context.session.graph,
run_context.original_args.feed_dict,
run_context.original_args.fetches, self._sent_graph_version)
return super(TensorBoardDebugHook, self).before_run(run_context)