From 43f8dbeb23420f63965df8afa4de10832050b96b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 28 Nov 2017 15:04:49 -0800 Subject: [PATCH 01/15] Add Stackdriver Trace V2 models --- trace/opencensus/trace/attributes.py | 60 +++++++++++ .../trace/exporters/stackdriver_exporter.py | 2 +- .../opencensus/trace/ext/sqlalchemy/trace.py | 2 +- trace/opencensus/trace/link.py | 70 ++++++++++++ trace/opencensus/trace/stack_trace.py | 102 ++++++++++++++++++ trace/opencensus/trace/status.py | 55 ++++++++++ trace/opencensus/trace/time_event.py | 61 +++++++++++ trace/opencensus/trace/utils.py | 40 +++++++ 8 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 trace/opencensus/trace/attributes.py create mode 100644 trace/opencensus/trace/link.py create mode 100644 trace/opencensus/trace/stack_trace.py create mode 100644 trace/opencensus/trace/status.py create mode 100644 trace/opencensus/trace/time_event.py create mode 100644 trace/opencensus/trace/utils.py diff --git a/trace/opencensus/trace/attributes.py b/trace/opencensus/trace/attributes.py new file mode 100644 index 000000000..dde4a2b72 --- /dev/null +++ b/trace/opencensus/trace/attributes.py @@ -0,0 +1,60 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def _format_attribute_value(value): + if isinstance(value, str): + value_type = 'string_value' + elif isinstance(value, int): + value_type = 'int_value' + elif isinstance(value, bool): + value_type = 'bool_value' + else: + raise TypeError("Value must be str, int, or bool.") + + return {value_type: value} + + +class Attributes(object): + """A set of attributes, each in the format [KEY]:[VALUE]. + + :type attributes: dict + :param attributes: The set of attributes. Each attribute's key can be up + to 128 bytes long. The value can be a string up to 256 + bytes, an integer, or the Boolean values true and false. + """ + def __init__(self, attributes=None): + if attributes is None: + attributes = {} + + self.attributes = attributes + + def set_attribute(self, key, value): + """Set a key value pair.""" + self.attributes[key] = value + + def get_attribute(self, key): + """Get a attribute value.""" + return self.attributes.get(key, None) + + def format_attributes_json(self): + """Convert the Attributes object to json format.""" + attributes_json = {} + + for key in self.attributes: + value = self.attributes.get(key) + value_json = _format_attribute_value(value) + attributes_json[key] = value_json + + return attributes_json diff --git a/trace/opencensus/trace/exporters/stackdriver_exporter.py b/trace/opencensus/trace/exporters/stackdriver_exporter.py index 5ca3ecf02..7844c4db6 100644 --- a/trace/opencensus/trace/exporters/stackdriver_exporter.py +++ b/trace/opencensus/trace/exporters/stackdriver_exporter.py @@ -90,7 +90,7 @@ class StackdriverExporter(base.Exporter): """ def __init__(self, client=None, project_id=None, transport=sync.SyncTransport): - # The client will handler the case when project_id is None + # The client will handle the case when project_id is None if client is None: client = Client(project=project_id) diff --git a/trace/opencensus/trace/ext/sqlalchemy/trace.py b/trace/opencensus/trace/ext/sqlalchemy/trace.py index 7b1ec1e57..4529f1f92 100644 --- a/trace/opencensus/trace/ext/sqlalchemy/trace.py +++ b/trace/opencensus/trace/ext/sqlalchemy/trace.py @@ -62,7 +62,7 @@ def _before_cursor_execute(conn, cursor, statement, parameters, _tracer = execution_context.get_opencensus_tracer() _span = _tracer.start_span() - _span.name = '[{}.query]{}'.format(MODULE_NAME, statement) + _span.name = '{}.query'.format(MODULE_NAME) # Set query statement label _tracer.add_label_to_current_span( diff --git a/trace/opencensus/trace/link.py b/trace/opencensus/trace/link.py new file mode 100644 index 000000000..182caf2f2 --- /dev/null +++ b/trace/opencensus/trace/link.py @@ -0,0 +1,70 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Type(object): + """The relationship of the current span relative to the linked span: child, + parent, or unspecified. + + Attributes: + TYPE_UNSPECIFIED (int): The relationship of the two spans is unknown. + CHILD_LINKED_SPAN (int): The linked span is a child of the current span. + PARENT_LINKED_SPAN (int): The linked span is a parent of the current span. + """ + TYPE_UNSPECIFIED = 0 + CHILD_LINKED_SPAN = 1 + PARENT_LINKED_SPAN = 2 + + +class Link(object): + """A pointer from the current span to another span in the same trace or in + a different trace. For example, this can be used in batching operations, + where a single batch handler processes multiple requests from different + traces or when the handler receives a request from a different project. + + :type trace_id: str + :param trace_id: The [TRACE_ID] for a trace within a project. + + :type span_id: str + :param span_id: The [SPAN_ID] for a span within a trace. + + :type type: Enum of :class:`~opencensus.trace.link.Type` + :param type: The relationship of the current span relative to the linked + span. + + :type attributes: :class:`~opencensus.trace.attributes.Attributes` + :param attributes: A set of attributes on the link. You have have up to 32 + attributes per link. + """ + def __init__(self, trace_id, span_id, type=None, attributes=None): + self.trace_id = trace_id + self.span_id = span_id + + if type is None: + type = Type.TYPE_UNSPECIFIED + + self.type = type + self.attributes = attributes + + def format_link_json(self): + """Conver a Link object to json format.""" + link_json = {} + link_json['trace_id'] = self.trace_id + link_json['span_id'] = self.span_id + link_json['type'] = self.type + + if self.attributes is not None: + link_json['attributes'] = self.attributes + + return link_json diff --git a/trace/opencensus/trace/stack_trace.py b/trace/opencensus/trace/stack_trace.py new file mode 100644 index 000000000..cf82a1639 --- /dev/null +++ b/trace/opencensus/trace/stack_trace.py @@ -0,0 +1,102 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +from opencensus.trace.utils import _get_truncatable_str + + +class StackFrame(object): + """Represents a single stack frame in a stack trace. + + :type func_name + """ + def __init__(self, + func_name, + original_func_name, + file_name, + line_num, + col_num, + load_module, + build_id, + source_version) + self.func_name = func_name + self.original_func_name = original_func_name + self.file_name = file_name + self.line_num = line_num + self.col_num = col_num + self.load_module = load_module + self.build_id = build_id + self.source_version = source_version + + def format_stack_frame_json(self): + """Convert StackFrame object to json format.""" + stack_frame_json = {} + stack_frame_json['function_name'] = _get_truncatable_str(self.func_name) + stack_frame_json['original_function_name'] = _get_truncatable_str( + self.original_func_name) + stack_frame_json['file_name'] = _get_truncatable_str(self.file_name) + stack_frame_json['line_number'] = self.line_num + stack_frame_json['col_number'] = self.col_num + stack_frame_json['load_module'] = { + 'module': _get_truncatable_str(self.load_module), + 'build_id': _get_truncatable_str(self.build_id), + } + stack_frame_json['source_version'] = _get_truncatable_str( + self.source_version) + + return stack_frame_json + + +class StackTrace(object): + """A call stack appearing in a trace. + + :type stack_frames: list + :param stack_frames: Stack frames in this stack trace. A maximum of 128 + frames are allowed. + + :type stack_trace_hash_id: str + :param stack_trace_hash_id: The hash ID is used to conserve network + bandwidth for duplicate stack traces within a + single trace. + """ + def __init__(self, stack_frames=None, stack_trace_hash_id=None): + if stack_frames is None: + stack_frames = [] + + if stack_trace_hash_id is None: + stack_trace_hash_id = _generate_hash_id() + + self.stack_frames = stack_frames + self.stack_trace_hash_id = stack_trace_hash_id + + def add_stack_frame(self, stack_frame): + """Add StackFrame to frames list.""" + self.stack_frames.append(stack_frame) + + def format_stack_trace_json(self): + """Convert a StackTrace object to json format.""" + stack_trace_json = {} + + if stack_frames: + stack_trace_json['stack_frames'] = self.stack_frames + + stack_trace_json['stack_trace_hash_id'] = self.stack_trace_hash_id + + return stack_trace_json + + +def generate_hash_id(): + """Generate a hash id.""" + return random.getrandbits(64) diff --git a/trace/opencensus/trace/status.py b/trace/opencensus/trace/status.py new file mode 100644 index 000000000..ac1f24003 --- /dev/null +++ b/trace/opencensus/trace/status.py @@ -0,0 +1,55 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Status(object): + """The Status type defines a logical error model that is suitable for + different programming environments, including REST APIs and RPC APIs. + It is used by gRPC. + + :type code: int + :param code: An enum value of :class: `~google.rpc.Code`. + + :type message: str + :param message: A developer-facing error message, should be in English. + + :type details: list + :param details: A list of messages that carry the error details. + There is a common set of message types for APIs to use. + e.g. [ + { + "@type": string, + field1: ..., + ... + }, + ] + See: https://cloud.google.com/trace/docs/reference/v2/ + rest/v2/Status#FIELDS.details + """ + def __init__(self, code, message, details=None): + self.code = code + self.message = message + self.details = details + + def format_status_json(self): + """Convert a Status object to json format.""" + status_json = {} + + status_json['code'] = self.code + status_json['message'] = self.message + + if self.details is not None: + status_json['details'] = self.details + + return status_json diff --git a/trace/opencensus/trace/time_event.py b/trace/opencensus/trace/time_event.py new file mode 100644 index 000000000..e3c82044b --- /dev/null +++ b/trace/opencensus/trace/time_event.py @@ -0,0 +1,61 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Annotation(object): + pass + + +class MessageEvent(object): + pass + + +class TimeEvent(object): + """A collection of TimeEvents. A TimeEvent is a time-stamped annotation on + the span, consisting of either user-supplied key:value pairs, or details + of a message sent/received between Spans. + + :type timestamp: :class:`~datetime.datetime` + :param timestamp: The timestamp indicating the time the event occurred. + + :type annotation: :class: `~opencensus.trace.time_event.Annotation` + :param annotation: (Optional) Text annotation with a set of attributes. + + :type message_event: :class: `~opencensus.trace.time_event.MessageEvent` + :param message_event: (Optional) An event describing a message + sent/received between Spans. + """ + def __init__(self, timestamp, annotation=None, message_event=None): + self.timestamp = timestamp.isoformat() + 'Z' + self.annotation = annotation + self.message_event = message_event + + def format_time_event_json(self): + """Convert a TimeEvent object to json format.""" + time_event = {} + + time_event['time'] = self.timestamp + + if self.annotation is not None: + time_event['annotation'] = self.annotation + + return time_event + + if self.message_event is not None: + time_event['message_event'] = self.message_event + + return time_event + + raise ValueError("TimeEvent can contain either an Annotation or a" + "MessageEvent object, but not both.") diff --git a/trace/opencensus/trace/utils.py b/trace/opencensus/trace/utils.py new file mode 100644 index 000000000..2276ed1fa --- /dev/null +++ b/trace/opencensus/trace/utils.py @@ -0,0 +1,40 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +UTF8 = 'utf-8' + +# Max length is 128 bytes for a truncatable string. +MAX_LENGTH = 128 + + +def _get_truncatable_str(str_to_convert): + """Truncate a string if exceed limit and record the truncated bytes + count. + """ + str_bytes = str_to_convert.encode(UTF8) + str_len = len(str_bytes) + + truncated_byte_count = 0 + + if str_len > MAX_LENGTH: + truncated_byte_count = str_len - MAX_LENGTH + str_bytes = str_bytes[:MAX_LENGTH] + + truncated = str_bytes.decode(UTF8) + + result = { + 'value': truncated, + 'truncated_byte_count': truncated_byte_count, + } + return result From 16f51bf3a2edb462fee7937fb369b2f5888a91da Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 28 Nov 2017 18:07:28 -0800 Subject: [PATCH 02/15] Add unit tests --- trace/opencensus/trace/attributes.py | 8 +- trace/opencensus/trace/link.py | 11 +- trace/opencensus/trace/stack_trace.py | 54 +++++- trace/opencensus/trace/status.py | 6 +- trace/opencensus/trace/time_event.py | 52 +++--- .../ext/sqlalchemy/test_sqlalchemy_trace.py | 4 +- .../ext/{test_utils.py => test_ext_utils.py} | 0 trace/tests/unit/test_attributes.py | 78 +++++++++ trace/tests/unit/test_link.py | 98 +++++++++++ trace/tests/unit/test_stack_trace.py | 157 ++++++++++++++++++ trace/tests/unit/test_status.py | 65 ++++++++ trace/tests/unit/test_time_event.py | 101 +++++++++++ trace/tests/unit/test_utils.py | 47 ++++++ 13 files changed, 634 insertions(+), 47 deletions(-) rename trace/tests/unit/ext/{test_utils.py => test_ext_utils.py} (100%) create mode 100644 trace/tests/unit/test_attributes.py create mode 100644 trace/tests/unit/test_link.py create mode 100644 trace/tests/unit/test_stack_trace.py create mode 100644 trace/tests/unit/test_status.py create mode 100644 trace/tests/unit/test_time_event.py create mode 100644 trace/tests/unit/test_utils.py diff --git a/trace/opencensus/trace/attributes.py b/trace/opencensus/trace/attributes.py index dde4a2b72..cef4ef282 100644 --- a/trace/opencensus/trace/attributes.py +++ b/trace/opencensus/trace/attributes.py @@ -14,11 +14,11 @@ def _format_attribute_value(value): - if isinstance(value, str): + if type(value).__name__ == 'str': value_type = 'string_value' - elif isinstance(value, int): + elif type(value).__name__ == 'int': value_type = 'int_value' - elif isinstance(value, bool): + elif type(value).__name__ == 'bool': value_type = 'bool_value' else: raise TypeError("Value must be str, int, or bool.") @@ -28,7 +28,7 @@ def _format_attribute_value(value): class Attributes(object): """A set of attributes, each in the format [KEY]:[VALUE]. - + :type attributes: dict :param attributes: The set of attributes. Each attribute's key can be up to 128 bytes long. The value can be a string up to 256 diff --git a/trace/opencensus/trace/link.py b/trace/opencensus/trace/link.py index 182caf2f2..722fccc71 100644 --- a/trace/opencensus/trace/link.py +++ b/trace/opencensus/trace/link.py @@ -20,7 +20,8 @@ class Type(object): Attributes: TYPE_UNSPECIFIED (int): The relationship of the two spans is unknown. CHILD_LINKED_SPAN (int): The linked span is a child of the current span. - PARENT_LINKED_SPAN (int): The linked span is a parent of the current span. + PARENT_LINKED_SPAN (int): The linked span is a parent of the current + span. """ TYPE_UNSPECIFIED = 0 CHILD_LINKED_SPAN = 1 @@ -32,17 +33,17 @@ class Link(object): a different trace. For example, this can be used in batching operations, where a single batch handler processes multiple requests from different traces or when the handler receives a request from a different project. - + :type trace_id: str :param trace_id: The [TRACE_ID] for a trace within a project. - + :type span_id: str :param span_id: The [SPAN_ID] for a span within a trace. - + :type type: Enum of :class:`~opencensus.trace.link.Type` :param type: The relationship of the current span relative to the linked span. - + :type attributes: :class:`~opencensus.trace.attributes.Attributes` :param attributes: A set of attributes on the link. You have have up to 32 attributes per link. diff --git a/trace/opencensus/trace/stack_trace.py b/trace/opencensus/trace/stack_trace.py index cf82a1639..c7440238f 100644 --- a/trace/opencensus/trace/stack_trace.py +++ b/trace/opencensus/trace/stack_trace.py @@ -19,8 +19,43 @@ class StackFrame(object): """Represents a single stack frame in a stack trace. - - :type func_name + + :type func_name: str + :param func_name: The fully-qualified name that uniquely identifies the + function or method that is active in this frame (up to + 1024 bytes). + + :type original_func_name: str + :param original_func_name: An un-mangled function name, if functionName is + mangled. The name can be fully-qualified + (up to 1024 bytes). + + :type file_name: str + :param file_name: The name of the source file where the function call + appears (up to 256 bytes). + + :type line_num: int + :param line_num: The line number in fileName where the function call + appears. + + :type col_num: int + :param col_num: The column number where the function call appears, if + available. This is important in JavaScript because of its + anonymous functions. + + :type load_module: str + :param load_module: For example: main binary, kernel modules, and dynamic + libraries such as libc.so, sharedlib.so + (up to 256 bytes). + + :type build_id: str + :param build_id: A unique identifier for the module, usually a hash of its + contents (up to 128 bytes). + + + :type source_version: str + :param source_version: The version of the deployed source code + (up to 128 bytes). """ def __init__(self, func_name, @@ -30,7 +65,7 @@ def __init__(self, col_num, load_module, build_id, - source_version) + source_version): self.func_name = func_name self.original_func_name = original_func_name self.file_name = file_name @@ -43,7 +78,8 @@ def __init__(self, def format_stack_frame_json(self): """Convert StackFrame object to json format.""" stack_frame_json = {} - stack_frame_json['function_name'] = _get_truncatable_str(self.func_name) + stack_frame_json['function_name'] = _get_truncatable_str( + self.func_name) stack_frame_json['original_function_name'] = _get_truncatable_str( self.original_func_name) stack_frame_json['file_name'] = _get_truncatable_str(self.file_name) @@ -61,11 +97,11 @@ def format_stack_frame_json(self): class StackTrace(object): """A call stack appearing in a trace. - + :type stack_frames: list :param stack_frames: Stack frames in this stack trace. A maximum of 128 frames are allowed. - + :type stack_trace_hash_id: str :param stack_trace_hash_id: The hash ID is used to conserve network bandwidth for duplicate stack traces within a @@ -76,20 +112,20 @@ def __init__(self, stack_frames=None, stack_trace_hash_id=None): stack_frames = [] if stack_trace_hash_id is None: - stack_trace_hash_id = _generate_hash_id() + stack_trace_hash_id = generate_hash_id() self.stack_frames = stack_frames self.stack_trace_hash_id = stack_trace_hash_id def add_stack_frame(self, stack_frame): """Add StackFrame to frames list.""" - self.stack_frames.append(stack_frame) + self.stack_frames.append(stack_frame.format_stack_frame_json()) def format_stack_trace_json(self): """Convert a StackTrace object to json format.""" stack_trace_json = {} - if stack_frames: + if self.stack_frames: stack_trace_json['stack_frames'] = self.stack_frames stack_trace_json['stack_trace_hash_id'] = self.stack_trace_hash_id diff --git a/trace/opencensus/trace/status.py b/trace/opencensus/trace/status.py index ac1f24003..518c59b81 100644 --- a/trace/opencensus/trace/status.py +++ b/trace/opencensus/trace/status.py @@ -17,13 +17,13 @@ class Status(object): """The Status type defines a logical error model that is suitable for different programming environments, including REST APIs and RPC APIs. It is used by gRPC. - + :type code: int :param code: An enum value of :class: `~google.rpc.Code`. - + :type message: str :param message: A developer-facing error message, should be in English. - + :type details: list :param details: A list of messages that carry the error details. There is a common set of message types for APIs to use. diff --git a/trace/opencensus/trace/time_event.py b/trace/opencensus/trace/time_event.py index e3c82044b..0eb40f132 100644 --- a/trace/opencensus/trace/time_event.py +++ b/trace/opencensus/trace/time_event.py @@ -12,50 +12,54 @@ # See the License for the specific language governing permissions and # limitations under the License. +from opencensus.trace.utils import _get_truncatable_str + class Annotation(object): - pass + """Text annotation with a set of attributes. + + :type description: str + :param description: A user-supplied message describing the event. + The maximum length for the description is 256 bytes. + + :type attributes: :class:`~opencensus.trace.attributes.Attributes` + :param attributes: A set of attributes on the annotation. + You can have up to 4 attributes per Annotation. + """ + def __init__(self, description, attributes=None): + self.description = description + self.attributes = attributes + + def format_annotation_json(self): + annotation_json = {} + annotation_json['description'] = _get_truncatable_str(self.description) + if self.attributes is not None: + annotation_json['attributes'] = self.attributes.\ + format_attributes_json() -class MessageEvent(object): - pass + return annotation_json class TimeEvent(object): """A collection of TimeEvents. A TimeEvent is a time-stamped annotation on the span, consisting of either user-supplied key:value pairs, or details of a message sent/received between Spans. - + :type timestamp: :class:`~datetime.datetime` :param timestamp: The timestamp indicating the time the event occurred. - + :type annotation: :class: `~opencensus.trace.time_event.Annotation` :param annotation: (Optional) Text annotation with a set of attributes. - - :type message_event: :class: `~opencensus.trace.time_event.MessageEvent` - :param message_event: (Optional) An event describing a message - sent/received between Spans. """ - def __init__(self, timestamp, annotation=None, message_event=None): + def __init__(self, timestamp, annotation): self.timestamp = timestamp.isoformat() + 'Z' self.annotation = annotation - self.message_event = message_event def format_time_event_json(self): """Convert a TimeEvent object to json format.""" time_event = {} - time_event['time'] = self.timestamp + time_event['annotation'] = self.annotation.format_annotation_json() - if self.annotation is not None: - time_event['annotation'] = self.annotation - - return time_event - - if self.message_event is not None: - time_event['message_event'] = self.message_event - - return time_event - - raise ValueError("TimeEvent can contain either an Annotation or a" - "MessageEvent object, but not both.") + return time_event diff --git a/trace/tests/unit/ext/sqlalchemy/test_sqlalchemy_trace.py b/trace/tests/unit/ext/sqlalchemy/test_sqlalchemy_trace.py index 0cbebafe1..751702989 100644 --- a/trace/tests/unit/ext/sqlalchemy/test_sqlalchemy_trace.py +++ b/trace/tests/unit/ext/sqlalchemy/test_sqlalchemy_trace.py @@ -75,7 +75,7 @@ def test__before_cursor_execute(self): 'sqlalchemy/cursor/method/name': 'execute' } - expected_name = '[sqlalchemy.query]{}'.format(query) + expected_name = 'sqlalchemy.query' self.assertEqual(mock_tracer.current_span.labels, expected_labels) self.assertEqual(mock_tracer.current_span.name, expected_name) @@ -101,7 +101,7 @@ def test__before_cursor_executemany(self): 'sqlalchemy/cursor/method/name': 'executemany' } - expected_name = '[sqlalchemy.query]{}'.format(query) + expected_name = 'sqlalchemy.query' self.assertEqual(mock_tracer.current_span.labels, expected_labels) self.assertEqual(mock_tracer.current_span.name, expected_name) diff --git a/trace/tests/unit/ext/test_utils.py b/trace/tests/unit/ext/test_ext_utils.py similarity index 100% rename from trace/tests/unit/ext/test_utils.py rename to trace/tests/unit/ext/test_ext_utils.py diff --git a/trace/tests/unit/test_attributes.py b/trace/tests/unit/test_attributes.py new file mode 100644 index 000000000..6f2da6611 --- /dev/null +++ b/trace/tests/unit/test_attributes.py @@ -0,0 +1,78 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import attributes as attributes_module + + +class TestAttributes(unittest.TestCase): + def test_constructor_default(self): + attributes = attributes_module.Attributes() + self.assertEqual(attributes.attributes, {}) + + def test_constructor_explicit(self): + attr = { + 'key': 'value' + } + attributes = attributes_module.Attributes(attr) + + self.assertEqual(attributes.attributes, attr) + + def test_set_attribute(self): + key = 'test key' + value = 'test value' + attributes = attributes_module.Attributes() + attributes.set_attribute(key=key, value=value) + + expected_attr = { + key: value + } + + self.assertEqual(expected_attr, attributes.attributes) + + def test_get_attribute(self): + attr = { + 'key': 'value' + } + attributes = attributes_module.Attributes(attr) + value = attributes.get_attribute('key') + + self.assertEqual(value, 'value') + + def test_format_attributes_json(self): + attrs = { + 'key1': 'test string', + 'key2': True, + 'key3': 100, + } + + attributes = attributes_module.Attributes(attrs) + attributes_json = attributes.format_attributes_json() + + expected_attributes_json = { + 'key1': { + 'string_value': 'test string' + }, + 'key2': { + 'bool_value': True + }, + 'key3': { + 'int_value': 100 + } + } + + self.assertEqual(expected_attributes_json, attributes_json) diff --git a/trace/tests/unit/test_link.py b/trace/tests/unit/test_link.py new file mode 100644 index 000000000..4a179093e --- /dev/null +++ b/trace/tests/unit/test_link.py @@ -0,0 +1,98 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import link as link_module + + +class TestLink(unittest.TestCase): + def test_constructor_default(self): + trace_id = 'test trace id' + span_id = 'test span id' + type = link_module.Type.TYPE_UNSPECIFIED + attributes = mock.Mock() + + link = link_module.Link( + trace_id=trace_id, + span_id=span_id, + attributes=attributes) + + self.assertEqual(link.trace_id, trace_id) + self.assertEqual(link.span_id, span_id) + self.assertEqual(link.type, type) + self.assertEqual(link.attributes, attributes) + + def test_constructor_explicit(self): + trace_id = 'test trace id' + span_id = 'test span id' + type = link_module.Type.CHILD_LINKED_SPAN + attributes = mock.Mock() + + link = link_module.Link( + trace_id=trace_id, + span_id=span_id, + type=type, + attributes=attributes) + + self.assertEqual(link.trace_id, trace_id) + self.assertEqual(link.span_id, span_id) + self.assertEqual(link.type, type) + self.assertEqual(link.attributes, attributes) + + + def test_format_link_json_with_attributes(self): + trace_id = 'test trace id' + span_id = 'test span id' + type = link_module.Type.CHILD_LINKED_SPAN + attributes = mock.Mock() + + link = link_module.Link( + trace_id=trace_id, + span_id=span_id, + type=type, + attributes=attributes) + + link_json = link.format_link_json() + + expected_link_json = { + 'trace_id': trace_id, + 'span_id': span_id, + 'type': type, + 'attributes': attributes + } + + self.assertEqual(expected_link_json, link_json) + + def test_format_link_json_without_attributes(self): + trace_id = 'test trace id' + span_id = 'test span id' + type = link_module.Type.CHILD_LINKED_SPAN + + link = link_module.Link( + trace_id=trace_id, + span_id=span_id, + type=type) + + link_json = link.format_link_json() + + expected_link_json = { + 'trace_id': trace_id, + 'span_id': span_id, + 'type': type + } + + self.assertEqual(expected_link_json, link_json) diff --git a/trace/tests/unit/test_stack_trace.py b/trace/tests/unit/test_stack_trace.py new file mode 100644 index 000000000..3744bfcde --- /dev/null +++ b/trace/tests/unit/test_stack_trace.py @@ -0,0 +1,157 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import stack_trace as stack_trace_module + + +class TestStackFrame(unittest.TestCase): + def test_constructor(self): + func_name = 'func name' + original_func_name = 'original func name' + file_name = 'file name' + line_num = 10 + col_num = 1 + load_module = 'module' + build_id = 100 + source_version = 'source version' + + stack_frame = stack_trace_module.StackFrame( + func_name, + original_func_name, + file_name, + line_num, + col_num, + load_module, + build_id, + source_version) + + self.assertEqual(stack_frame.func_name, func_name) + self.assertEqual(stack_frame.original_func_name, original_func_name) + self.assertEqual(stack_frame.file_name, file_name) + self.assertEqual(stack_frame.line_num, line_num) + self.assertEqual(stack_frame.col_num, col_num) + self.assertEqual(stack_frame.load_module, load_module) + self.assertEqual(stack_frame.build_id, build_id) + self.assertEqual(stack_frame.source_version, source_version) + + def test_format_stack_frame_json(self): + func_name = 'func name' + original_func_name = 'original func name' + file_name = 'file name' + line_num = 10 + col_num = 1 + load_module = 'module' + build_id = 100 + source_version = 'source version' + + stack_frame = stack_trace_module.StackFrame( + func_name, + original_func_name, + file_name, + line_num, + col_num, + load_module, + build_id, + source_version) + + def mock_get_truncatable_str(str): + return str + + patch = mock.patch( + 'opencensus.trace.stack_trace._get_truncatable_str', + mock_get_truncatable_str) + + expected_stack_frame_json = { + 'function_name': func_name, + 'original_function_name': original_func_name, + 'file_name': file_name, + 'line_number': line_num, + 'col_number': col_num, + 'load_module': { + 'module': load_module, + 'build_id': build_id + }, + 'source_version': source_version + } + + with patch: + stack_frame_json = stack_frame.format_stack_frame_json() + + self.assertEqual(stack_frame_json, expected_stack_frame_json) + + +class TestStackTrace(unittest.TestCase): + def test_constructor_default(self): + hash_id = 1100 + patch = mock.patch( + 'opencensus.trace.stack_trace.generate_hash_id', + return_value=hash_id) + + with patch: + stack_trace = stack_trace_module.StackTrace() + + self.assertEqual(stack_trace.stack_frames, []) + self.assertEqual(stack_trace.stack_trace_hash_id, hash_id) + + def test_constructor_explicit(self): + stack_frames = [mock.Mock()] + hash_id = 1100 + stack_trace = stack_trace_module.StackTrace(stack_frames, hash_id) + + self.assertEqual(stack_trace.stack_frames, stack_frames) + self.assertEqual(stack_trace.stack_trace_hash_id, hash_id) + + def test_add_stack_frame(self): + stack_trace = stack_trace_module.StackTrace() + stack_frame = mock.Mock() + stack_frame_json = 'test stack frame' + stack_frame.format_stack_frame_json.return_value = stack_frame_json + stack_trace.add_stack_frame(stack_frame) + + self.assertEqual(stack_trace.stack_frames, [stack_frame_json]) + + def test_format_stack_trace_json_with_stack_frame(self): + hash_id = 1100 + stack_frame = [mock.Mock()] + + stack_trace = stack_trace_module.StackTrace( + stack_frames=stack_frame, + stack_trace_hash_id=hash_id) + + stack_trace_json = stack_trace.format_stack_trace_json() + + expected_stack_trace_json = { + 'stack_frames': stack_frame, + 'stack_trace_hash_id': hash_id + } + + self.assertEqual(expected_stack_trace_json, stack_trace_json) + + def test_format_stack_trace_json_without_stack_frame(self): + hash_id = 1100 + + stack_trace = stack_trace_module.StackTrace( + stack_trace_hash_id=hash_id) + + stack_trace_json = stack_trace.format_stack_trace_json() + + expected_stack_trace_json = { + 'stack_trace_hash_id': hash_id + } + + self.assertEqual(expected_stack_trace_json, stack_trace_json) diff --git a/trace/tests/unit/test_status.py b/trace/tests/unit/test_status.py new file mode 100644 index 000000000..70b8b8e7c --- /dev/null +++ b/trace/tests/unit/test_status.py @@ -0,0 +1,65 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import status as status_module + + +class TestStatus(unittest.TestCase): + def test_constructor(self): + code = 100 + message = 'test message' + status = status_module.Status(code=code, message=message) + + self.assertEqual(status.code, code) + self.assertEqual(status.message, message) + self.assertIsNone(status.details) + + def test_format_status_json_with_details(self): + code = 100 + message = 'test message' + details = [ + { + '@type': 'string', + 'field1': 'value', + }, + ] + status = status_module.Status( + code=code, message=message, details=details) + status_json = status.format_status_json() + + expected_status_json = { + 'code': code, + 'message': message, + 'details': details + } + + self.assertEqual(expected_status_json, status_json) + + def test_format_status_json_without_details(self): + code = 100 + message = 'test message' + + status = status_module.Status(code=code, message=message) + status_json = status.format_status_json() + + expected_status_json = { + 'code': code, + 'message': message + } + + self.assertEqual(expected_status_json, status_json) diff --git a/trace/tests/unit/test_time_event.py b/trace/tests/unit/test_time_event.py new file mode 100644 index 000000000..141e38104 --- /dev/null +++ b/trace/tests/unit/test_time_event.py @@ -0,0 +1,101 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import time_event as time_event_module + + +class TestAnnotation(unittest.TestCase): + def test_constructor(self): + description = 'test description' + attributes = mock.Mock() + + annotation = time_event_module.Annotation(description, attributes) + + self.assertEqual(annotation.description, description) + self.assertEqual(annotation.attributes, attributes) + + def test_format_annotation_json_with_attributes(self): + description = 'test description' + attrs_json = {} + attributes = mock.Mock() + attributes.format_attributes_json.return_value = attrs_json + + annotation = time_event_module.Annotation(description, attributes) + + annotation_json = annotation.format_annotation_json() + + expected_annotation_json = { + 'description': { + 'value': description, + 'truncated_byte_count': 0 + }, + 'attributes': {} + } + + self.assertEqual(annotation_json, expected_annotation_json) + + def test_format_annotation_json_without_attributes(self): + description = 'test description' + + annotation = time_event_module.Annotation(description) + + annotation_json = annotation.format_annotation_json() + + expected_annotation_json = { + 'description': { + 'value': description, + 'truncated_byte_count': 0 + } + } + + self.assertEqual(annotation_json, expected_annotation_json) + + +class TestTimeEvent(unittest.TestCase): + def test_constructor(self): + import datetime + + timestamp = datetime.datetime.utcnow() + annotation = mock.Mock() + + time_event = time_event_module.TimeEvent( + timestamp=timestamp, + annotation=annotation) + + self.assertEqual(time_event.timestamp, timestamp.isoformat() + 'Z') + self.assertEqual(time_event.annotation, annotation) + + def test_format_time_event_json(self): + import datetime + + timestamp = datetime.datetime.utcnow() + mock_annotation = 'test annotation' + annotation = mock.Mock() + annotation.format_annotation_json.return_value = mock_annotation + + time_event = time_event_module.TimeEvent( + timestamp=timestamp, + annotation=annotation) + + time_event_json = time_event.format_time_event_json() + expected_time_event_json = { + 'time': timestamp.isoformat() + 'Z', + 'annotation': mock_annotation + } + + self.assertEqual(time_event_json, expected_time_event_json) diff --git a/trace/tests/unit/test_utils.py b/trace/tests/unit/test_utils.py new file mode 100644 index 000000000..52c6691e3 --- /dev/null +++ b/trace/tests/unit/test_utils.py @@ -0,0 +1,47 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.trace import utils + + +class TestUtils(unittest.TestCase): + def test__get_truncatable_str(self): + str_to_convert = 'test string' + truncatable_str = utils._get_truncatable_str(str_to_convert) + + expected_str = { + 'value': str_to_convert, + 'truncated_byte_count': 0 + } + + self.assertEqual(expected_str, truncatable_str) + + def test__get_truncatable_str_length_exceeds(self): + max_len = 5 + str_to_convert = 'length exceeded' + patch = mock.patch('opencensus.trace.utils.MAX_LENGTH', max_len) + + with patch: + truncatable_str = utils._get_truncatable_str(str_to_convert) + + expected_str = { + 'value': 'lengt', + 'truncated_byte_count': 10 + } + + self.assertEqual(expected_str, truncatable_str) From 432d1787b566bef049d66ec55d3dfed85d625fe7 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 29 Nov 2017 10:05:27 -0800 Subject: [PATCH 03/15] Fix coverage --- trace/tests/unit/test_attributes.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/trace/tests/unit/test_attributes.py b/trace/tests/unit/test_attributes.py index 6f2da6611..838701dc5 100644 --- a/trace/tests/unit/test_attributes.py +++ b/trace/tests/unit/test_attributes.py @@ -76,3 +76,13 @@ def test_format_attributes_json(self): } self.assertEqual(expected_attributes_json, attributes_json) + + def test_format_attributes_json_type_error(self): + attrs = { + 'key1': mock.Mock(), + } + + attributes = attributes_module.Attributes(attrs) + + with self.assertRaises(TypeError): + attributes_json = attributes.format_attributes_json() From ead63866707945d89ef56ff9ceb51f9da9d11b05 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 29 Nov 2017 18:08:12 -0800 Subject: [PATCH 04/15] String value in attributes should be truncatable_str --- trace/opencensus/trace/attributes.py | 3 +++ trace/tests/unit/test_attributes.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/trace/opencensus/trace/attributes.py b/trace/opencensus/trace/attributes.py index cef4ef282..4595fb5af 100644 --- a/trace/opencensus/trace/attributes.py +++ b/trace/opencensus/trace/attributes.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from opencensus.trace.utils import _get_truncatable_str + def _format_attribute_value(value): if type(value).__name__ == 'str': value_type = 'string_value' + value = _get_truncatable_str(value) elif type(value).__name__ == 'int': value_type = 'int_value' elif type(value).__name__ == 'bool': diff --git a/trace/tests/unit/test_attributes.py b/trace/tests/unit/test_attributes.py index 838701dc5..a42f62df6 100644 --- a/trace/tests/unit/test_attributes.py +++ b/trace/tests/unit/test_attributes.py @@ -65,7 +65,10 @@ def test_format_attributes_json(self): expected_attributes_json = { 'key1': { - 'string_value': 'test string' + 'string_value': { + 'value': 'test string', + 'truncated_byte_count': 0 + } }, 'key2': { 'bool_value': True From bbbcda3d2107e7fcbd56badf620771ca9a9de1fa Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 30 Nov 2017 11:40:15 -0800 Subject: [PATCH 05/15] Add MessageEvent model --- trace/opencensus/trace/time_event.py | 89 ++++++++++++++++++++++- trace/tests/unit/test_time_event.py | 105 ++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 6 deletions(-) diff --git a/trace/opencensus/trace/time_event.py b/trace/opencensus/trace/time_event.py index 0eb40f132..2dcf27fb6 100644 --- a/trace/opencensus/trace/time_event.py +++ b/trace/opencensus/trace/time_event.py @@ -15,6 +15,20 @@ from opencensus.trace.utils import _get_truncatable_str +class Type(object): + """ + Indicates whether the message was sent or received. + + Attributes: + TYPE_UNSPECIFIED (int): Unknown event type. + SENT (int): Indicates a sent message. + RECEIVED (int): Indicates a received message. + """ + TYPE_UNSPECIFIED = 0 + SENT = 1 + RECEIVED = 2 + + class Annotation(object): """Text annotation with a set of attributes. @@ -41,25 +55,96 @@ def format_annotation_json(self): return annotation_json +class MessageEvent(object): + """An event describing a message sent/received between Spans. + + :type type: Enum of :class: `~opencensus.trace.time_event.Type` + :param type: Indicates whether the message was sent or received. + + :type id: str (int64 format) + :param id: An identifier for the MessageEvent's message that can be used + to match SENT and RECEIVED MessageEvents. It is recommended to + be unique within a Span. + + :type uncompressed_size_bytes: str (int64 format) + :param uncompressed_size_bytes: The number of uncompressed bytes sent or + received. + + :type compressed_size_bytes: str (int64 format) + :param compressed_size_bytes: The number of compressed bytes sent or + received. If missing assumed to be the same + size as uncompressed. + + """ + def __init__(self, id, type=None, uncompressed_size_bytes=None, + compressed_size_bytes=None): + if type is None: + type = Type.TYPE_UNSPECIFIED + + if compressed_size_bytes is None and \ + uncompressed_size_bytes is not None: + compressed_size_bytes = uncompressed_size_bytes + + self.id = id + self.type = type + self.uncompressed_size_bytes = uncompressed_size_bytes + self.compressed_size_bytes = compressed_size_bytes + + def format_message_event_json(self): + message_event_json = {} + + message_event_json['id'] = self.id + message_event_json['type'] = self.type + + if self.uncompressed_size_bytes is not None: + message_event_json[ + 'uncompressed_size_bytes'] = self.uncompressed_size_bytes + + if self.compressed_size_bytes is not None: + message_event_json[ + 'compressed_size_bytes'] = self.compressed_size_bytes + + return message_event_json + + class TimeEvent(object): """A collection of TimeEvents. A TimeEvent is a time-stamped annotation on the span, consisting of either user-supplied key:value pairs, or details of a message sent/received between Spans. + Note: A TimeEvent can contain either an Annotation object or a MessageEvent + object, but not both. + :type timestamp: :class:`~datetime.datetime` :param timestamp: The timestamp indicating the time the event occurred. :type annotation: :class: `~opencensus.trace.time_event.Annotation` :param annotation: (Optional) Text annotation with a set of attributes. + + :type message_event: :class: `~opencensus.trace.time_event.MessageEvent` + :param message_event: An event describing a message sent/received between + spans. """ - def __init__(self, timestamp, annotation): + def __init__(self, timestamp, annotation=None, message_event=None): self.timestamp = timestamp.isoformat() + 'Z' + + if annotation is not None and message_event is not None: + raise ValueError("A TimeEvent can contain either an Annotation" + "object or a MessageEvent object, but not both.") + self.annotation = annotation + self.message_event = message_event def format_time_event_json(self): """Convert a TimeEvent object to json format.""" time_event = {} time_event['time'] = self.timestamp - time_event['annotation'] = self.annotation.format_annotation_json() + + if self.annotation is not None: + time_event['annotation'] = self.annotation.format_annotation_json() + + if self.message_event is not None: + time_event['message_event'] = \ + self.message_event.format_message_event_json() return time_event diff --git a/trace/tests/unit/test_time_event.py b/trace/tests/unit/test_time_event.py index 141e38104..02b0fcdc3 100644 --- a/trace/tests/unit/test_time_event.py +++ b/trace/tests/unit/test_time_event.py @@ -66,21 +66,97 @@ def test_format_annotation_json_without_attributes(self): self.assertEqual(annotation_json, expected_annotation_json) +class TestMessageEvent(unittest.TestCase): + def test_constructor_default(self): + id = '1234' + + message_event = time_event_module.MessageEvent(id) + + self.assertEqual(message_event.id, id) + self.assertEqual( + message_event.type, time_event_module.Type.TYPE_UNSPECIFIED) + self.assertIsNone(message_event.uncompressed_size_bytes) + self.assertIsNone(message_event.compressed_size_bytes) + + def test_constructor_explicit(self): + id = '1234' + type = time_event_module.Type.SENT + uncompressed_size_bytes = '100' + + message_event = time_event_module.MessageEvent( + id, type, uncompressed_size_bytes) + + self.assertEqual(message_event.id, id) + self.assertEqual( + message_event.type, type) + self.assertEqual( + message_event.uncompressed_size_bytes, uncompressed_size_bytes) + self.assertEqual( + message_event.compressed_size_bytes, uncompressed_size_bytes) + + def test_format_message_event_json(self): + id = '1234' + type = time_event_module.Type.SENT + uncompressed_size_bytes = '100' + + message_event = time_event_module.MessageEvent( + id, type, uncompressed_size_bytes) + + expected_message_event_json = { + 'type': type, + 'id': id, + 'uncompressed_size_bytes': uncompressed_size_bytes, + 'compressed_size_bytes': uncompressed_size_bytes + } + + message_event_json = message_event.format_message_event_json() + + self.assertEqual(expected_message_event_json, message_event_json) + + def test_format_message_event_json_no_size(self): + id = '1234' + type = time_event_module.Type.SENT + + message_event = time_event_module.MessageEvent(id, type) + + expected_message_event_json = { + 'type': type, + 'id': id, + } + + message_event_json = message_event.format_message_event_json() + + self.assertEqual(expected_message_event_json, message_event_json) + + class TestTimeEvent(unittest.TestCase): def test_constructor(self): import datetime timestamp = datetime.datetime.utcnow() - annotation = mock.Mock() + message_event = mock.Mock() time_event = time_event_module.TimeEvent( timestamp=timestamp, - annotation=annotation) + message_event=message_event) self.assertEqual(time_event.timestamp, timestamp.isoformat() + 'Z') - self.assertEqual(time_event.annotation, annotation) + self.assertEqual(time_event.message_event, message_event) - def test_format_time_event_json(self): + def test_constructor_value_error(self): + import datetime + + timestamp = datetime.datetime.utcnow() + annotation = mock.Mock() + message_event = mock.Mock() + + with self.assertRaises(ValueError): + time_event = time_event_module.TimeEvent( + timestamp=timestamp, + annotation=annotation, + message_event=message_event) + + def test_format_time_event_json_annotation(self): import datetime timestamp = datetime.datetime.utcnow() @@ -99,3 +175,24 @@ def test_format_time_event_json(self): } self.assertEqual(time_event_json, expected_time_event_json) + + def test_format_time_event_json_message_event(self): + import datetime + + timestamp = datetime.datetime.utcnow() + mock_message_event = 'test annotation' + message_event = mock.Mock() + message_event.format_message_event_json.return_value = \ + mock_message_event + + time_event = time_event_module.TimeEvent( + timestamp=timestamp, + message_event=message_event) + + time_event_json = time_event.format_time_event_json() + expected_time_event_json = { + 'time': timestamp.isoformat() + 'Z', + 'message_event': mock_message_event + } + + self.assertEqual(time_event_json, expected_time_event_json) From c5ebe8fb0da014f8f5206a069030af47d1744754 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 30 Nov 2017 17:44:01 -0800 Subject: [PATCH 06/15] [WIP] Update StackdriverExporter to use V2 API --- .../trace/exporters/stackdriver_exporter.py | 35 +++++---- trace/opencensus/trace/span.py | 73 +++++++++++++++++-- 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/trace/opencensus/trace/exporters/stackdriver_exporter.py b/trace/opencensus/trace/exporters/stackdriver_exporter.py index 7844c4db6..3beb16d67 100644 --- a/trace/opencensus/trace/exporters/stackdriver_exporter.py +++ b/trace/opencensus/trace/exporters/stackdriver_exporter.py @@ -98,26 +98,35 @@ def __init__(self, client=None, project_id=None, self.project_id = client.project self.transport = transport(self) - def emit(self, trace): + def emit(self, spans): """ - :type trace: dict - :param trace: Trace collected. + :type spans: dict + :param spans: Spans collected. """ - stackdriver_traces = self.translate_to_stackdriver(trace) - self.client.patch_traces(stackdriver_traces) + name = 'projects/{}'.format(self.project_id) + stackdriver_spans = self.translate_to_stackdriver(spans) + self.client.batch_write_spans(name, stackdriver_spans) def export(self, trace): self.transport.export(trace) - def translate_to_stackdriver(self, trace): + def translate_to_stackdriver(self, spans): """ - :type trace: dict - :param trace: Trace collected. + :type spans: dict + :param spans: Spans collected. :rtype: dict - :returns: Traces in Google Cloud StackDriver Trace format. + :returns: Spans in Google Cloud StackDriver Trace format. """ - set_labels(trace) - trace['projectId'] = self.project_id - traces = {'traces': [trace]} - return traces + trace_id = spans.get('traceId') + spans_json = spans.get('spans') + + for span_json in spans_json: + span_name = 'projects/{}/traces/{}/spans/{}'.format( + self.project_id, trace_id, span_json.get('spanId')) + span_json['name'] = span_name + span_json['spanId'] = str(span_json['spanId']) + set_labels(span_json) + + spans = {'spans': [spans_json]} + return spans diff --git a/trace/opencensus/trace/span.py b/trace/opencensus/trace/span.py index 63ea9963d..6fcc09eaf 100644 --- a/trace/opencensus/trace/span.py +++ b/trace/opencensus/trace/span.py @@ -15,10 +15,11 @@ from datetime import datetime from itertools import chain +from opencensus.trace import stack_trace from opencensus.trace.enums import Enum from opencensus.trace.span_context import generate_span_id from opencensus.trace.tracer import base - +from opencensus.trace.utils import _get_truncatable_str class Span(object): """A span is an individual timed event which forms a node of the trace @@ -58,6 +59,27 @@ class Span(object): :type span_id: int :param span_id: Identifier for the span, unique within a trace. + :type stack_trace: :class: `~opencensus.trace.stack_trace.StackTrace` + :param stack_trace: (Optional) A call stack appearing in a trace + + :type time_events: list + :param time_events: (Optional) A set of time events. You can have up to 32 + annotations and 128 message events per span. + + :type links: list + :param links: (Optional) Links associated with the span. You can have up + to 128 links per Span. + + :type status: :class: `~opencensus.trace.status.Status` + :param status: (Optional) An optional final status for this span. + + :type same_process_as_parent_span: bool + :param same_process_as_parent_span: (Optional) A highly recommended but not + required flag that identifies when a + trace crosses a process boundary. + True when the parent_span belongs to + the same process as the current span. + :type context_tracer: :class:`~opencensus.trace.tracer.context_tracer. ContextTracer` :param context_tracer: The tracer that holds a stack of spans. If this is @@ -70,15 +92,18 @@ class Span(object): def __init__( self, name, - kind=Enum.SpanKind.SPAN_KIND_UNSPECIFIED, parent_span=None, labels=None, start_time=None, end_time=None, span_id=None, + stack_trace=None, + time_events=None, + links=None, + status=None, + same_process_as_parent_span=None, context_tracer=None): self.name = name - self.kind = kind self.parent_span = parent_span self.start_time = start_time self.end_time = end_time @@ -94,8 +119,19 @@ def __init__( if parent_span is None: parent_span = base.NullContextManager() + if time_events is None: + time_events = [] + + if links is None: + links = [] + self.labels = labels self.span_id = span_id + self.stack_trace = stack_trace + self.time_events = time_events + self.links = links + self.status = status + self.same_process_as_parent_span = same_process_as_parent_span self._child_spans = [] self.context_tracer = context_tracer @@ -129,6 +165,14 @@ def add_label(self, label_key, label_value): """ self.labels[label_key] = label_value + def add_time_event(self, time_event): + """Add a TimeEvent.""" + self.time_events.append(time_event) + + def add_link(self, link): + """Add a Link.""" + self.links.append(link) + def start(self): """Set the start time for a span.""" self.start_time = datetime.utcnow().isoformat() + 'Z' @@ -167,8 +211,7 @@ def format_span_json(span): :returns: Formatted Span. """ span_json = { - 'name': span.name, - 'kind': span.kind, + 'displayName': _get_truncatable_str(span.name), 'spanId': span.span_id, 'startTime': span.start_time, 'endTime': span.end_time, @@ -182,7 +225,25 @@ def format_span_json(span): if parent_span_id is not None: span_json['parentSpanId'] = parent_span_id - if span.labels is not None: + if span.labels: span_json['labels'] = span.labels + if span.stack_trace is not None: + span_json['stackTrace'] = span.stack_trace.format_stack_trace_json() + + if span.time_events: + span_json['timeEvents'] = [ + time_event.format_time_event_json() + for time_event in span.time_events] + + if span.links: + span_json['links'] = [link.format_link_json() for link in span.links] + + if span.status is not None: + span_json['status'] = span.status.format_status_json() + + if span.same_process_as_parent_span is not None: + span_json['sameProcessAsParentSpan'] = \ + span.same_process_as_parent_span + return span_json From ef408cac77ae1d6b4eee9361f6c94d406cca40fa Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 1 Dec 2017 17:37:05 -0800 Subject: [PATCH 07/15] [WIP] Update StackdriverExporter to use V2 --- trace/opencensus/trace/attributes.py | 6 ++- .../trace/exporters/stackdriver_exporter.py | 43 +++++++++-------- trace/opencensus/trace/span.py | 7 ++- trace/opencensus/trace/span_context.py | 4 +- .../exporters/test_stackdriver_exporter.py | 46 ++++++++++++------- .../tests/unit/tracer/test_context_tracer.py | 21 +++++++++ 6 files changed, 86 insertions(+), 41 deletions(-) diff --git a/trace/opencensus/trace/attributes.py b/trace/opencensus/trace/attributes.py index 4595fb5af..b9ab10251 100644 --- a/trace/opencensus/trace/attributes.py +++ b/trace/opencensus/trace/attributes.py @@ -60,4 +60,8 @@ def format_attributes_json(self): value_json = _format_attribute_value(value) attributes_json[key] = value_json - return attributes_json + result = { + 'attributeMap': attributes_json + } + + return result diff --git a/trace/opencensus/trace/exporters/stackdriver_exporter.py b/trace/opencensus/trace/exporters/stackdriver_exporter.py index 0b0b14c62..2dd1dee80 100644 --- a/trace/opencensus/trace/exporters/stackdriver_exporter.py +++ b/trace/opencensus/trace/exporters/stackdriver_exporter.py @@ -19,6 +19,8 @@ from google.cloud.trace.client import Client +from opencensus.trace.utils import _get_truncatable_str + # Environment variable set in App Engine when vm:true is set. _APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' @@ -118,27 +120,30 @@ def translate_to_stackdriver(self, spans): :rtype: dict :returns: Spans in Google Cloud StackDriver Trace format. """ - set_attributes(trace) - spans = trace.get('spans') - trace_id = trace.get('traceId') - spans_json = [] - - for span in spans: + set_attributes(spans) + spans_json = spans.get('spans') + trace_id = spans.get('traceId') + spans_list = [] + + for span in spans_json: + span_name = 'projects/{}/traces/{}/spans/{}'.format( + self.project_id, trace_id, span.get('spanId')) span_json = { - 'name': span.get('name'), + 'name': span_name, + 'displayName': span.get('displayName'), 'startTime': span.get('startTime'), 'endTime': span.get('endTime'), - 'spanId': span.get('spanId'), - 'parentSpanId': span.get('parentSpanId'), - 'labels': span.get('attributes') + 'spanId': str(span.get('spanId')), + 'parentSpanId': str(span.get('parentSpanId')), + 'attributes': span.get('attributes'), + 'links': span.get('links'), + 'status': span.get('status'), + 'stackTrace': span.get('stackTrace'), + 'timeEvents': span.get('timeEvents'), + 'sameProcessAsParentSpan': span.get('sameProcessAsParentSpan'), + 'childSpanCount': span.get('childSpanCount') } - spans_json.append(span_json) - - trace_json = { - 'projectId': self.project_id, - 'traceId': trace_id, - 'spans': spans_json - } + spans_list.append(span_json) - traces = {'traces': [trace_json]} - return traces + spans = {'spans': spans_list} + return spans diff --git a/trace/opencensus/trace/span.py b/trace/opencensus/trace/span.py index a26f0854d..80a876392 100644 --- a/trace/opencensus/trace/span.py +++ b/trace/opencensus/trace/span.py @@ -15,6 +15,7 @@ from datetime import datetime from itertools import chain +from opencensus.trace import attributes from opencensus.trace import stack_trace from opencensus.trace.enums import Enum from opencensus.trace.span_context import generate_span_id @@ -215,6 +216,7 @@ def format_span_json(span): 'spanId': span.span_id, 'startTime': span.start_time, 'endTime': span.end_time, + 'childSpanCount': len(span._child_spans) } parent_span_id = None @@ -225,8 +227,9 @@ def format_span_json(span): if parent_span_id is not None: span_json['parentSpanId'] = parent_span_id - if span.attributes is not None: - span_json['attributes'] = span.attributes + if span.attributes: + span_json['attributes'] = attributes.Attributes( + span.attributes).format_attributes_json() if span.stack_trace is not None: span_json['stackTrace'] = span.stack_trace.format_stack_trace_json() diff --git a/trace/opencensus/trace/span_context.py b/trace/opencensus/trace/span_context.py index f2a787cdf..3a4e0c797 100644 --- a/trace/opencensus/trace/span_context.py +++ b/trace/opencensus/trace/span_context.py @@ -150,13 +150,13 @@ def check_trace_id(self, trace_id): def generate_span_id(): - """Return the random generated span ID for a span. + """Return the random generated span ID for a span. Must be 16 digits. :rtype: int :returns: Identifier for the span. Must be a 64-bit integer other than 0 and unique within a trace. """ - span_id = random.getrandbits(64) + span_id = random.randint(10**15, 10**16 - 1) return span_id diff --git a/trace/tests/unit/exporters/test_stackdriver_exporter.py b/trace/tests/unit/exporters/test_stackdriver_exporter.py index 72b6c8fc1..54d6e86e1 100644 --- a/trace/tests/unit/exporters/test_stackdriver_exporter.py +++ b/trace/tests/unit/exporters/test_stackdriver_exporter.py @@ -97,13 +97,17 @@ def test_translate_to_stackdriver(self): trace = { 'spans': [ { - 'name': span_name, + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, 'spanId': span_id, 'startTime': start_time, 'endTime': end_time, 'parentSpanId': parent_span_id, 'attributes': attributes, - 'someRandomKey': 'this should not be included in result' + 'someRandomKey': 'this should not be included in result', + 'childSpanCount': 0 } ], 'traceId': trace_id @@ -116,28 +120,36 @@ def test_translate_to_stackdriver(self): project_id=project_id) - traces = exporter.translate_to_stackdriver(trace) + spans = exporter.translate_to_stackdriver(trace) expected_traces = { - 'traces': [ + 'spans': [ { - 'projectId': project_id, - 'traceId': trace_id, - 'spans': [ - { - 'name': span_name, - 'spanId': span_id, - 'startTime': start_time, - 'endTime': end_time, - 'parentSpanId': parent_span_id, - 'labels': attributes - } - ] + 'name': 'projects/{}/traces/{}/spans/{}'.format( + project_id, trace_id, span_id), + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, + 'spanId': str(span_id), + 'startTime': start_time, + 'endTime': end_time, + 'parentSpanId': str(parent_span_id), + 'attributes': {}, + 'status': None, + 'links': None, + 'stackTrace': None, + 'timeEvents': None, + 'childSpanCount': 0, + 'sameProcessAsParentSpan': None } ] } - self.assertEqual(traces, expected_traces) + print(spans) + print(expected_traces) + + self.assertEqual(spans, expected_traces) class Test_set_attributes_gae(unittest.TestCase): diff --git a/trace/tests/unit/tracer/test_context_tracer.py b/trace/tests/unit/tracer/test_context_tracer.py index d93d65baa..40af3fd91 100644 --- a/trace/tests/unit/tracer/test_context_tracer.py +++ b/trace/tests/unit/tracer/test_context_tracer.py @@ -117,6 +117,13 @@ def test_end_span_active(self, mock_current_span): exporter = mock.Mock() tracer = context_tracer.ContextTracer(exporter=exporter) mock_span = mock.Mock() + mock_span.name = 'span' + mock_span._child_spans = [] + mock_span.status = None + mock_span.links = None + mock_span.stack_trace = None + mock_span.time_events = None + mock_span.attributes = {} mock_span.__iter__ = mock.Mock( return_value=iter([mock_span])) parent_span_id = 1234 @@ -134,6 +141,13 @@ def test_end_span_without_parent(self, mock_current_span): tracer = context_tracer.ContextTracer() mock_span = mock.Mock() + mock_span.name = 'span' + mock_span._child_spans = [] + mock_span.status = None + mock_span.links = None + mock_span.stack_trace = None + mock_span.time_events = None + mock_span.attributes = {} mock_span.__iter__ = mock.Mock( return_value=iter([mock_span])) mock_current_span.return_value = mock_span @@ -151,6 +165,13 @@ def test_end_span_batch_export(self, mock_current_span): tracer = context_tracer.ContextTracer(exporter=exporter) tracer._spans_list = [span] mock_span = mock.Mock() + mock_span.name = 'span' + mock_span._child_spans = [] + mock_span.status = None + mock_span.links = None + mock_span.stack_trace = None + mock_span.time_events = None + mock_span.attributes = {} mock_span.__iter__ = mock.Mock( return_value=iter([mock_span])) parent_span_id = 1234 From 6cd5b80fe4fd479d2757e8c01fbee989da52674b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 4 Dec 2017 12:08:59 -0800 Subject: [PATCH 08/15] Address comments --- trace/opencensus/trace/attributes.py | 33 +++++++++---------- .../trace/exporters/stackdriver_exporter.py | 6 +++- trace/opencensus/trace/span.py | 28 +++++++++++++--- trace/opencensus/trace/utils.py | 27 +++++++++++++++ 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/trace/opencensus/trace/attributes.py b/trace/opencensus/trace/attributes.py index b9ab10251..4c61e8568 100644 --- a/trace/opencensus/trace/attributes.py +++ b/trace/opencensus/trace/attributes.py @@ -12,17 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.trace.utils import _get_truncatable_str +from opencensus.trace import utils def _format_attribute_value(value): - if type(value).__name__ == 'str': - value_type = 'string_value' - value = _get_truncatable_str(value) - elif type(value).__name__ == 'int': - value_type = 'int_value' - elif type(value).__name__ == 'bool': + if isinstance(value, bool): value_type = 'bool_value' + elif isinstance(value, int): + value_type = 'int_value' + elif isinstance(value, str): + value_type = 'string_value' + value = utils._get_truncatable_str(value) else: raise TypeError("Value must be str, int, or bool.") @@ -38,27 +38,26 @@ class Attributes(object): bytes, an integer, or the Boolean values true and false. """ def __init__(self, attributes=None): - if attributes is None: - attributes = {} - - self.attributes = attributes + self.attributes = dict(attributes or {}) def set_attribute(self, key, value): """Set a key value pair.""" self.attributes[key] = value + def delete_attribute(self, key): + """Delete an attribute given a key if existed.""" + self.attributes.pop(key, None) + def get_attribute(self, key): """Get a attribute value.""" return self.attributes.get(key, None) def format_attributes_json(self): """Convert the Attributes object to json format.""" - attributes_json = {} - - for key in self.attributes: - value = self.attributes.get(key) - value_json = _format_attribute_value(value) - attributes_json[key] = value_json + attribute_json = { + utils.check_str_length(key): _format_attribute_value(value) \ + for key, value in self.attributes + } result = { 'attributeMap': attributes_json diff --git a/trace/opencensus/trace/exporters/stackdriver_exporter.py b/trace/opencensus/trace/exporters/stackdriver_exporter.py index 2dd1dee80..da5ca159b 100644 --- a/trace/opencensus/trace/exporters/stackdriver_exporter.py +++ b/trace/opencensus/trace/exporters/stackdriver_exporter.py @@ -113,7 +113,11 @@ def export(self, trace): self.transport.export(trace) def translate_to_stackdriver(self, spans): - """ + """Translate the spans json to Stackdriver format. + + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ + projects.traces/batchWrite + :type spans: dict :param spans: Spans collected. diff --git a/trace/opencensus/trace/span.py b/trace/opencensus/trace/span.py index 80a876392..843468993 100644 --- a/trace/opencensus/trace/span.py +++ b/trace/opencensus/trace/span.py @@ -16,7 +16,9 @@ from itertools import chain from opencensus.trace import attributes -from opencensus.trace import stack_trace +from opencensus.trace import link as link_module +from opencensus.trace import stack_trace as stack_trace_module +from opencensus.trace import time_event as time_event_module from opencensus.trace.enums import Enum from opencensus.trace.span_context import generate_span_id from opencensus.trace.tracer import base @@ -167,12 +169,28 @@ def add_attribute(self, attribute_key, attribute_value): self.attributes[attribute_key] = attribute_value def add_time_event(self, time_event): - """Add a TimeEvent.""" - self.time_events.append(time_event) + """Add a TimeEvent. + + :type time_event: :class: `~opencensus.trace.time_event.TimeEvent` + :param time_event: A TimeEvent object. + """ + if isinstance(time_event, time_event_module.TimeEvent): + self.time_events.append(time_event) + else: + raise TypeError("Type Error: received {}, but requires TimeEvent.". + format(type(time_event).__name__)) def add_link(self, link): - """Add a Link.""" - self.links.append(link) + """Add a Link. + + :type link: :class: `~opencensus.trace.link.Link` + :param link: A Link object. + """ + if isinstance(link, link_module.Link): + self.links.append(link) + else: + raise TypeError("Type Error: received {}, but requires Link.". + format(type(link).__name__)) def start(self): """Set the start time for a span.""" diff --git a/trace/opencensus/trace/utils.py b/trace/opencensus/trace/utils.py index 2276ed1fa..71515c46a 100644 --- a/trace/opencensus/trace/utils.py +++ b/trace/opencensus/trace/utils.py @@ -38,3 +38,30 @@ def _get_truncatable_str(str_to_convert): 'truncated_byte_count': truncated_byte_count, } return result + + +def check_str_length(str_to_check, limit=None): + """Check the length of a string. If exceeds limit, then truncate it. + + :type str_to_check: str + :param str_to_check: String to check. + + :type limit: int + :param limit: The upper limit of the length. + + :rtype: str + :returns: The string it self if not exceeded length, or truncated string + if exceeded. + """ + if limit is None: + limit = MAX_LENGTH + + str_bytes = str_to_check.encode(UTF8) + str_len = len(str_bytes) + + if str_len > limit: + str_bytes = str_bytes[:limit] + + result = str_bytes.decode(UTF8) + + return result From 9d6da023360a6bd1351766340de3ecbcdf3702cf Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 5 Dec 2017 17:53:24 -0800 Subject: [PATCH 09/15] [WIP] Add unit tests --- opencensus/trace/attributes.py | 6 +- opencensus/trace/enums.py | 41 ----- .../trace/exporters/stackdriver_exporter.py | 5 +- opencensus/trace/link.py | 2 +- opencensus/trace/span.py | 25 ++- opencensus/trace/utils.py | 8 +- .../exporters/test_stackdriver_exporter.py | 11 +- tests/unit/trace/test_attributes.py | 35 +++-- tests/unit/trace/test_request_tracer.py | 11 +- tests/unit/trace/test_span.py | 147 +++++++++++++++--- 10 files changed, 184 insertions(+), 107 deletions(-) delete mode 100644 opencensus/trace/enums.py diff --git a/opencensus/trace/attributes.py b/opencensus/trace/attributes.py index 4c61e8568..01e3390c9 100644 --- a/opencensus/trace/attributes.py +++ b/opencensus/trace/attributes.py @@ -54,9 +54,9 @@ def get_attribute(self, key): def format_attributes_json(self): """Convert the Attributes object to json format.""" - attribute_json = { - utils.check_str_length(key): _format_attribute_value(value) \ - for key, value in self.attributes + attributes_json = { + utils.check_str_length(key): _format_attribute_value(value) + for key, value in self.attributes.items() } result = { diff --git a/opencensus/trace/enums.py b/opencensus/trace/enums.py deleted file mode 100644 index c1041e8b9..000000000 --- a/opencensus/trace/enums.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Wrappers for protocol buffer enum types. - -See -https://cloud.google.com/trace/docs/reference/v1/rpc/google.devtools. -cloudtrace.v1#google.devtools.cloudtrace.v1.ListTracesRequest.ViewType - -https://cloud.google.com/trace/docs/reference/v1/rpc/google.devtools. -cloudtrace.v1#google.devtools.cloudtrace.v1.TraceSpan.SpanKind -""" - - -class Enum(object): - class SpanKind(object): - """ - Type of span. Can be used to specify additional relationships between - spans in addition to a parent/child relationship. - - Attributes: - SPAN_KIND_UNSPECIFIED (int): Unspecified. - RPC_SERVER (int): Indicates that the span covers server-side handling - of an RPC or other remote network request. - RPC_CLIENT (int): Indicates that the span covers the client-side - wrapper around an RPC or other remote request. - """ - SPAN_KIND_UNSPECIFIED = 0 - RPC_SERVER = 1 - RPC_CLIENT = 2 diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index da5ca159b..d4ff64bee 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -19,7 +19,6 @@ from google.cloud.trace.client import Client -from opencensus.trace.utils import _get_truncatable_str # Environment variable set in App Engine when vm:true is set. _APPENGINE_FLEXIBLE_ENV_VM = 'GAE_APPENGINE_HOSTNAME' @@ -114,10 +113,10 @@ def export(self, trace): def translate_to_stackdriver(self, spans): """Translate the spans json to Stackdriver format. - + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ projects.traces/batchWrite - + :type spans: dict :param spans: Spans collected. diff --git a/opencensus/trace/link.py b/opencensus/trace/link.py index 722fccc71..e66a1f0e0 100644 --- a/opencensus/trace/link.py +++ b/opencensus/trace/link.py @@ -59,7 +59,7 @@ def __init__(self, trace_id, span_id, type=None, attributes=None): self.attributes = attributes def format_link_json(self): - """Conver a Link object to json format.""" + """Convert a Link object to json format.""" link_json = {} link_json['trace_id'] = self.trace_id link_json['span_id'] = self.span_id diff --git a/opencensus/trace/span.py b/opencensus/trace/span.py index 843468993..f15cb1999 100644 --- a/opencensus/trace/span.py +++ b/opencensus/trace/span.py @@ -17,13 +17,12 @@ from opencensus.trace import attributes from opencensus.trace import link as link_module -from opencensus.trace import stack_trace as stack_trace_module from opencensus.trace import time_event as time_event_module -from opencensus.trace.enums import Enum from opencensus.trace.span_context import generate_span_id from opencensus.trace.tracer import base from opencensus.trace.utils import _get_truncatable_str + class Span(object): """A span is an individual timed event which forms a node of the trace tree. Each span has its name, span id and parent id. The parent id @@ -36,12 +35,6 @@ class Span(object): :type name: str :param name: The name of the span. - :type kind: :class:`~opencensus.trace.enums.Enums.SpanKind` - :param kind: Distinguishes between spans generated in a particular context. - For example, two spans with the same name may be - distinguished using RPC_CLIENT and RPC_SERVER to identify - queueing latency associated with the span. - :type parent_span: :class:`~opencensus.trace.span.Span` :param parent_span: (Optional) Parent span. @@ -170,7 +163,7 @@ def add_attribute(self, attribute_key, attribute_value): def add_time_event(self, time_event): """Add a TimeEvent. - + :type time_event: :class: `~opencensus.trace.time_event.TimeEvent` :param time_event: A TimeEvent object. """ @@ -182,7 +175,7 @@ def add_time_event(self, time_event): def add_link(self, link): """Add a Link. - + :type link: :class: `~opencensus.trace.link.Link` :param link: A Link object. """ @@ -253,12 +246,16 @@ def format_span_json(span): span_json['stackTrace'] = span.stack_trace.format_stack_trace_json() if span.time_events: - span_json['timeEvents'] = [ - time_event.format_time_event_json() - for time_event in span.time_events] + span_json['timeEvents'] = { + 'timeEvent': [time_event.format_time_event_json() + for time_event in span.time_events] + } if span.links: - span_json['links'] = [link.format_link_json() for link in span.links] + span_json['links'] = { + 'link': [ + link.format_link_json() for link in span.links] + } if span.status is not None: span_json['status'] = span.status.format_status_json() diff --git a/opencensus/trace/utils.py b/opencensus/trace/utils.py index 71515c46a..063a537fb 100644 --- a/opencensus/trace/utils.py +++ b/opencensus/trace/utils.py @@ -31,7 +31,7 @@ def _get_truncatable_str(str_to_convert): truncated_byte_count = str_len - MAX_LENGTH str_bytes = str_bytes[:MAX_LENGTH] - truncated = str_bytes.decode(UTF8) + truncated = str_bytes.decode(UTF8, errors='ignore') result = { 'value': truncated, @@ -42,13 +42,13 @@ def _get_truncatable_str(str_to_convert): def check_str_length(str_to_check, limit=None): """Check the length of a string. If exceeds limit, then truncate it. - + :type str_to_check: str :param str_to_check: String to check. :type limit: int :param limit: The upper limit of the length. - + :rtype: str :returns: The string it self if not exceeded length, or truncated string if exceeded. @@ -62,6 +62,6 @@ def check_str_length(str_to_check, limit=None): if str_len > limit: str_bytes = str_bytes[:limit] - result = str_bytes.decode(UTF8) + result = str_bytes.decode(UTF8, errors='ignore') return result diff --git a/tests/unit/trace/exporters/test_stackdriver_exporter.py b/tests/unit/trace/exporters/test_stackdriver_exporter.py index 54d6e86e1..b6b550be2 100644 --- a/tests/unit/trace/exporters/test_stackdriver_exporter.py +++ b/tests/unit/trace/exporters/test_stackdriver_exporter.py @@ -67,7 +67,7 @@ def test_export(self): self.assertTrue(exporter.transport.export_called) def test_emit(self): - trace = {'spans': [], 'traceId': '6e0c63257de34c92bf9efcd03927272e'} + spans = {'spans': []} client = mock.Mock() project_id = 'PROJECT' @@ -77,13 +77,12 @@ def test_emit(self): client=client, project_id=project_id) - exporter.emit(trace) + exporter.emit(spans) - trace['projectId'] = project_id - traces = {'traces': [trace]} + name = 'projects/{}'.format(project_id) - client.patch_traces.assert_called_with(traces) - self.assertTrue(client.patch_traces.called) + client.batch_write_spans.assert_called_with(name, spans) + self.assertTrue(client.batch_write_spans.called) def test_translate_to_stackdriver(self): project_id = 'PROJECT' diff --git a/tests/unit/trace/test_attributes.py b/tests/unit/trace/test_attributes.py index a42f62df6..cb39f25cb 100644 --- a/tests/unit/trace/test_attributes.py +++ b/tests/unit/trace/test_attributes.py @@ -44,6 +44,19 @@ def test_set_attribute(self): self.assertEqual(expected_attr, attributes.attributes) + def test_delete_attribute(self): + attr = { + 'key1': 'value1', + 'key2': 'value2' + } + attributes = attributes_module.Attributes(attr) + attributes.delete_attribute('key1') + + self.assertEqual(attributes.attributes, { + 'key2': 'value2' + }) + + def test_get_attribute(self): attr = { 'key': 'value' @@ -64,17 +77,19 @@ def test_format_attributes_json(self): attributes_json = attributes.format_attributes_json() expected_attributes_json = { - 'key1': { - 'string_value': { - 'value': 'test string', - 'truncated_byte_count': 0 + 'attributeMap': { + 'key1': { + 'string_value': { + 'value': 'test string', + 'truncated_byte_count': 0 + } + }, + 'key2': { + 'bool_value': True + }, + 'key3': { + 'int_value': 100 } - }, - 'key2': { - 'bool_value': True - }, - 'key3': { - 'int_value': 100 } } diff --git a/tests/unit/trace/test_request_tracer.py b/tests/unit/trace/test_request_tracer.py index 2878c7d14..b13e8b4f7 100644 --- a/tests/unit/trace/test_request_tracer.py +++ b/tests/unit/trace/test_request_tracer.py @@ -194,10 +194,19 @@ def test_end_span_sampled(self): sampler.should_sample.return_value = True tracer = request_tracer.RequestTracer(sampler=sampler) span = mock.Mock() + span._child_spans = [] + span.attributes = {} + span.time_events = [] + span.links = [] span.__iter__ = mock.Mock( return_value=iter([span])) execution_context.set_current_span(span) - tracer.end_span() + + patch = mock.patch( + 'opencensus.trace.span._get_truncatable_str', mock.Mock()) + + with patch: + tracer.end_span() self.assertTrue(span.finish.called) diff --git a/tests/unit/trace/test_span.py b/tests/unit/trace/test_span.py index ade604aa5..97943910a 100644 --- a/tests/unit/trace/test_span.py +++ b/tests/unit/trace/test_span.py @@ -16,6 +16,10 @@ import mock +from opencensus.trace.stack_trace import StackTrace +from opencensus.trace.status import Status +from opencensus.trace.time_event import TimeEvent + class TestSpan(unittest.TestCase): @@ -31,8 +35,6 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_constructor_defaults(self): - from opencensus.trace.enums import Enum - span_id = 'test_span_id' span_name = 'test_span_name' @@ -45,7 +47,6 @@ def test_constructor_defaults(self): self.assertEqual(span.name, span_name) self.assertEqual(span.span_id, span_id) - self.assertEqual(span.kind, Enum.SpanKind.SPAN_KIND_UNSPECIFIED) self.assertIsNone(span.parent_span) self.assertEqual(span.attributes, {}) self.assertIsNone(span.start_time) @@ -56,11 +57,8 @@ def test_constructor_defaults(self): def test_constructor_explicit(self): from datetime import datetime - from opencensus.trace.enums import Enum - span_id = 'test_span_id' span_name = 'test_span_name' - kind = Enum.SpanKind.RPC_CLIENT parent_span = mock.Mock() start_time = datetime.utcnow().isoformat() + 'Z' end_time = datetime.utcnow().isoformat() + 'Z' @@ -68,37 +66,44 @@ def test_constructor_explicit(self): '/http/status_code': '200', '/component': 'HTTP load balancer', } + time_events = mock.Mock() + links = mock.Mock() + stack_trace = mock.Mock() + status = mock.Mock() context_tracer = mock.Mock() span = self._make_one( name=span_name, - kind=kind, parent_span=parent_span, attributes=attributes, start_time=start_time, end_time=end_time, span_id=span_id, + stack_trace=stack_trace, + time_events=time_events, + links=links, + status=status, context_tracer=context_tracer) self.assertEqual(span.name, span_name) self.assertEqual(span.span_id, span_id) - self.assertEqual(span.kind, kind) self.assertEqual(span.parent_span, parent_span) self.assertEqual(span.attributes, attributes) self.assertEqual(span.start_time, start_time) self.assertEqual(span.end_time, end_time) + self.assertEqual(span.time_events, time_events) + self.assertEqual(span.stack_trace, stack_trace) + self.assertEqual(span.links, links) + self.assertEqual(span.status, status) self.assertEqual(span.children, []) self.assertEqual(span.context_tracer, context_tracer) def test_span(self): - from opencensus.trace.enums import Enum - span_id = 'test_span_id' root_span_name = 'root_span' child_span_name = 'child_span' root_span = self._make_one(root_span_name) root_span._child_spans = [] - kind = Enum.SpanKind.SPAN_KIND_UNSPECIFIED patch = mock.patch( 'opencensus.trace.span.generate_span_id', @@ -114,7 +119,6 @@ def test_span(self): self.assertEqual(result_child_span.name, child_span_name) self.assertEqual(result_child_span.span_id, span_id) - self.assertEqual(result_child_span.kind, kind) self.assertEqual(result_child_span.parent_span, root_span) self.assertEqual(result_child_span.attributes, {}) self.assertIsNone(result_child_span.start_time) @@ -130,6 +134,37 @@ def test_add_attribute(self): self.assertEqual(span.attributes[attribute_key], attribute_value) span.attributes.pop(attribute_key, None) + def test_add_time_event(self): + from opencensus.trace.time_event import TimeEvent + import datetime + + span_name = 'test_span_name' + span = self._make_one(span_name) + time_event = mock.Mock() + + with self.assertRaises(TypeError): + span.add_time_event(time_event) + + time_event = TimeEvent(datetime.datetime.now()) + span.add_time_event(time_event) + + self.assertEqual(len(span.time_events), 1) + + def test_add_link(self): + from opencensus.trace.link import Link + + span_name = 'test_span_name' + span = self._make_one(span_name) + link = mock.Mock() + + with self.assertRaises(TypeError): + span.add_link(link) + + link = Link(span_id='1234', trace_id='4567') + span.add_link(link) + + self.assertEqual(len(span.links), 1) + def test_start(self): span_name = 'root_span' span = self._make_one(span_name) @@ -189,45 +224,67 @@ class Test_format_span_json(unittest.TestCase): def test_format_span_json_no_parent_span(self): from opencensus.trace.span import format_span_json - from opencensus.trace.enums import Enum name = 'test span' - kind = Enum.SpanKind.SPAN_KIND_UNSPECIFIED span_id = 1234 start_time = '2017-06-25' end_time = '2017-06-26' span = mock.Mock() span.name = name - span.kind = kind span.span_id = span_id span.start_time = start_time span.end_time = end_time span.parent_span = None span.attributes = None + span.stack_trace = None + span.status = None + span._child_spans = [] + span.time_events = [] + span.links = [] + span.same_process_as_parent_span = None expected_span_json = { - 'name': name, - 'kind': kind, 'spanId': span_id, 'startTime': start_time, 'endTime': end_time, + 'displayName': { + 'truncated_byte_count': 0, + 'value': 'test span'}, + 'childSpanCount': 0, } span_json = format_span_json(span) self.assertEqual(span_json, expected_span_json) - def test_format_span_json_with_parent_span(self): + @mock.patch.object(StackTrace, 'format_stack_trace_json') + @mock.patch.object(Status, 'format_status_json') + @mock.patch.object(TimeEvent, 'format_time_event_json') + def test_format_span_json_with_parent_span( + self, time_event_mock, status_mock, stack_trace_mock): + import datetime + + from opencensus.trace.link import Link from opencensus.trace.span import format_span_json - from opencensus.trace.enums import Enum name = 'test span' - kind = Enum.SpanKind.SPAN_KIND_UNSPECIFIED span_id = 1234 + trace_id = '3456' attributes = { '/http/status_code': '200', '/component': 'HTTP load balancer', } + + links = { + 'link': [ + { + 'trace_id': trace_id, + 'span_id': span_id, + 'type': 0 + }, + ], + } + start_time = '2017-06-25' end_time = '2017-06-26' parent_span = mock.Mock() @@ -237,21 +294,63 @@ def test_format_span_json_with_parent_span(self): span = mock.Mock() span.parent_span = parent_span span.name = name - span.kind = kind span.attributes = attributes span.span_id = span_id span.start_time = start_time span.end_time = end_time + span._child_spans = [] + span.time_events = [TimeEvent(datetime.datetime.now())] + span.stack_trace = StackTrace() + span.status = Status(code='200', message='test') + span.links = [Link(trace_id, span_id)] + span.same_process_as_parent_span = True + + mock_stack_trace = 'stack trace' + mock_status = 'status' + mock_time_event = 'time event' + + stack_trace_mock.return_value = mock_stack_trace + status_mock.return_value = mock_status + time_event_mock.return_value = mock_time_event expected_span_json = { - 'name': name, - 'kind': kind, 'spanId': span_id, 'parentSpanId': parent_span_id, 'startTime': start_time, 'endTime': end_time, - 'attributes': attributes, + 'attributes': { + 'attributeMap': { + '/component': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'HTTP load balancer' + } + }, + '/http/status_code': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '200' + } + } + } + }, + 'links': links, + 'stackTrace': mock_stack_trace, + 'status': mock_status, + 'timeEvents': { + 'timeEvent': + [mock_time_event] + }, + 'displayName': { + 'truncated_byte_count': 0, + 'value': 'test span'}, + 'childSpanCount': 0, + 'sameProcessAsParentSpan': True } span_json = format_span_json(span) + + print(span_json) + + print(expected_span_json) self.assertEqual(span_json, expected_span_json) From 89a144d237724412711bb3100b62a91aca19b61b Mon Sep 17 00:00:00 2001 From: Angela Li Date: Tue, 5 Dec 2017 18:35:56 -0800 Subject: [PATCH 10/15] Ignore errors when truncating non-ascii character --- opencensus/trace/attributes.py | 2 +- .../trace/exporters/stackdriver_exporter.py | 2 +- opencensus/trace/utils.py | 21 +++++++------------ tests/unit/trace/test_utils.py | 18 ++++++++++++++++ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/opencensus/trace/attributes.py b/opencensus/trace/attributes.py index 01e3390c9..ace91e7c6 100644 --- a/opencensus/trace/attributes.py +++ b/opencensus/trace/attributes.py @@ -55,7 +55,7 @@ def get_attribute(self, key): def format_attributes_json(self): """Convert the Attributes object to json format.""" attributes_json = { - utils.check_str_length(key): _format_attribute_value(value) + utils.check_str_length(key)[0]: _format_attribute_value(value) for key, value in self.attributes.items() } diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index d4ff64bee..8bdf9b704 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -113,7 +113,7 @@ def export(self, trace): def translate_to_stackdriver(self, spans): """Translate the spans json to Stackdriver format. - + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ projects.traces/batchWrite diff --git a/opencensus/trace/utils.py b/opencensus/trace/utils.py index 063a537fb..fbed08f60 100644 --- a/opencensus/trace/utils.py +++ b/opencensus/trace/utils.py @@ -22,16 +22,7 @@ def _get_truncatable_str(str_to_convert): """Truncate a string if exceed limit and record the truncated bytes count. """ - str_bytes = str_to_convert.encode(UTF8) - str_len = len(str_bytes) - - truncated_byte_count = 0 - - if str_len > MAX_LENGTH: - truncated_byte_count = str_len - MAX_LENGTH - str_bytes = str_bytes[:MAX_LENGTH] - - truncated = str_bytes.decode(UTF8, errors='ignore') + truncated, truncated_byte_count = check_str_length(str_to_convert) result = { 'value': truncated, @@ -49,19 +40,21 @@ def check_str_length(str_to_check, limit=None): :type limit: int :param limit: The upper limit of the length. - :rtype: str + :rtype: tuple :returns: The string it self if not exceeded length, or truncated string - if exceeded. + if exceeded and the truncated byte count. """ if limit is None: limit = MAX_LENGTH str_bytes = str_to_check.encode(UTF8) str_len = len(str_bytes) + truncated_byte_count = 0 if str_len > limit: + truncated_byte_count = str_len - limit str_bytes = str_bytes[:limit] - result = str_bytes.decode(UTF8, errors='ignore') + result = str(str_bytes.decode(UTF8, errors='ignore')) - return result + return (result, truncated_byte_count) diff --git a/tests/unit/trace/test_utils.py b/tests/unit/trace/test_utils.py index 52c6691e3..4fd31b33e 100644 --- a/tests/unit/trace/test_utils.py +++ b/tests/unit/trace/test_utils.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + # Copyright 2017, OpenCensus Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,3 +48,18 @@ def test__get_truncatable_str_length_exceeds(self): } self.assertEqual(expected_str, truncatable_str) + + def test_check_str_length(self): + limit = 5 + + str_to_check = u'test测试' + + (result, truncated_byte_count) = utils.check_str_length( + str_to_check, limit) + + expected_result = 'test' + + # Should only have 4 bytes remained, dropped off the invalid part if + # truncated in the middle of a character. + self.assertEqual(expected_result, result) + self.assertEqual(truncated_byte_count, 5) From 98d22fb9f8bdc308c136eb9f311d3f48f9c5ac37 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 6 Dec 2017 11:46:11 -0800 Subject: [PATCH 11/15] Fix system tests --- opencensus/trace/attributes.py | 2 +- .../trace/exporters/stackdriver_exporter.py | 7 +- .../basic_trace/basic_trace_system_test.py | 70 +++++++++---------- .../exporters/test_stackdriver_exporter.py | 41 +++++++++-- tests/unit/trace/test_attributes.py | 10 ++- 5 files changed, 86 insertions(+), 44 deletions(-) diff --git a/opencensus/trace/attributes.py b/opencensus/trace/attributes.py index ace91e7c6..ea2e4176d 100644 --- a/opencensus/trace/attributes.py +++ b/opencensus/trace/attributes.py @@ -24,7 +24,7 @@ def _format_attribute_value(value): value_type = 'string_value' value = utils._get_truncatable_str(value) else: - raise TypeError("Value must be str, int, or bool.") + return None return {value_type: value} diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index 8bdf9b704..85c91a646 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -131,13 +131,18 @@ def translate_to_stackdriver(self, spans): for span in spans_json: span_name = 'projects/{}/traces/{}/spans/{}'.format( self.project_id, trace_id, span.get('spanId')) + parent_span_id = None + + if span.get('parentSpanId') is not None: + parent_span_id = str(span.get('parentSpanId')) + span_json = { 'name': span_name, 'displayName': span.get('displayName'), 'startTime': span.get('startTime'), 'endTime': span.get('endTime'), 'spanId': str(span.get('spanId')), - 'parentSpanId': str(span.get('parentSpanId')), + 'parentSpanId': parent_span_id, 'attributes': span.get('attributes'), 'links': span.get('links'), 'status': span.get('status'), diff --git a/tests/system/trace/basic_trace/basic_trace_system_test.py b/tests/system/trace/basic_trace/basic_trace_system_test.py index 4b3b149e9..cdbbf7c75 100644 --- a/tests/system/trace/basic_trace/basic_trace_system_test.py +++ b/tests/system/trace/basic_trace/basic_trace_system_test.py @@ -24,49 +24,49 @@ def func_to_trace(): class TestBasicTrace(unittest.TestCase): def test_request_tracer(self): - import json +import json - from opencensus.trace import request_tracer - from opencensus.trace.samplers import always_on - from opencensus.trace.exporters import file_exporter - from opencensus.trace.propagation import google_cloud_format +from opencensus.trace import request_tracer +from opencensus.trace.samplers import always_on +from opencensus.trace.exporters import file_exporter +from opencensus.trace.propagation import google_cloud_format - trace_id = 'f8739df974a4481f98748cd92b27177d' - span_id = '16971691944144156899' - trace_option = 1 +trace_id = 'f8739df974a4481f98748cd92b27177d' +span_id = '16971691944144156899' +trace_option = 1 - trace_header = '{}/{};o={}'.format(trace_id, span_id, trace_option) +trace_header = '{}/{};o={}'.format(trace_id, span_id, trace_option) - sampler = always_on.AlwaysOnSampler() - exporter = file_exporter.FileExporter() - propagator = google_cloud_format.GoogleCloudFormatPropagator() - span_context = propagator.from_header(header=trace_header) +sampler = always_on.AlwaysOnSampler() +exporter = file_exporter.FileExporter() +propagator = google_cloud_format.GoogleCloudFormatPropagator() +span_context = propagator.from_header(header=trace_header) - tracer = request_tracer.RequestTracer( - span_context=span_context, - sampler=sampler, - exporter=exporter, - propagator=propagator - ) +tracer = request_tracer.RequestTracer( + span_context=span_context, + sampler=sampler, + exporter=exporter, + propagator=propagator +) - with tracer.span(name='root_span') as root: - func_to_trace() - parent_span_id = root.span_id - with root.span(name='child_span'): - func_to_trace() +with tracer.span(name='root_span') as root: + func_to_trace() + parent_span_id = root.span_id + with root.span(name='child_span'): + func_to_trace() - tracer.finish() +tracer.finish() - file = open(file_exporter.DEFAULT_FILENAME, 'r') - trace_json = json.loads(file.read()) +file = open(file_exporter.DEFAULT_FILENAME, 'r') +trace_json = json.loads(file.read()) - spans = trace_json.get('spans') +spans = trace_json.get('spans') - self.assertEqual(trace_json.get('traceId'), trace_id) - self.assertEqual(len(spans), 2) +self.assertEqual(trace_json.get('traceId'), trace_id) +self.assertEqual(len(spans), 2) - for span in spans: - if span.get('name') == 'root_span': - self.assertEqual(str(span.get('parentSpanId')), span_id) - else: - self.assertEqual(span.get('parentSpanId'), parent_span_id) +for span in spans: + if span.get('displayName').get('value') == 'root_span': + self.assertEqual(str(span.get('parentSpanId')), span_id) + else: + self.assertEqual(span.get('parentSpanId'), parent_span_id) diff --git a/tests/unit/trace/exporters/test_stackdriver_exporter.py b/tests/unit/trace/exporters/test_stackdriver_exporter.py index b6b550be2..65578af8a 100644 --- a/tests/unit/trace/exporters/test_stackdriver_exporter.py +++ b/tests/unit/trace/exporters/test_stackdriver_exporter.py @@ -67,7 +67,41 @@ def test_export(self): self.assertTrue(exporter.transport.export_called) def test_emit(self): - spans = {'spans': []} + spans = {'spans': + [ + { + 'displayName': { + 'value': 'span', + 'truncated_byte_count': 0 + }, + 'spanId': '1111', + } + ] + } + + stackdriver_spans = { + 'spans': [ + { + 'status': None, + 'childSpanCount': None, + 'links': None, + 'startTime': None, + 'spanId': '1111', + 'stackTrace': None, + 'displayName': + { + 'truncated_byte_count': 0, + 'value': 'span' + }, + 'name': 'projects/PROJECT/traces/None/spans/1111', + 'parentSpanId': None, + 'attributes': None, + 'timeEvents': None, + 'endTime': None, + 'sameProcessAsParentSpan': None + } + ] + } client = mock.Mock() project_id = 'PROJECT' @@ -81,7 +115,7 @@ def test_emit(self): name = 'projects/{}'.format(project_id) - client.batch_write_spans.assert_called_with(name, spans) + client.batch_write_spans.assert_called_with(name, stackdriver_spans) self.assertTrue(client.batch_write_spans.called) def test_translate_to_stackdriver(self): @@ -145,9 +179,6 @@ def test_translate_to_stackdriver(self): ] } - print(spans) - print(expected_traces) - self.assertEqual(spans, expected_traces) diff --git a/tests/unit/trace/test_attributes.py b/tests/unit/trace/test_attributes.py index cb39f25cb..dbcbba5ff 100644 --- a/tests/unit/trace/test_attributes.py +++ b/tests/unit/trace/test_attributes.py @@ -100,7 +100,13 @@ def test_format_attributes_json_type_error(self): 'key1': mock.Mock(), } + expected_json = { + 'attributeMap': { + 'key1': None + } + } + attributes = attributes_module.Attributes(attrs) + attributes_json = attributes.format_attributes_json() - with self.assertRaises(TypeError): - attributes_json = attributes.format_attributes_json() + self.assertEqual(attributes_json, expected_json) From 0d9e60dd7a8b6c202f0062eef2a5a3b7f477249a Mon Sep 17 00:00:00 2001 From: Angela Li Date: Wed, 6 Dec 2017 13:43:31 -0800 Subject: [PATCH 12/15] fix --- .../basic_trace/basic_trace_system_test.py | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/system/trace/basic_trace/basic_trace_system_test.py b/tests/system/trace/basic_trace/basic_trace_system_test.py index cdbbf7c75..38f916566 100644 --- a/tests/system/trace/basic_trace/basic_trace_system_test.py +++ b/tests/system/trace/basic_trace/basic_trace_system_test.py @@ -24,49 +24,49 @@ def func_to_trace(): class TestBasicTrace(unittest.TestCase): def test_request_tracer(self): -import json + import json -from opencensus.trace import request_tracer -from opencensus.trace.samplers import always_on -from opencensus.trace.exporters import file_exporter -from opencensus.trace.propagation import google_cloud_format + from opencensus.trace import request_tracer + from opencensus.trace.samplers import always_on + from opencensus.trace.exporters import file_exporter + from opencensus.trace.propagation import google_cloud_format -trace_id = 'f8739df974a4481f98748cd92b27177d' -span_id = '16971691944144156899' -trace_option = 1 + trace_id = 'f8739df974a4481f98748cd92b27177d' + span_id = '16971691944144156899' + trace_option = 1 -trace_header = '{}/{};o={}'.format(trace_id, span_id, trace_option) + trace_header = '{}/{};o={}'.format(trace_id, span_id, trace_option) -sampler = always_on.AlwaysOnSampler() -exporter = file_exporter.FileExporter() -propagator = google_cloud_format.GoogleCloudFormatPropagator() -span_context = propagator.from_header(header=trace_header) + sampler = always_on.AlwaysOnSampler() + exporter = file_exporter.FileExporter() + propagator = google_cloud_format.GoogleCloudFormatPropagator() + span_context = propagator.from_header(header=trace_header) -tracer = request_tracer.RequestTracer( - span_context=span_context, - sampler=sampler, - exporter=exporter, - propagator=propagator -) + tracer = request_tracer.RequestTracer( + span_context=span_context, + sampler=sampler, + exporter=exporter, + propagator=propagator + ) -with tracer.span(name='root_span') as root: - func_to_trace() - parent_span_id = root.span_id - with root.span(name='child_span'): - func_to_trace() + with tracer.span(name='root_span') as root: + func_to_trace() + parent_span_id = root.span_id + with root.span(name='child_span'): + func_to_trace() -tracer.finish() + tracer.finish() -file = open(file_exporter.DEFAULT_FILENAME, 'r') -trace_json = json.loads(file.read()) + file = open(file_exporter.DEFAULT_FILENAME, 'r') + trace_json = json.loads(file.read()) -spans = trace_json.get('spans') + spans = trace_json.get('spans') -self.assertEqual(trace_json.get('traceId'), trace_id) -self.assertEqual(len(spans), 2) + self.assertEqual(trace_json.get('traceId'), trace_id) + self.assertEqual(len(spans), 2) -for span in spans: - if span.get('displayName').get('value') == 'root_span': - self.assertEqual(str(span.get('parentSpanId')), span_id) - else: - self.assertEqual(span.get('parentSpanId'), parent_span_id) + for span in spans: + if span.get('displayName').get('value') == 'root_span': + self.assertEqual(str(span.get('parentSpanId')), span_id) + else: + self.assertEqual(span.get('parentSpanId'), parent_span_id) From d55bc0986b7efc968ee7a008343ecbf5c90178f9 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 7 Dec 2017 11:49:36 -0800 Subject: [PATCH 13/15] [WIP] Update to use V2 API --- requirements-test.txt | 2 +- setup.py | 2 +- tests/system/trace/django/django_system_test.py | 5 +++-- tests/system/trace/flask/flask_system_test.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index d238891de..8de2c90d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ Django==1.11.7 Flask==0.12.2 -google-cloud-trace==0.16.0 +google-cloud-trace==0.17.0 mock==2.0.0 mysql-connector==2.1.6 psycopg2==2.7.3.1 diff --git a/setup.py b/setup.py index f563425d7..b7592f9ac 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ from setuptools import setup, find_packages install_requires = [ - 'google-cloud-trace>=0.16.0, <0.17dev', + 'google-cloud-trace>=0.17.0, <0.18dev', ] setup( diff --git a/tests/system/trace/django/django_system_test.py b/tests/system/trace/django/django_system_test.py index a629540d3..37b43fee7 100644 --- a/tests/system/trace/django/django_system_test.py +++ b/tests/system/trace/django/django_system_test.py @@ -39,6 +39,7 @@ def wait_app_to_start(): cmd = 'wget --retry-connrefused --tries=5 {}'.format(BASE_URL) subprocess.check_call(cmd, shell=True) + def generate_header(): """Generate a trace header.""" trace_id = uuid.uuid4().hex @@ -64,7 +65,7 @@ def run_application(): class TestDjangoTrace(unittest.TestCase): def setUp(self): - from google.cloud import trace + from google.cloud.trace.v1 import client as client_module # Generate trace headers trace_id, span_id, trace_header = generate_header() @@ -86,7 +87,7 @@ def setUp(self): wait_app_to_start() # Initialize the stackdriver trace client - self.client = trace.Client(project=PROJECT) + self.client = client_module.Client(project=PROJECT) def tearDown(self): # Kill the django application process diff --git a/tests/system/trace/flask/flask_system_test.py b/tests/system/trace/flask/flask_system_test.py index 2d14217d2..39d72de01 100644 --- a/tests/system/trace/flask/flask_system_test.py +++ b/tests/system/trace/flask/flask_system_test.py @@ -64,7 +64,7 @@ def run_application(): class TestFlaskTrace(unittest.TestCase): def setUp(self): - from google.cloud import trace + from google.cloud.trace.v1 import client as client_module # Generate trace headers trace_id, span_id, trace_header = generate_header() @@ -86,7 +86,7 @@ def setUp(self): wait_app_to_start() # Initialize the stackdriver trace client - self.client = trace.Client(project=PROJECT) + self.client = client_module.Client(project=PROJECT) def tearDown(self): # Kill the flask application process From c0dc5f83f4316e6065c9c81cdef794b8b0324acf Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 7 Dec 2017 13:56:20 -0800 Subject: [PATCH 14/15] Update system tests --- opencensus/trace/attributes.py | 9 ++++++ .../trace/exporters/stackdriver_exporter.py | 10 +++---- .../system/trace/django/django_system_test.py | 23 +-------------- tests/system/trace/flask/flask_system_test.py | 29 ++----------------- .../exporters/test_stackdriver_exporter.py | 13 +++++---- tests/unit/trace/test_attributes.py | 4 +-- tests/unit/trace/test_span.py | 1 + 7 files changed, 28 insertions(+), 61 deletions(-) diff --git a/opencensus/trace/attributes.py b/opencensus/trace/attributes.py index ea2e4176d..95da7c6aa 100644 --- a/opencensus/trace/attributes.py +++ b/opencensus/trace/attributes.py @@ -59,6 +59,15 @@ def format_attributes_json(self): for key, value in self.attributes.items() } + attributes_json = {} + + for key, value in self.attributes.items(): + key = utils.check_str_length(key)[0] + value = _format_attribute_value(value) + + if value is not None: + attributes_json[key] = value + result = { 'attributeMap': attributes_json } diff --git a/opencensus/trace/exporters/stackdriver_exporter.py b/opencensus/trace/exporters/stackdriver_exporter.py index 85c91a646..8713104c3 100644 --- a/opencensus/trace/exporters/stackdriver_exporter.py +++ b/opencensus/trace/exporters/stackdriver_exporter.py @@ -131,10 +131,6 @@ def translate_to_stackdriver(self, spans): for span in spans_json: span_name = 'projects/{}/traces/{}/spans/{}'.format( self.project_id, trace_id, span.get('spanId')) - parent_span_id = None - - if span.get('parentSpanId') is not None: - parent_span_id = str(span.get('parentSpanId')) span_json = { 'name': span_name, @@ -142,7 +138,6 @@ def translate_to_stackdriver(self, spans): 'startTime': span.get('startTime'), 'endTime': span.get('endTime'), 'spanId': str(span.get('spanId')), - 'parentSpanId': parent_span_id, 'attributes': span.get('attributes'), 'links': span.get('links'), 'status': span.get('status'), @@ -151,6 +146,11 @@ def translate_to_stackdriver(self, spans): 'sameProcessAsParentSpan': span.get('sameProcessAsParentSpan'), 'childSpanCount': span.get('childSpanCount') } + + if span.get('parentSpanId') is not None: + parent_span_id = str(span.get('parentSpanId')) + span_json['parentSpanId'] = parent_span_id + spans_list.append(span_json) spans = {'spans': spans_list} diff --git a/tests/system/trace/django/django_system_test.py b/tests/system/trace/django/django_system_test.py index 37b43fee7..9871037df 100644 --- a/tests/system/trace/django/django_system_test.py +++ b/tests/system/trace/django/django_system_test.py @@ -43,7 +43,7 @@ def wait_app_to_start(): def generate_header(): """Generate a trace header.""" trace_id = uuid.uuid4().hex - span_id = random.getrandbits(64) + span_id = random.randint(10**15, 10**16 - 1) trace_option = 1 header = '{}/{};o={}'.format(trace_id, span_id, trace_option) @@ -106,9 +106,6 @@ def test_with_retry(self): self.assertEqual(trace.get('projectId'), PROJECT) self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertEqual(len(spans), 1) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) for span in spans: labels = span.get('labels') @@ -131,9 +128,6 @@ def test_with_retry(self): # Should have 2 spans, one for django request, one for mysql query self.assertEqual(len(spans), 2) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) request_succeeded = False @@ -166,9 +160,6 @@ def test_with_retry(self): # Should have 2 spans, one for django request, one for postgresql query self.assertEqual(len(trace.get('spans')), 2) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) request_succeeded = False @@ -200,7 +191,6 @@ def test_with_retry(self): self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertNotEqual(len(trace.get('spans')), 0) - has_parent_span = False request_succeeded = False for span in spans: @@ -209,11 +199,6 @@ def test_with_retry(self): self.assertEqual(labels.get('/http/status_code'), '200') request_succeeded = True - if span.get('name') == 'app.views.sqlalchemy_mysql_trace': - self.assertEqual(span.get('parentSpanId'), str(self.span_id)) - has_parent_span = True - - self.assertTrue(has_parent_span) self.assertTrue(request_succeeded) test_with_retry(self) @@ -232,7 +217,6 @@ def test_with_retry(self): self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertNotEqual(len(trace.get('spans')), 0) - has_parent_span = False request_succeeded = False for span in spans: @@ -241,11 +225,6 @@ def test_with_retry(self): self.assertEqual(labels.get('/http/status_code'), '200') request_succeeded = True - if span.get('name') == 'app.views.sqlalchemy_postgresql_trace': - self.assertEqual(span.get('parentSpanId'), str(self.span_id)) - has_parent_span = True - - self.assertTrue(has_parent_span) self.assertTrue(request_succeeded) test_with_retry(self) diff --git a/tests/system/trace/flask/flask_system_test.py b/tests/system/trace/flask/flask_system_test.py index 39d72de01..3ca0bc4e6 100644 --- a/tests/system/trace/flask/flask_system_test.py +++ b/tests/system/trace/flask/flask_system_test.py @@ -42,7 +42,7 @@ def wait_app_to_start(): def generate_header(): """Generate a trace header.""" trace_id = uuid.uuid4().hex - span_id = random.getrandbits(64) + span_id = random.randint(10**15, 10**16 - 1) trace_option = 1 header = '{}/{};o={}'.format(trace_id, span_id, trace_option) @@ -105,9 +105,6 @@ def test_with_retry(self): self.assertEqual(trace.get('projectId'), PROJECT) self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertEqual(len(spans), 1) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) for span in spans: labels = span.get('labels') @@ -130,9 +127,6 @@ def test_with_retry(self): # Should have 2 spans, one for flask request, one for mysql query self.assertEqual(len(spans), 2) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) request_succeeded = False @@ -165,9 +159,6 @@ def test_with_retry(self): # Should have 2 spans, one for flask request, one for postgresql query self.assertEqual(len(spans), 2) - self.assertEqual( - spans[0].get('parentSpanId'), - str(self.span_id)) request_succeeded = False @@ -199,21 +190,14 @@ def test_with_retry(self): self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertNotEqual(len(spans), 0) - has_parent_span = False request_succeeded = False for span in spans: - if span.get('name') == \ - '[GET]http://localhost:8080/sqlalchemy-mysql': - self.assertEqual(span.get('parentSpanId'), str(self.span_id)) - has_parent_span = True - request_succeeded = True - labels = span.get('labels') if '/http/status_code' in labels.keys(): self.assertEqual(labels.get('/http/status_code'), '200') + request_succeeded = True - self.assertTrue(has_parent_span) self.assertTrue(request_succeeded) test_with_retry(self) @@ -232,21 +216,14 @@ def test_with_retry(self): self.assertEqual(trace.get('traceId'), str(self.trace_id)) self.assertNotEqual(len(trace.get('spans')), 0) - has_parent_span = False request_succeeded = False for span in spans: - if span.get('name') == \ - '[GET]http://localhost:8080/sqlalchemy-postgresql': - self.assertEqual(span.get('parentSpanId'), str(self.span_id)) - has_parent_span = True - request_succeeded = True - labels = span.get('labels') if '/http/status_code' in labels.keys(): self.assertEqual(labels.get('/http/status_code'), '200') + request_succeeded = True - self.assertTrue(has_parent_span) self.assertTrue(request_succeeded) test_with_retry(self) diff --git a/tests/unit/trace/exporters/test_stackdriver_exporter.py b/tests/unit/trace/exporters/test_stackdriver_exporter.py index 65578af8a..343ecba44 100644 --- a/tests/unit/trace/exporters/test_stackdriver_exporter.py +++ b/tests/unit/trace/exporters/test_stackdriver_exporter.py @@ -87,6 +87,7 @@ def test_emit(self): 'links': None, 'startTime': None, 'spanId': '1111', + 'attributes': None, 'stackTrace': None, 'displayName': { @@ -94,8 +95,6 @@ def test_emit(self): 'value': 'span' }, 'name': 'projects/PROJECT/traces/None/spans/1111', - 'parentSpanId': None, - 'attributes': None, 'timeEvents': None, 'endTime': None, 'sameProcessAsParentSpan': None @@ -123,7 +122,9 @@ def test_translate_to_stackdriver(self): trace_id = '6e0c63257de34c92bf9efcd03927272e' span_name = 'test span' span_id = 1234 - attributes = {} + attributes = {'attributeMap': { + 'key': 'value'} + } parent_span_id = 1111 start_time = 'test start time' end_time = 'test end time' @@ -152,7 +153,6 @@ def test_translate_to_stackdriver(self): client=client, project_id=project_id) - spans = exporter.translate_to_stackdriver(trace) expected_traces = { @@ -164,11 +164,11 @@ def test_translate_to_stackdriver(self): 'value': span_name, 'truncated_byte_count': 0 }, + 'attributes': {'attributeMap': {'key': 'value'}}, 'spanId': str(span_id), 'startTime': start_time, 'endTime': end_time, 'parentSpanId': str(parent_span_id), - 'attributes': {}, 'status': None, 'links': None, 'stackTrace': None, @@ -179,6 +179,9 @@ def test_translate_to_stackdriver(self): ] } + print(spans) + print(expected_traces) + self.assertEqual(spans, expected_traces) diff --git a/tests/unit/trace/test_attributes.py b/tests/unit/trace/test_attributes.py index dbcbba5ff..99e54f971 100644 --- a/tests/unit/trace/test_attributes.py +++ b/tests/unit/trace/test_attributes.py @@ -101,9 +101,7 @@ def test_format_attributes_json_type_error(self): } expected_json = { - 'attributeMap': { - 'key1': None - } + 'attributeMap': {} } attributes = attributes_module.Attributes(attrs) diff --git a/tests/unit/trace/test_span.py b/tests/unit/trace/test_span.py index 97943910a..a4b30516a 100644 --- a/tests/unit/trace/test_span.py +++ b/tests/unit/trace/test_span.py @@ -273,6 +273,7 @@ def test_format_span_json_with_parent_span( attributes = { '/http/status_code': '200', '/component': 'HTTP load balancer', + 'none_key': None } links = { From 6ddb0b1c4bac1dbae8d55c4ee08139b2819cf36c Mon Sep 17 00:00:00 2001 From: Angela Li Date: Thu, 7 Dec 2017 23:50:54 -0800 Subject: [PATCH 15/15] Address comments --- opencensus/trace/span_context.py | 3 ++- opencensus/trace/utils.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/opencensus/trace/span_context.py b/opencensus/trace/span_context.py index 3a4e0c797..97c02d961 100644 --- a/opencensus/trace/span_context.py +++ b/opencensus/trace/span_context.py @@ -150,7 +150,8 @@ def check_trace_id(self, trace_id): def generate_span_id(): - """Return the random generated span ID for a span. Must be 16 digits. + """Return the random generated span ID for a span. Must be 16 digits + as Stackdriver Trace V2 API only accepts 16 digits span ID. :rtype: int :returns: Identifier for the span. Must be a 64-bit integer other diff --git a/opencensus/trace/utils.py b/opencensus/trace/utils.py index fbed08f60..22e7ab35b 100644 --- a/opencensus/trace/utils.py +++ b/opencensus/trace/utils.py @@ -22,7 +22,8 @@ def _get_truncatable_str(str_to_convert): """Truncate a string if exceed limit and record the truncated bytes count. """ - truncated, truncated_byte_count = check_str_length(str_to_convert) + truncated, truncated_byte_count = check_str_length( + str_to_convert, MAX_LENGTH) result = { 'value': truncated, @@ -31,7 +32,7 @@ def _get_truncatable_str(str_to_convert): return result -def check_str_length(str_to_check, limit=None): +def check_str_length(str_to_check, limit=MAX_LENGTH): """Check the length of a string. If exceeds limit, then truncate it. :type str_to_check: str @@ -44,9 +45,6 @@ def check_str_length(str_to_check, limit=None): :returns: The string it self if not exceeded length, or truncated string if exceeded and the truncated byte count. """ - if limit is None: - limit = MAX_LENGTH - str_bytes = str_to_check.encode(UTF8) str_len = len(str_bytes) truncated_byte_count = 0