From 188358867546c87a2fbb1566a33eb5e6af2091fc Mon Sep 17 00:00:00 2001 From: Jack Burnett Date: Wed, 16 Nov 2022 13:03:25 +0000 Subject: [PATCH 1/4] Add a to dict method to RecordMessage to improve performance --- singer_sdk/_singerlib/messages.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py index 36c76f6bc..dc0ab9b28 100644 --- a/singer_sdk/_singerlib/messages.py +++ b/singer_sdk/_singerlib/messages.py @@ -47,6 +47,7 @@ def to_dict(self) -> dict[str, t.Any]: Returns: A dictionary with the defined message fields. """ + return asdict(self, dict_factory=exclude_null_dict) @classmethod @@ -79,6 +80,18 @@ class RecordMessage(Message): time_extracted: datetime | None = None """The time the record was extracted.""" + def to_dict(self) -> dict[str, t.Any]: + result = { + 'type': 'RECORD', + 'stream': self.stream, + 'record': self.record + } + if self.version is not None: + result["version"] = self.version + if self.time_extracted is not None: + result["time_extracted"] = self.time_extracted + return result + def __post_init__(self) -> None: """Post-init processing. From c0c7a4f033d3f9f78c76518f358b1c4eccfb4249 Mon Sep 17 00:00:00 2001 From: Jack Burnett Date: Wed, 16 Nov 2022 13:07:43 +0000 Subject: [PATCH 2/4] Fix typing --- singer_sdk/_singerlib/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py index dc0ab9b28..73be9e556 100644 --- a/singer_sdk/_singerlib/messages.py +++ b/singer_sdk/_singerlib/messages.py @@ -81,7 +81,7 @@ class RecordMessage(Message): """The time the record was extracted.""" def to_dict(self) -> dict[str, t.Any]: - result = { + result: dict[str, t.Any] = { 'type': 'RECORD', 'stream': self.stream, 'record': self.record From 516065d8e47e86d5e2520e47b2d54a91437c77a2 Mon Sep 17 00:00:00 2001 From: Jack Burnett Date: Wed, 16 Nov 2022 13:11:39 +0000 Subject: [PATCH 3/4] lint --- singer_sdk/_singerlib/messages.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py index 73be9e556..1aedd8e56 100644 --- a/singer_sdk/_singerlib/messages.py +++ b/singer_sdk/_singerlib/messages.py @@ -47,7 +47,6 @@ def to_dict(self) -> dict[str, t.Any]: Returns: A dictionary with the defined message fields. """ - return asdict(self, dict_factory=exclude_null_dict) @classmethod @@ -81,6 +80,10 @@ class RecordMessage(Message): """The time the record was extracted.""" def to_dict(self) -> dict[str, t.Any]: + """ Converts to a dictionary. + + This overrides the default conversion logic, since it uses unnecessary deep copying and is very slow + """ result: dict[str, t.Any] = { 'type': 'RECORD', 'stream': self.stream, From 316e83ef5b5976d0075ae591fa735144eaef34ef Mon Sep 17 00:00:00 2001 From: Jack Burnett Date: Wed, 16 Nov 2022 13:16:28 +0000 Subject: [PATCH 4/4] format --- singer_sdk/_singerlib/messages.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py index 1aedd8e56..2b255b412 100644 --- a/singer_sdk/_singerlib/messages.py +++ b/singer_sdk/_singerlib/messages.py @@ -80,14 +80,18 @@ class RecordMessage(Message): """The time the record was extracted.""" def to_dict(self) -> dict[str, t.Any]: - """ Converts to a dictionary. + """Return a dictionary representation of the message. + + This overrides the default conversion logic, since it uses unnecessary + deep copying and is very slow. - This overrides the default conversion logic, since it uses unnecessary deep copying and is very slow + Returns: + A dictionary with the defined message fields. """ result: dict[str, t.Any] = { - 'type': 'RECORD', - 'stream': self.stream, - 'record': self.record + "type": "RECORD", + "stream": self.stream, + "record": self.record, } if self.version is not None: result["version"] = self.version