Adding time_extracted and bookmark_properties #55
Conversation
singer/catalog.py
Outdated
| @@ -11,7 +11,7 @@ class CatalogEntry(object): | |||
| def __init__(self, tap_stream_id=None, stream=None, | |||
| key_properties=None, schema=None, replication_key=None, | |||
| is_view=None, database=None, table=None, row_count=None, | |||
| stream_alias=None, metadata=None): | |||
| stream_alias=None, metadata=None, bookmark_properties=None): | |||
There was a problem hiding this comment.
You should not need to change the catalog at all. You can revert all the changes on this file.
singer/messages.py
Outdated
| @@ -37,9 +37,10 @@ class RecordMessage(Message): | |||
|
|
|||
| ''' | |||
|
|
|||
| def __init__(self, stream, record, version=None): | |||
| def __init__(self, stream, record, time_extracted=None, version=None): | |||
There was a problem hiding this comment.
You should add time_extracted after version in case someone is calling RecordMessage(stream, record, version).
There was a problem hiding this comment.
We should require time_extracted to be an "aware" datetime.datetime object. Maybe just add a check here to validate that it's either None or a datetime.datetime with a tzinfo property.
singer/messages.py
Outdated
| @@ -48,6 +49,8 @@ def asdict(self): | |||
| 'stream': self.stream, | |||
| 'record': self.record, | |||
| } | |||
| if self.time_extracted is not None: | |||
| result['time_extracted'] = self.time_extracted | |||
There was a problem hiding this comment.
I think you'll need to change this to convert the timestamp to ISO format.
…is set to UTC for time_extracted
singer/messages.py
Outdated
| self.stream = stream | ||
| self.record = record | ||
| self.version = version | ||
| self.time_extracted = time_extracted | ||
| if time_extracted is not None: |
There was a problem hiding this comment.
You can just do if time_extracted:. You don't need the is not None. I know we have the is not None elsewhere in here but we've since learned that omitting the is not None is the preferred style.
singer/messages.py
Outdated
| self.time_extracted = time_extracted | ||
| if time_extracted is not None: | ||
| if not u.is_aware_datetime(time_extracted): | ||
| raise Exception("'time_extracted' must be an aware " |
There was a problem hiding this comment.
I'd raise ValueError here.
singer/messages.py
Outdated
| @@ -50,6 +55,8 @@ def asdict(self): | |||
| } | |||
| if self.version is not None: | |||
| result['version'] = self.version | |||
| if self.time_extracted is not None: | |||
singer/messages.py
Outdated
| 'type': 'SCHEMA', | ||
| 'stream': self.stream, | ||
| 'schema': self.schema, | ||
| 'key_properties': self.key_properties | ||
| } | ||
| if self.bookmark_properties is not None: |
Version 0.2.0 includes the 'time_extracted' and 'bookmark_properties' that correspond to these changes