Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Schema Updates and various fixes #7

Merged
merged 3 commits into from
May 9, 2018
Merged

Schema Updates and various fixes #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
58044f9
First pass at data diff
dmosorast May 9, 2018
0d8ffd3
minor cleanup; set valid-replication-keys metadata
May 9, 2018
f17388c
only sync the correct ticket rows
May 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions tap_zendesk/discover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,3 @@ def discover_streams(client):
s = s(client)
streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': s.load_schema(), 'metadata': s.load_metadata()})
return streams

# * Ticket Comments (P2)
# * Others from Audit (P2)

# P2?
# Use "search" Endpoint; for things like ticket_metrics
# * zenpy_client.search("some query", type='ticket_metrics', sort_by='created_at', sort_order='desc')

# P2?
# "Relationships" (?)
# tickets + ticket_fields = a ticket row
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/group-memberships.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"group_id": {
"type": [
Expand All @@ -34,7 +35,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"id": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/groups.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -26,7 +27,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"deleted": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/macros.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -72,7 +73,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"active": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/organizations.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"tags": {
"type": [
Expand Down Expand Up @@ -75,7 +76,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"name": {
"type": [
Expand Down
11 changes: 6 additions & 5 deletions tap_zendesk/schemas/ticket-audits.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,8 @@
"value": {
"type": [
"null",
"string",
"array"
"array",
"string"
],
"items": {
"type": [
Expand Down Expand Up @@ -373,8 +373,8 @@
"previous_value": {
"type": [
"null",
"string",
"array"
"array",
"string"
],
"items": {
"type": [
Expand Down Expand Up @@ -559,7 +559,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"ticket_id": {
"type": [
Expand Down
6 changes: 4 additions & 2 deletions tap_zendesk/schemas/ticket-fields.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"title_in_portal": {
"type": [
Expand Down Expand Up @@ -114,7 +115,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"tag": {
"type": [
Expand Down
27 changes: 18 additions & 9 deletions tap_zendesk/schemas/ticket-metrics.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"first_resolution_time_in_minutes": {
"type": [
Expand Down Expand Up @@ -138,7 +139,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"on_hold_time_in_minutes": {
"type": [
Expand Down Expand Up @@ -196,7 +198,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"requester_wait_time_in_minutes": {
"type": [
Expand All @@ -222,13 +225,15 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"updated_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"url": {
"type": [
Expand All @@ -240,25 +245,29 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"assigned_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"solved_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"assignee_updated_at": {
"type": [
"null",
"string"
]
],
"format": "date-time"
}
},
"type": [
Expand Down
24 changes: 7 additions & 17 deletions tap_zendesk/schemas/tickets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"raw_subject": {
"type": [
Expand All @@ -130,7 +131,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"custom_fields": {
"items": {
Expand All @@ -141,20 +143,7 @@
"integer"
]
},
"value": {
"items": {
"type": [
"null",
"string"
]
},
"type": [
"null",
"string",
"boolean",
"array"
]
}
"value": { }
},
"type": [
"null",
Expand Down Expand Up @@ -182,7 +171,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"followup_ids": {
"items": {
Expand Down
9 changes: 6 additions & 3 deletions tap_zendesk/schemas/users.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"external_id": {
"type": [
Expand Down Expand Up @@ -270,7 +271,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"suspended": {
"type": [
Expand Down Expand Up @@ -301,7 +303,8 @@
"type": [
"null",
"string"
]
],
"format": "date-time"
},
"alias": {
"type": [
Expand Down
14 changes: 11 additions & 3 deletions tap_zendesk/streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ def load_metadata(self):

mdata = metadata.write(mdata, (), 'table-key-properties', self.key_properties)
mdata = metadata.write(mdata, (), 'forced-replication-method', self.replication_method)
mdata = metadata.write(mdata, (), 'valid-replication-keys', [self.replication_key])

for field_name in schema['properties'].keys():
if field_name in KEY_PROPERTIES:
if field_name in KEY_PROPERTIES or field_name == self.replication_key:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
else:
mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')
Expand Down Expand Up @@ -137,8 +138,15 @@ def sync(self, state):
bookmark = self.get_bookmark(state)
tickets = self.client.tickets.incremental(start_time=bookmark)
for ticket in tickets:
if utils.strptime_with_tz(ticket.updated_at) < bookmark:
# NB: Skip tickets that might show up because of Zendesk behavior:
# The Incremental Ticket Export endpoint also returns tickets that
# were updated for reasons not related to ticket events, such as a system update or a database backfill.
continue
self.update_bookmark(state, ticket.updated_at)
yield ticket
ticket_dict = ticket.to_dict()
ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting
yield ticket_dict

class TicketAudits(Stream):
name = "ticket-audits"
Expand Down Expand Up @@ -188,7 +196,7 @@ def sync(self, state):
class Tags(Stream):
name = "tags"
replication_method = "FULL_TABLE"
key_properties = []
key_properties = ["name"]

def sync(self, state): # pylint: disable=unused-argument
# NB: Setting page to force it to paginate all tags, instead of just the
Expand Down