Skip to content

Commit

Permalink
Google Calendar source & destination fixes (#732)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tovbinm
tovbinm committed Oct 18, 2022
1 parent 3857a48 commit 7a5849a
Show file tree
Hide file tree
Showing 13 changed files with 146 additions and 194 deletions.
87 changes: 11 additions & 76 deletions destinations/airbyte-faros-destination/src/converters/googlecalendar/calendar_models.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ export interface Schema$Event {
* Whether attendees may have been omitted from the event's representation. When retrieving an event, this may be due to a restriction specified by the maxAttendee query parameter. When updating an event, this can be used to only update the participant's response. Optional. The default is False.
*/
attendeesOmitted?: boolean | null;
/**
* Calendar ID the event belongs to
*/
calendarId?: string | null;
/**
* The color of the event. This is an ID referring to an entry in the event section of the colors definition (see the colors endpoint). Optional.
*/
Expand Down Expand Up @@ -216,110 +220,41 @@ export interface Schema$Event {
visibility?: string | null;
}

export interface Schema$CalendarListEntry {
/**
* The effective access role that the authenticated user has on the calendar. Read-only. Possible values are:
* - "freeBusyReader" - Provides read access to free/busy information.
* - "reader" - Provides read access to the calendar. Private events will appear to users with reader access, but event details will be hidden.
* - "writer" - Provides read and write access to the calendar. Private events will appear to users with writer access, and event details will be visible.
* - "owner" - Provides ownership of the calendar. This role has all of the permissions of the writer role with the additional ability to see and manipulate ACLs.
*/
accessRole?: string | null;
/**
* The main color of the calendar in the hexadecimal format "#0088aa". This property supersedes the index-based colorId property. To set or change this property, you need to specify colorRgbFormat=true in the parameters of the insert, update and patch methods. Optional.
*/
backgroundColor?: string | null;
/**
* The color of the calendar. This is an ID referring to an entry in the calendar section of the colors definition (see the colors endpoint). This property is superseded by the backgroundColor and foregroundColor properties and can be ignored when using these properties. Optional.
*/
colorId?: string | null;
export interface Schema$Calendar {
/**
* Conferencing properties for this calendar, for example what types of conferences are allowed.
*/
conferenceProperties?: Schema$ConferenceProperties;
/**
* The default reminders that the authenticated user has for this calendar.
*/
defaultReminders?: Schema$EventReminder[];
/**
* Whether this calendar list entry has been deleted from the calendar list. Read-only. Optional. The default is False.
*/
deleted?: boolean | null;
/**
* Description of the calendar. Optional. Read-only.
* Description of the calendar. Optional.
*/
description?: string | null;
/**
* ETag of the resource.
*/
etag?: string | null;
/**
* The foreground color of the calendar in the hexadecimal format "#ffffff". This property supersedes the index-based colorId property. To set or change this property, you need to specify colorRgbFormat=true in the parameters of the insert, update and patch methods. Optional.
*/
foregroundColor?: string | null;
/**
* Whether the calendar has been hidden from the list. Optional. The attribute is only returned when the calendar is hidden, in which case the value is true.
*/
hidden?: boolean | null;
/**
* Identifier of the calendar.
* Identifier of the calendar. To retrieve IDs call the calendarList.list() method.
*/
id?: string | null;
/**
* Type of the resource ("calendar#calendarListEntry").
* Type of the resource ("calendar#calendar").
*/
kind?: string | null;
/**
* Geographic location of the calendar as free-form text. Optional. Read-only.
* Geographic location of the calendar as free-form text. Optional.
*/
location?: string | null;
/**
* The notifications that the authenticated user is receiving for this calendar.
*/
notificationSettings?: {
notifications?: Schema$CalendarNotification[];
} | null;
/**
* Whether the calendar is the primary calendar of the authenticated user. Read-only. Optional. The default is False.
*/
primary?: boolean | null;
/**
* Whether the calendar content shows up in the calendar UI. Optional. The default is False.
*/
selected?: boolean | null;
/**
* Title of the calendar. Read-only.
* Title of the calendar.
*/
summary?: string | null;
/**
* The summary that the authenticated user has set for this calendar. Optional.
*/
summaryOverride?: string | null;
/**
* The time zone of the calendar. Optional. Read-only.
* The time zone of the calendar. (Formatted as an IANA Time Zone Database name, e.g. "Europe/Zurich".) Optional.
*/
timeZone?: string | null;
}

interface Schema$CalendarNotification {
/**
* The method used to deliver the notification. The possible value is:
* - "email" - Notifications are sent via email.
* Required when adding a notification.
*/
method?: string | null;
/**
* The type of notification. Possible values are:
* - "eventCreation" - Notification sent when a new event is put on the calendar.
* - "eventChange" - Notification sent when an event is changed.
* - "eventCancellation" - Notification sent when an event is cancelled.
* - "eventResponse" - Notification sent when an attendee responds to the event invitation.
* - "agenda" - An agenda with the events of the day (sent out in the morning).
* Required when adding a notification.
*/
type?: string | null;
}

interface Schema$ConferenceData {
/**
* The ID of the conference.
Expand Down
4 changes: 2 additions & 2 deletions destinations/airbyte-faros-destination/src/converters/googlecalendar/common.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import {AirbyteRecord} from 'faros-airbyte-cdk';

import {Converter} from '../converter';
import {Schema$CalendarListEntry, Schema$Event} from './calendar_models';
import {Schema$Calendar, Schema$Event} from './calendar_models';

export type Event = Schema$Event;
export type Calendar = Schema$CalendarListEntry;
export type Calendar = Schema$Calendar;

export interface CategoryRef {
category: string;
Expand Down
74 changes: 48 additions & 26 deletions destinations/airbyte-faros-destination/src/converters/googlecalendar/events.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ export class Events extends GoogleCalendarConverter {
// Locations cache to avoid querying the API for the same location
private locationsCache = new Map<string, Location>();

private usersSeen = new Set<string>();

readonly destinationModels: ReadonlyArray<DestinationModel> = [
'cal_Event',
'cal_EventGuestAssociation',
Expand All @@ -23,41 +25,57 @@ export class Events extends GoogleCalendarConverter {
const source = this.streamName.source;
const event = record.record.data as Event;
const res: DestinationRecord[] = [];
const eventRef = {uid: event.id, calendar: {uid: event.id, source}};

event.attendees?.forEach((attender) => {
const attenderRef = {uid: attender.id, source};
res.push({
model: 'cal_User',
record: {
...attenderRef,
email: attender.email,
displayName: attender.displayName,
},
});
const eventRef = {
uid: event.id,
calendar: {uid: event.calendarId, source},
};

for (const attendee of event.attendees ?? []) {
const uid = this.userUid(attendee);
if (!uid) continue;

const attendeeRef = {uid, source};
if (!this.usersSeen.has(uid)) {
this.usersSeen.add(uid);
res.push({
model: 'cal_User',
record: {
...attendeeRef,
email: attendee.email,
displayName: attendee.displayName,
},
});
}
res.push({
model: 'cal_EventGuestAssociation',
record: {
event: eventRef,
guest: attenderRef,
guest: attendeeRef,
status: GoogleCalendarCommon.EventGuestStatus(
attender.responseStatus
attendee.responseStatus
),
},
});
});
}

let organizer = null;
if (event.organizer) {
organizer = {uid: event.organizer?.id || event.organizer?.email, source};
res.push({
model: 'cal_User',
record: {
...organizer,
email: event.organizer?.email,
displayName: event.organizer?.displayName,
},
});
let organizerRef = null;
const organizerUid = this.userUid(event.organizer);

if (organizerUid) {
organizerRef = {uid: organizerUid, source};

if (!this.usersSeen.has(organizerUid)) {
this.usersSeen.add(organizerUid);
res.push({
model: 'cal_User',
record: {
...organizerRef,
email: event.organizer?.email,
displayName: event.organizer?.displayName,
},
});
}
}

const start = Utils.toDate(
Expand Down Expand Up @@ -124,7 +142,7 @@ export class Events extends GoogleCalendarConverter {
durationMs,
url: event.htmlLink,
location,
organizer,
organizer: organizerRef,
type: GoogleCalendarCommon.EventType(event.eventType),
visibility: GoogleCalendarCommon.EventVisibility(event.transparency),
privacy: GoogleCalendarCommon.EventPrivacy(event.visibility),
Expand All @@ -133,4 +151,8 @@ export class Events extends GoogleCalendarConverter {
});
return res;
}

private userUid(user: any): string | undefined {
return user?.id || user?.email;
}
}
2 changes: 1 addition & 1 deletion destinations/airbyte-faros-destination/test/converters/googlecalendar.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ describe('googlecalendar', () => {
cal_Calendar: 1,
cal_Event: 3,
cal_EventGuestAssociation: 2,
cal_User: 5,
cal_User: 2,
geo_Address: 2,
geo_Coordinates: 2,
geo_Location: 2,
Expand Down
6 changes: 3 additions & 3 deletions destinations/airbyte-faros-destination/test/resources/googlecalendar/all-streams.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
{"record":{"stream":"mytestsource__googlecalendar__calendars","emitted_at":1637564413884,"data":{"kind":"calendar#calendarListEntry","etag":"\"1637221140246000\"","id":"5vokul6v3odfgpopgq0pec51k0@group.calendar.google.com","summary":"First created calendar in this account","timeZone":"UTC","colorId":"10","backgroundColor":"#b3dc6c","foregroundColor":"#000000","selected":true,"accessRole":"owner","defaultReminders":[]}},"type":"RECORD"}
{"log":{"level":"INFO","message":"Finished syncing calendars stream. Read 1 records"},"type":"LOG"}
{"log":{"level":"INFO","message":"Syncing events stream in full mode"},"type":"LOG"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414343,"data":{"kind":"calendar#event","etag":"\"3274444024130000\"","id":"pu9eddmgti5ui6gqgnhf8ki75g","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=cHU5ZWRkbWd0aTV1aTZncWduaGY4a2k3NWcgZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:53:32.000Z","updated":"2021-11-18T07:53:32.065Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T20:00:00Z","timeZone":"UTC"},"end":{"dateTime":"2022-01-01T00:00:00Z","timeZone":"UTC"},"iCalUID":"pu9eddmgti5ui6gqgnhf8ki75g@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default","location":"United States, California, office #1","attendees": [{"id":"1","email":"test@gmail.com","displayName":"Test Attender","responseStatus":"Accepted"}]}},"type":"RECORD"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414343,"data":{"kind":"calendar#event","etag":"\"3274444171862000\"","id":"violnsh17k2gpmf15s74t07uak","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=dmlvbG5zaDE3azJncG1mMTVzNzR0MDd1YWsgZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:54:45.000Z","updated":"2021-11-18T07:54:45.931Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T12:00:00Z","timeZone":"Europe/Zurich"},"end":{"dateTime":"2021-12-31T15:00:00Z","timeZone":"Europe/Zurich"},"iCalUID":"violnsh17k2gpmf15s74t07uak@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default", "location": "Online"}},"type":"RECORD"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414344,"data":{"kind":"calendar#event","etag":"\"3274444185994000\"","id":"eeo5b8rm34097ahr16eovtdceo","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=ZWVvNWI4cm0zNDA5N2FocjE2ZW92dGRjZW8gZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:54:52.000Z","updated":"2021-11-18T07:54:52.997Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T11:00:00Z","timeZone":"Europe/Kiev"},"end":{"dateTime":"2021-12-31T15:00:00Z","timeZone":"Europe/Kiev"},"iCalUID":"eeo5b8rm34097ahr16eovtdceo@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default", "attendees": [{ "id": "1", "email": "test@gmail.com", "displayName": "Test Attender", "responseStatus": "Accepted" }]}},"type":"RECORD"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414343,"data":{"kind":"calendar#event","etag":"\"3274444024130000\"","calendarId": "5vokul6v3odfgpopgq0pec51k0@group.calendar.google.com","id":"pu9eddmgti5ui6gqgnhf8ki75g","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=cHU5ZWRkbWd0aTV1aTZncWduaGY4a2k3NWcgZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:53:32.000Z","updated":"2021-11-18T07:53:32.065Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T20:00:00Z","timeZone":"UTC"},"end":{"dateTime":"2022-01-01T00:00:00Z","timeZone":"UTC"},"iCalUID":"pu9eddmgti5ui6gqgnhf8ki75g@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default","location":"United States, California, office #1","attendees": [{"id":"1","email":"test@gmail.com","displayName":"Test Attender","responseStatus":"Accepted"}]}},"type":"RECORD"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414343,"data":{"kind":"calendar#event","etag":"\"3274444171862000\"","calendarId": "5vokul6v3odfgpopgq0pec51k0@group.calendar.google.com","id":"violnsh17k2gpmf15s74t07uak","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=dmlvbG5zaDE3azJncG1mMTVzNzR0MDd1YWsgZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:54:45.000Z","updated":"2021-11-18T07:54:45.931Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T12:00:00Z","timeZone":"Europe/Zurich"},"end":{"dateTime":"2021-12-31T15:00:00Z","timeZone":"Europe/Zurich"},"iCalUID":"violnsh17k2gpmf15s74t07uak@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default", "location": "Online"}},"type":"RECORD"}
{"record":{"stream":"mytestsource__googlecalendar__events","emitted_at":1637564414344,"data":{"kind":"calendar#event","etag":"\"3274444185994000\"","calendarId": "5vokul6v3odfgpopgq0pec51k0@group.calendar.google.com","id":"eeo5b8rm34097ahr16eovtdceo","status":"confirmed","htmlLink":"https://www.google.com/calendar/event?eid=ZWVvNWI4cm0zNDA5N2FocjE2ZW92dGRjZW8gZmFyb3MtdGVzdEBmYXJvcy10ZXN0LTMzMjIwOS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbQ","created":"2021-11-18T07:54:52.000Z","updated":"2021-11-18T07:54:52.997Z","creator":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"organizer":{"email":"faros-test@faros-test-332209.iam.gserviceaccount.com","self":true},"start":{"dateTime":"2021-12-31T11:00:00Z","timeZone":"Europe/Kiev"},"end":{"dateTime":"2021-12-31T15:00:00Z","timeZone":"Europe/Kiev"},"iCalUID":"eeo5b8rm34097ahr16eovtdceo@google.com","sequence":0,"reminders":{"useDefault":true},"eventType":"default", "attendees": [{ "id": "1", "email": "test@gmail.com", "displayName": "Test Attender", "responseStatus": "Accepted" }]}},"type":"RECORD"}
{"log":{"level":"INFO","message":"Finished syncing events stream. Read 3 records"},"type":"LOG"}
{"log":{"level":"INFO","message":"Finished syncing GooglecalendarSource"},"type":"LOG"}
Loading

0 comments on commit 7a5849a

Please sign in to comment.