Merge pull request #24 from antonplagemann/development

v3.2.0 Added nickname support & improved matching

GMSync.py

@@ -17,7 +17,7 @@
 from MonicaHelper import Monica
 from SyncHelper import Sync
 
-VERSION = "v3.1.3"
+VERSION = "v3.2.0"
 DATABASE_FILENAME = "syncState.db"
 LOG_FILENAME = 'Sync.log'
 # Google -> Monica contact syncing script
@@ -104,7 +104,7 @@ def main() -> None:
         msg = f"Script aborted: {type(e).__name__}: {str(e)}\n"
         log.exception(e)
         print("\n" + msg)
-        sys.exit(1)
+        raise SystemExit(1) from e
 
 
 if __name__ == '__main__':

GoogleHelper.py

@@ -117,7 +117,8 @@ def getContactNames(self, googleContact: dict) -> Tuple[str, str, str, str, str,
         middleName = names.get("middleName", '')
         prefix = names.get("honorificPrefix", '')
         suffix = names.get("honorificSuffix", '')
-        return givenName, middleName, familyName, displayName, prefix, suffix
+        nickname = googleContact.get('nicknames', [{}])[0].get('value', '')
+        return givenName, middleName, familyName, displayName, prefix, suffix, nickname
 
     def getContactAsString(self, googleContact: dict) -> str:
         '''Get some content from a Google contact to identify it as a user and return it as string.'''

MonicaHelper.py

@@ -21,6 +21,7 @@ def __init__(self, log: Logger, databaseHandler: Database, token: str, base_url:
         self.parameters = {'limit': 100}
         self.dataAlreadyFetched = False
         self.contacts = []
+        self.genderMapping = {}
         self.updatedContacts = {}
         self.createdContacts = {}
         self.deletedContacts = {}
@@ -51,6 +52,31 @@ def updateStatistics(self) -> None:
         self.updatedContacts = {key: value for key, value in self.updatedContacts.items()
                                 if key not in self.createdContacts}
 
+    def getGenderMapping(self) -> dict:
+        '''Fetches all genders from Monica and saves them to a dictionary.'''
+        # Only fetch if not present yet
+        if self.genderMapping:
+            return self.genderMapping
+
+        while True:
+        # Get genders
+            response = requests.get(
+                self.base_url + f"/genders", headers=self.header, params=self.parameters)
+            self.apiRequests += 1
+
+            # If successful
+            if response.status_code == 200:
+                genders = response.json()['data']
+                genderMapping = {gender['type']: gender['id'] for gender in genders}
+                self.genderMapping = genderMapping
+                return self.genderMapping
+            else:
+                error = response.json()['error']['message']
+                if self.__isSlowDownError(response, error):
+                    continue
+                self.log.error(f"Failed to fetch genders from Monica: {error}")
+                raise Exception("Error fetching genders from Monica!")
+
     def updateContact(self, monicaId: str, data: dict) -> None:
         '''Updates a given contact and its id via api call.'''
         name = f"{data['first_name']} {data['last_name']}"
@@ -101,7 +127,7 @@ def deleteContact(self, monicaId: str, name: str) -> None:
                 self.log.error(f"'{name}' ('{monicaId}'): Failed to complete delete request: {error}")
                 raise Exception("Error deleting Monica contact!")
 
-    def createContact(self, data: dict) -> dict:
+    def createContact(self, data: dict, referenceId: str) -> dict:
         '''Creates a given Monica contact via api call and returns the created contact.'''
         name = f"{data['first_name']} {data['last_name']}".strip()
 
@@ -116,13 +142,13 @@ def createContact(self, data: dict) -> dict:
                 contact = response.json()['data']
                 self.createdContacts[contact['id']] = True
                 self.contacts.append(contact)
-                self.log.info(f"'{name}' ('{contact['id']}'): Contact created successfully")
+                self.log.info(f"'{referenceId}' ('{contact['id']}'): Contact created successfully")
                 return contact
             else:
                 error = response.json()['error']['message']
                 if self.__isSlowDownError(response, error):
                     continue
-                self.log.info(f"'{name}': Error creating Monica contact: {error}")
+                self.log.info(f"'{referenceId}': Error creating Monica contact: {error}")
                 raise Exception("Error creating Monica contact!")
 
     def getContacts(self) -> List[dict]:
@@ -470,15 +496,15 @@ def __isSlowDownError(self, response: Response, error: str) -> bool:
 class MonicaContactUploadForm():
     '''Creates json form for creating or updating Monica contacts.'''
 
-    def __init__(self, firstName: str, lastName: str = None, nickName: str = None,
+    def __init__(self, monica: Monica, firstName: str, lastName: str = None, nickName: str = None,
                  middleName: str = None, genderType: str = 'O', birthdateDay: str = None,
                  birthdateMonth: str = None, birthdateYear: str = None,
                  birthdateAgeBased: bool = None, isBirthdateKnown: bool = False,
                  isDeceased: bool = False, isDeceasedDateKnown: bool = False,
                  deceasedDay: int = None, deceasedMonth: int = None,
                  deceasedYear: int = None, deceasedAgeBased: bool = None,
                  createReminders: bool = True) -> None:
-        genderId = {'M': 1, 'F': 2, 'O': 3}.get(genderType, 3)
+        genderId = monica.getGenderMapping()[genderType]
         self.data = {
             "first_name": firstName,
             "last_name": lastName,

README.md

@@ -11,7 +11,7 @@ That being said: Be welcome to use it, fork it, copy it for your own projects, a
 ## Features
 
 - One-way sync (Google → Monica)
-  - Syncs the following details: first name, last name, middle name, birthday, job title, company, addresses, phone numbers, email addresses, labels (tags), notes (see [limits](#limits))
+  - Syncs the following details: first name, last name, middle name, nickname, birthday, job title, company, addresses, phone numbers, email addresses, labels (tags), notes (see [limits](#limits))
 - Advanced matching of already present Monica contacts (e.g. from earlier contact import)
 - User choice prompt before any modification to your Monica data during initial sync (you can choose to abort before the script makes any change).
 - Fast delta sync using Google sync tokens
@@ -25,8 +25,7 @@ That being said: Be welcome to use it, fork it, copy it for your own projects, a
 - **Do not delete synced contacts at Monica.** This will cause a sync error which you can resolve by doing initial sync again.
 - Monica limits API usage to 60 calls per minute. As every contact needs *at least* 2 API calls, **the script can not sync more than 30 contacts per minute** (thus affecting primarily initial and full sync).
 - Delta sync will fail if there are more than 7 days between the last sync (Google restriction). In this case, the script will automatically do full sync instead
-- No support for custom Monica gender types. Will be overwritten with standard type O (other) during sync.
-- No support for nickname and gender sync (support can be added, file an issue if you want it). Nicknames and genders will be overwritten during sync.
+- No support for gender sync (support can be added, file an issue if you want it).
 - A label itself won't be deleted automatically if it has been removed from the last contact.
 - If there is a Google note it will be synced with exactly one Monica note. To this end, a small text will be added to the synced note at Monica. This makes it easy for you to distinguish synced and Monica-only notes. This means **you can update and create as many *additional* notes as you want at Monica**, they will not be overwritten.
 - Monica contacts require a first name. If a Google contact does not have any name, it will be skipped.

SyncHelper.py

@@ -256,6 +256,9 @@ def __syncNotes(self, googleContact: dict, monicaContact: dict) -> None:
                 "contact_id": monicaContact["id"],
                 "is_favorited": False
                 }
+                # Convert normal newlines to markdown newlines
+                googleNote["body"] = googleNote["body"].replace("\n", "  \n")
+
                 if not monicaNotes:
                     # If there is no Monica note sync the Google note
                     googleNote["body"] += identifier
@@ -867,18 +870,11 @@ def __mergeAndUpdateNBD(self, monicaContact: dict, googleContact: dict) -> None:
         firstName, lastName = self.__getMonicaNamesFromGoogleContact(googleContact)
         middleName = self.google.getContactNames(googleContact)[1]
         displayName = self.google.getContactNames(googleContact)[3]
+        nickName = self.google.getContactNames(googleContact)[6]
         # First name is required for Monica
         if not firstName:
             firstName = displayName
             lastName = ''
-        if not any([firstName, lastName, middleName, displayName]):
-            self.log.info(f"Empty name for '{googleContact['resourceName']}' detected -> using Monica names instead.")
-            # Get all Monica names
-            firstName = monicaContact['first_name'] or ''
-            lastName = monicaContact['last_name'] or ''
-            fullName = monicaContact['complete_name'] or ''
-            nickname = monicaContact['nickname'] or ''
-            middleName = self.__getMonicaMiddleName(firstName, lastName, nickname, fullName)
 
         # Get birthday
         birthday = googleContact.get("birthdays", None)
@@ -899,7 +895,7 @@ def __mergeAndUpdateNBD(self, monicaContact: dict, googleContact: dict) -> None:
             deceasedDay = date.day
 
         # Assemble form object
-        googleForm = MonicaContactUploadForm(firstName=firstName, lastName=lastName, nickName=monicaContact["nickname"],
+        googleForm = MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, nickName=nickName,
                                        middleName=middleName, genderType=monicaContact["gender_type"],
                                        birthdateDay=birthdateDay, birthdateMonth=birthdateMonth,
                                        birthdateYear=birthdateYear, isBirthdateKnown=bool(birthday),
@@ -948,7 +944,7 @@ def __getMonicaForm(self, monicaContact: dict) -> MonicaContactUploadForm:
             deceasedDay = date.day
 
         # Assemble form object
-        return MonicaContactUploadForm(firstName=firstName, lastName=lastName, nickName=monicaContact["nickname"],
+        return MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, nickName=nickname,
                                        middleName=middleName, genderType=monicaContact["gender_type"],
                                        birthdateDay=birthdateDay, birthdateMonth=birthdateMonth,
                                        birthdateYear=birthdateYear, isBirthdateKnown=bool(birthdayTimestamp),
@@ -963,6 +959,7 @@ def __createMonicaContact(self, googleContact: dict) -> dict:
         firstName, lastName = self.__getMonicaNamesFromGoogleContact(googleContact)
         middleName = self.google.getContactNames(googleContact)[1]
         displayName = self.google.getContactNames(googleContact)[3]
+        nickName = self.google.getContactNames(googleContact)[6]
         # First name is required for Monica
         if not firstName:
             firstName = displayName
@@ -977,12 +974,13 @@ def __createMonicaContact(self, googleContact: dict) -> dict:
             birthdateDay = birthday[0].get("date", {}).get("day", None)
 
         # Assemble form object
-        form = MonicaContactUploadForm(firstName=firstName, lastName=lastName, middleName=middleName,
-                                       birthdateDay=birthdateDay, birthdateMonth=birthdateMonth,
-                                       birthdateYear=birthdateYear, isBirthdateKnown=bool(birthday),
+        form = MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, middleName=middleName,
+                                       nickName=nickName, birthdateDay=birthdateDay,
+                                       birthdateMonth=birthdateMonth, birthdateYear=birthdateYear,
+                                       isBirthdateKnown=bool(birthday), 
                                        createReminders=self.monica.createReminders)
         # Upload contact
-        monicaContact = self.monica.createContact(data=form.data)
+        monicaContact = self.monica.createContact(data=form.data, referenceId=googleContact['resourceName'])
         return monicaContact
 
     def __convertGoogleTimestamp(self, timestamp: str) -> datetime:
@@ -1074,20 +1072,37 @@ def __simpleMonicaIdSearch(self, googleContact: dict) -> Union[str, None]:
         Tries to find a matching Monica contact and returns its id or None if not found'''
         # Initialization
         gContactGivenName = self.google.getContactNames(googleContact)[0]
+        gContactMiddleName = self.google.getContactNames(googleContact)[1]
         gContactFamilyName = self.google.getContactNames(googleContact)[2]
         gContactDisplayName = self.google.getContactNames(googleContact)[3]
         candidates = []
 
         # Process every Monica contact
         for monicaContact in self.monica.getContacts():
-            if (str(monicaContact['id']) not in self.mapping.values()
-                and (gContactDisplayName == monicaContact['complete_name']
-                     or (gContactGivenName
-                         and gContactFamilyName
-                         and ' '.join([gContactGivenName, gContactFamilyName]) == monicaContact['complete_name']))):
-                    # If the id isnt in the database and full name matches add potential candidate to list
-                    # Sometimes Google does some strange naming things with 'honoricPrefix' etc.; try to mitigate that
-                    candidates.append(monicaContact)
+            # Get monica data
+            mContactId = str(monicaContact['id'])
+            mContactFirstName = monicaContact['first_name'] or ''
+            mContactLastName = monicaContact['last_name'] or ''
+            mContactFullName = monicaContact['complete_name'] or ''
+            mContactNickname = monicaContact['nickname'] or ''
+            mContactMiddleName = self.__getMonicaMiddleName(mContactFirstName, mContactLastName, mContactNickname, mContactFullName)
+            # Check if the Monica contact is already assigned to a Google contact
+            isMonicaContactAssigned = mContactId in self.mapping.values()
+            # Check if display names match
+            isDisplayNameMatch = (gContactDisplayName == mContactFullName)
+            # Pre-check that the Google contact has a given and a family name
+            hasNames = gContactGivenName and gContactFamilyName
+            # Check if names match when ignoring honorifix prefixes
+            isWithoutPrefixMatch = hasNames and (' '.join([gContactGivenName, gContactFamilyName]) == mContactFullName)
+            # Check if first, middle and last name matches
+            isFirstLastMiddleNameMatch = (mContactFirstName == gContactGivenName
+                                          and mContactMiddleName == gContactMiddleName
+                                          and mContactLastName == gContactFamilyName)
+            # Assemble all conditions
+            matches = [isDisplayNameMatch, isWithoutPrefixMatch, isFirstLastMiddleNameMatch]
+            if not isMonicaContactAssigned and any(matches):
+                # Add possible candidate
+                candidates.append(monicaContact)
 
         # If there is only one candidate
         if len(candidates) == 1:
@@ -1108,7 +1123,7 @@ def __simpleMonicaIdSearch(self, googleContact: dict) -> Union[str, None]:
     def __getMonicaNamesFromGoogleContact(self, googleContact: dict) -> Tuple[str, str]:
         '''Creates first and last name from a Google contact with respect to honoric
         suffix/prefix.'''
-        givenName, _, familyName, _, prefix, suffix = self.google.getContactNames(googleContact)
+        givenName, _, familyName, _, prefix, suffix, _ = self.google.getContactNames(googleContact)
         if prefix:
             givenName = f"{prefix} {givenName}".strip()
         if suffix: