diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..834d90e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/.github +**/.git +**/.gitignore +**/.vscode +**/__pycache__ +**/logs/* +**/data/* +*.properties +*.bat +*.json +*.pickle +*.log +*.db +*.md +*.yml +**/conf.py +data +logs +test +.mypy_cache +Dockerfile +.env +.env.example +.pylintrc +.dockerignore diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2807b6b --- /dev/null +++ b/.env.example @@ -0,0 +1,38 @@ +# Example config file with every option +# Separate multiple values with comma + +# -------------Required----------------------- +# Your Monica api token (without 'Bearer ') +TOKEN=YOUR_TOKEN_HERE + +# -------------Optional----------------------- +# Your Monica base url +BASE_URL=https://app.monicahq.com/api + +# Create reminders for birthdays and deceased days? +CREATE_REMINDERS=True +# Delete Monica contact if the corresponding Google contact has been deleted? +DELETE_ON_SYNC=True +# Do a street reversal in address sync if the first character is a number? +# (e.g. from '13 Auenweg' to 'Auenweg 13') +STREET_REVERSAL=False + +# What fields should be synced? (both directions) +# Names and birthday are mandatory +FIELDS=career,address,phone,email,labels,notes + +# Define contact labels/tags/groups you want to include or exclude from sync. +# Exclude labels have the higher priority. +# Both lists empty means every contact is included +# Example: 'GOOGLE_LABELS_INCLUDE=Family,My Friends' will only process contacts labeled as 'Family' or 'My Friends'. +# Applies for Google -> Monica sync +GOOGLE_LABELS_INCLUDE= +GOOGLE_LABELS_EXCLUDE= +# Applies for Monica -> Google sync back +MONICA_LABELS_INCLUDE= +MONICA_LABELS_EXCLUDE= + +# Define custom file paths +DATABASE_FILE=data/syncState.db +GOOGLE_TOKEN_FILE=data/token.pickle +GOOGLE_CREDENTIALS_FILE=data/credentials.json diff --git a/.github/actions/cleanup-environment/action.yml b/.github/actions/cleanup-environment/action.yml new file mode 100644 index 0000000..f56deb3 --- /dev/null +++ b/.github/actions/cleanup-environment/action.yml @@ -0,0 +1,30 @@ +name: 'Cleanup test environment' +description: 'Restores changed Google contacts and renames log files' + +inputs: + TEST_RUNNER: + description: 'The system used to run the scripts (python/docker)' + required: true + REPO_TOKEN: + description: 'A GitHub Personal Access token with repo scope' + required: true + +runs: + using: "composite" + steps: + - name: Restore original data + run: python test/ChaosMonkey.py --restore + shell: bash + + - name: Rename log files + run: | + mv logs/monkey.log logs/${{ inputs.TEST_RUNNER }}_monkey.log + mv logs/setup.log logs/${{ inputs.TEST_RUNNER }}_setup.log + mv data/syncState.db data/${{ inputs.TEST_RUNNER }}_syncState.db + shell: bash + + - name: Upload Google token to repo secrets + run: python test/UpdateToken.py + env: + REPO_TOKEN: ${{ inputs.REPO_TOKEN }} + shell: bash diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml new file mode 100644 index 0000000..c99c9ba --- /dev/null +++ b/.github/actions/setup-environment/action.yml @@ -0,0 +1,64 @@ +name: 'Setup test environment' +description: 'Sets up a new Monica instance, installs dependencies, and creates/retrieves sync tokens' + +inputs: + GOOGLE_TOKEN: + description: The token.pickle file content (base64 encoded string) + required: true + MONICA_URL: + required: false + default: http://localhost:8080/api + CREATE_REMINDERS: + required: true + DELETE_ON_SYNC: + required: true + STREET_REVERSAL: + required: true + FIELDS: + required: true + GOOGLE_LABELS_INCLUDE: + required: true + GOOGLE_LABELS_EXCLUDE: + required: true + MONICA_LABELS_INCLUDE: + required: true + MONICA_LABELS_EXCLUDE: + required: true + +runs: + using: "composite" + steps: + - name: Start new Monica instance + run: docker-compose -f test/docker-compose-monica.yml up -d + shell: bash + + - name: Create env file + run: | + touch .env data/token.pickle + echo "${{ inputs.GOOGLE_TOKEN }}" >> data/token.pickle + echo BASE_URL="${{ inputs.MONICA_URL }}" >> .env + echo CREATE_REMINDERS="${{ inputs.CREATE_REMINDERS || true }}" >> .env + echo DELETE_ON_SYNC="${{ inputs.DELETE_ON_SYNC || true }}" >> .env + echo STREET_REVERSAL="${{ inputs.STREET_REVERSAL || false }}" >> .env + echo FIELDS="${{ inputs.FIELDS || 'career,address,phone,email,labels,notes' }}" >> .env + echo GOOGLE_LABELS_INCLUDE="${{ inputs.GOOGLE_LABELS_INCLUDE }}" >> .env + echo GOOGLE_LABELS_EXCLUDE="${{ inputs.GOOGLE_LABELS_EXCLUDE }}" >> .env + echo MONICA_LABELS_INCLUDE="${{ inputs.MONICA_LABELS_INCLUDE }}" >> .env + echo MONICA_LABELS_EXCLUDE="${{ inputs.MONICA_LABELS_EXCLUDE }}" >> .env + shell: bash + + - name: Install requirements + run: python -m pip install --upgrade pip && pip install -r requirements.txt -r test/requirements.txt + shell: bash + + - name: Create API token at Monica instance + run: python test/SetupToken.py + shell: bash + + - name: Create database file and monkey state + run: python test/ChaosMonkey.py + shell: bash + + - name: Set folder permissions for non-root containers + run: sudo chmod 777 data logs -R + shell: bash diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..00855ef --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,37 @@ +name: "CodeQL" + +on: + push: + branches: + - main + pull_request: + types: ['opened', 'reopened', 'synchronize'] + schedule: + - cron: '31 0 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/docker-cd-release.yml b/.github/workflows/docker-cd-release.yml new file mode 100644 index 0000000..1c19452 --- /dev/null +++ b/.github/workflows/docker-cd-release.yml @@ -0,0 +1,41 @@ +name: Docker Release CD + +on: + release: + types: [published] + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: "Create tags for Docker image" + id: docker_meta + uses: docker/metadata-action@v3.6.1 + with: + tag-latest: true + images: antonplagemann/google-monica-sync + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + no-cache: true + push: true + tags: ${{ steps.docker_meta.outputs.tags }} diff --git a/.github/workflows/docker-cd.yml b/.github/workflows/docker-cd.yml new file mode 100644 index 0000000..2d41442 --- /dev/null +++ b/.github/workflows/docker-cd.yml @@ -0,0 +1,35 @@ +name: Docker CD + +on: + push: + branches: + - main + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + no-cache: true + push: true + tags: antonplagemann/google-monica-sync:latest diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000..d59d235 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,341 @@ +name: Python CI +# 1. Run linting and static code checks +# 2. Test python syncing script +# 3. Build and publish docker container +# 4. Test published docker container + +on: + pull_request: + types: ['opened', 'reopened', 'synchronize'] + + workflow_dispatch: + inputs: + numChaos: + description: 'Number of items the chaos monkey should manipulate during test' + required: false + default: '4' + CREATE_REMINDERS: + description: Create reminders for birthdays and deceased days? + required: false + type: boolean + default: true + DELETE_ON_SYNC: + description: Delete Monica contact if the corresponding Google contact has been deleted? + required: false + type: boolean + default: true + STREET_REVERSAL: + description: Do a street reversal in address sync if the first character is a number? + required: false + type: boolean + default: false + FIELDS: + description: What fields should be synced? (both directions) + required: false + default: career,address,phone,email,labels,notes + GOOGLE_LABELS_INCLUDE: + description: Define Google contact labels/tags/groups you want to include in sync + required: false + default: + GOOGLE_LABELS_EXCLUDE: + description: Define Google contact labels/tags/groups you want to exclude from sync + required: false + default: + MONICA_LABELS_INCLUDE: + description: Define Monica contact labels/tags/groups you want to include in sync + required: false + default: + MONICA_LABELS_EXCLUDE: + description: Define Monica contact labels/tags/groups you want to exclude from sync + required: false + default: + +# 1. Run linting and static code checks +jobs: + black_formatting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable + with: + options: "-l 105 --check --diff" + src: "." + codespell_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: .*,*.csv + bandit_security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: jpetrucciani/bandit-check@master + with: + path: '.' + bandit_flags: '--recursive --skip B403,B101,B301' + flake8_lint: + runs-on: ubuntu-latest + steps: + - name: Check out source repository + uses: actions/checkout@v2 + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: flake8 Lint + uses: py-actions/flake8@v2 + with: + ignore: "E203,W503,E231,E402" + max-line-length: "105" + path: "." + args: '--count --show-source --statistics' + isort_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - uses: isort/isort-action@master + with: + configuration: "--check-only --profile black" + mypy_typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: jpetrucciani/mypy-check@master + with: + path: '.' + mypy_flags: '--install-types --non-interactive' + +# 2. Test python syncing script + python_test: + runs-on: ubuntu-latest + needs: [black_formatting, codespell_check, bandit_security, flake8_lint, isort_check, mypy_typecheck] + env: + TEST_RUNNER: python + steps: + # Setup environment + - name: Check out source repository + uses: actions/checkout@v2 + - name: Setup python environment + uses: actions/setup-python@v2 + - name: Setup testing environment + uses: ./.github/actions/setup-environment + with: + GOOGLE_TOKEN: ${{ secrets.GOOGLE_TOKEN }} + CREATE_REMINDERS: ${{ github.event.inputs.CREATE_REMINDERS }} + DELETE_ON_SYNC: ${{ github.event.inputs.DELETE_ON_SYNC }} + STREET_REVERSAL: ${{ github.event.inputs.STREET_REVERSAL }} + FIELDS: ${{ github.event.inputs.FIELDS }} + GOOGLE_LABELS_INCLUDE: ${{ github.event.inputs.GOOGLE_LABELS_INCLUDE }} + GOOGLE_LABELS_EXCLUDE: ${{ github.event.inputs.GOOGLE_LABELS_EXCLUDE }} + MONICA_LABELS_INCLUDE: ${{ github.event.inputs.MONICA_LABELS_INCLUDE }} + MONICA_LABELS_EXCLUDE: ${{ github.event.inputs.MONICA_LABELS_EXCLUDE }} + + # Test initial sync + - name: Prepare initial sync + run: python test/ChaosMonkey.py --initial --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run initial sync + run: python GMSync.py --initial + timeout-minutes: 5 + - name: Check initial sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_initial.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_initial.log)" ]] + + # Test delta sync + - name: Prepare delta sync + run: python test/ChaosMonkey.py --delta --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run delta sync + run: python GMSync.py --delta + timeout-minutes: 5 + - name: Check delta sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_delta.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_delta.log)" ]] + + # Test full sync + - name: Prepare full sync + run: python test/ChaosMonkey.py --full --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run full sync + run: python GMSync.py --full + timeout-minutes: 5 + - name: Check full sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_full.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_full.log)" ]] + + # Test sync back + - name: Prepare sync back + run: python test/ChaosMonkey.py --syncback --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run sync back + run: python GMSync.py --syncback + timeout-minutes: 5 + - name: Check sync back results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_syncback.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_syncback.log)" ]] + + # Test database check + - name: Prepare database check + run: python test/ChaosMonkey.py --check --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run database check + run: python GMSync.py --check + timeout-minutes: 5 + - name: Check database check results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_databasecheck.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_databasecheck.log)" ]] + + # End testing + - name: Cleanup testing environment + uses: ./.github/actions/cleanup-environment + if: always() + with: + TEST_RUNNER: ${{ env.TEST_RUNNER }} + REPO_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + + - name: Upload log files + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + logs/ + data/${{ env.TEST_RUNNER }}_syncState.db + +# 3. Build and publish docker container + docker_build: + runs-on: ubuntu-latest + needs: python_test + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + no-cache: true + push: true + tags: antonplagemann/google-monica-sync:next + +# 4. Test published docker container + docker_test: + runs-on: ubuntu-latest + needs: docker_build + env: + TEST_RUNNER: docker + steps: + # Setup environment + - name: Check out source repository + uses: actions/checkout@v2 + - name: Setup python environment + uses: actions/setup-python@v2 + - name: Setup testing environment + uses: ./.github/actions/setup-environment + with: + GOOGLE_TOKEN: ${{ secrets.GOOGLE_TOKEN }} + CREATE_REMINDERS: ${{ github.event.inputs.CREATE_REMINDERS }} + DELETE_ON_SYNC: ${{ github.event.inputs.DELETE_ON_SYNC }} + STREET_REVERSAL: ${{ github.event.inputs.STREET_REVERSAL }} + FIELDS: ${{ github.event.inputs.FIELDS }} + GOOGLE_LABELS_INCLUDE: ${{ github.event.inputs.GOOGLE_LABELS_INCLUDE }} + GOOGLE_LABELS_EXCLUDE: ${{ github.event.inputs.GOOGLE_LABELS_EXCLUDE }} + MONICA_LABELS_INCLUDE: ${{ github.event.inputs.MONICA_LABELS_INCLUDE }} + MONICA_LABELS_EXCLUDE: ${{ github.event.inputs.MONICA_LABELS_EXCLUDE }} + + # Test initial sync + - name: Prepare initial sync + run: python test/ChaosMonkey.py --initial --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run initial sync + run: docker-compose -f test/docker-compose-sync.yml -f test/docker-compose-sync-initial.yml --env-file .env up + timeout-minutes: 5 + - name: Check initial sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_initial.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_initial.log)" ]] + + # Test delta sync + - name: Prepare delta sync + run: python test/ChaosMonkey.py --delta --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run delta sync + run: docker-compose -f test/docker-compose-sync.yml -f test/docker-compose-sync-delta.yml --env-file .env up + timeout-minutes: 5 + - name: Check delta sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_delta.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_delta.log)" ]] + + # Test full sync + - name: Prepare full sync + run: python test/ChaosMonkey.py --full --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run full sync + run: docker-compose -f test/docker-compose-sync.yml -f test/docker-compose-sync-full.yml --env-file .env up + timeout-minutes: 5 + - name: Check full sync results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_full.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_full.log)" ]] + + # Test sync back + - name: Prepare sync back + run: python test/ChaosMonkey.py --syncback --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run sync back + run: docker-compose -f test/docker-compose-sync.yml -f test/docker-compose-sync-syncback.yml --env-file .env up + timeout-minutes: 5 + - name: Check sync back results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_syncback.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_syncback.log)" ]] + + # Test database check + - name: Prepare database check + run: python test/ChaosMonkey.py --check --num ${{ github.event.inputs.numChaos || 4 }} + - name: Run database check + run: docker-compose -f test/docker-compose-sync.yml -f test/docker-compose-sync-check.yml --env-file .env up + timeout-minutes: 5 + - name: Check database check results + if: always() + run: | + mv logs/sync.log logs/${{ env.TEST_RUNNER }}_sync_databasecheck.log && \ + [[ -z "$(grep -e ERROR -e WARNING logs/${{ env.TEST_RUNNER }}_sync_databasecheck.log)" ]] + + # End testing + - name: Cleanup testing environment + uses: ./.github/actions/cleanup-environment + if: always() + with: + TEST_RUNNER: ${{ env.TEST_RUNNER }} + REPO_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + + - name: Upload log files + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: | + logs/ + data/${{ env.TEST_RUNNER }}_syncState.db diff --git a/.github/workflows/sonarcloud-analysis.yml b/.github/workflows/sonarcloud-analysis.yml new file mode 100644 index 0000000..e20ce47 --- /dev/null +++ b/.github/workflows/sonarcloud-analysis.yml @@ -0,0 +1,22 @@ +name: SonarCloud + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + sonarcloud: + name: SonarCloud + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.gitignore b/.gitignore index 4336146..fa452a3 100644 --- a/.gitignore +++ b/.gitignore @@ -124,12 +124,10 @@ dmypy.json .pyre/ # Dev files -.vscode/** -token.pickle +*.pickle credentials.json +.vscode conf.py -token.pickle* -syncState.db -syncState.db-journal -GoogleSampleData.json -MonicaSampleData.json +*.db +*.db-journal +*.bat diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..011f6c8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,29 @@ +{ + "python.linting.pylintEnabled": false, + "python.linting.enabled": true, + "python.languageServer": "Pylance", + "python.formatting.provider": "black", + "python.formatting.blackArgs": [ + "-l 105" + ], + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=105", + "--ignore=E203,W503,E231,E402" + ], + "python.linting.banditArgs": [ + "--skip B403,B101,B301" + ], + "python.linting.banditEnabled": true, + "python.sortImports.args": [ + "--profile black" + ], + "python.linting.mypyArgs": [ + "--follow-imports=silent", + "--ignore-missing-imports", + "--show-column-numbers", + "--no-pretty", + "--install-types" + ], + "python.linting.mypyEnabled": true, +} diff --git a/DatabaseHelper.py b/DatabaseHelper.py deleted file mode 100644 index 0bfcf4a..0000000 --- a/DatabaseHelper.py +++ /dev/null @@ -1,170 +0,0 @@ -import sqlite3 -from datetime import datetime -from logging import Logger -from typing import Tuple, Union, List - - -class DatabaseEntry(): - '''Creates a database row object for inserting into the database. - Needs at least a Monica id AND a Google id.''' - - def __init__(self, googleId: str, monicaId: str, googleFullName: str = 'NULL', - monicaFullName: str = 'NULL', googleLastChanged: str = 'NULL', - monicaLastChanged: str = 'NULL') -> None: - insertSql = ''' - INSERT INTO sync(googleId, monicaId, googleFullName, monicaFullName, - googleLastChanged, monicaLastChanged) - VALUES(?,?,?,?,?,?) - ''' - self.insertStatement = insertSql, (googleId, str(monicaId), - googleFullName, monicaFullName, - googleLastChanged, monicaLastChanged) - - def getInsertStatement(self) -> Tuple[str, tuple]: - return self.insertStatement - -class Database(): - '''Handles all database related stuff.''' - - def __init__(self, log: Logger, filename: str) -> None: - self.log = log - # pylint: disable=no-member - self.connection = sqlite3.connect(filename) - self.cursor = self.connection.cursor() - self.__initializeDatabase() - - def deleteAndInitialize(self) -> None: - '''Deletes all tables from the database and creates new ones.''' - deleteSyncTableSql = ''' - DROP TABLE IF EXISTS sync; - ''' - deleteConfigTableSql = ''' - DROP TABLE IF EXISTS config; - ''' - self.cursor.execute(deleteSyncTableSql) - self.cursor.execute(deleteConfigTableSql) - self.connection.commit() - self.__initializeDatabase() - - def __initializeDatabase(self): - """Initializes the database with all tables.""" - createSyncTableSql = ''' - CREATE TABLE IF NOT EXISTS sync ( - googleId VARCHAR(50) NOT NULL UNIQUE, - monicaId VARCHAR(10) NOT NULL UNIQUE, - googleFullName VARCHAR(50) NULL, - monicaFullName VARCHAR(50) NULL, - googleLastChanged DATETIME NULL, - monicaLastChanged DATETIME NULL); - ''' - createConfigTableSql = ''' - CREATE TABLE IF NOT EXISTS config ( - googleNextSyncToken VARCHAR(100) NULL UNIQUE, - tokenLastUpdated DATETIME NULL); - ''' - self.cursor.execute(createSyncTableSql) - self.cursor.execute(createConfigTableSql) - self.connection.commit() - - def insertData(self, databaseEntry: DatabaseEntry) -> None: - '''Inserts the given data into the database.''' - self.cursor.execute(*databaseEntry.getInsertStatement()) - self.connection.commit() - - def update(self, googleId: str = None, monicaId: str = None, - googleFullName: str = None, monicaFullName: str = None, - googleLastChanged: str = None, monicaLastChanged: str = None) -> None: - '''Updates a dataset in the database. Needs at least a Monica id OR a Google id and the related data.''' - UNKNOWN_ARGUMENTS = "Unknown database update arguments!" - if monicaId: - if monicaFullName: - self.__updateFullNameByMonicaId(str(monicaId), monicaFullName) - if monicaLastChanged: - self.__updateMonicaLastChanged(str(monicaId), monicaLastChanged) - else: - self.log.error(UNKNOWN_ARGUMENTS) - if googleId: - if googleFullName: - self.__updateFullNameByGoogleId(googleId, googleFullName) - if googleLastChanged: - self.__updateGoogleLastChanged(googleId, googleLastChanged) - else: - self.log.error(UNKNOWN_ARGUMENTS) - if not monicaId and not googleId: - self.log.error(UNKNOWN_ARGUMENTS) - - def __updateFullNameByMonicaId(self, monicaId: str, monicaFullName: str) -> None: - insertSql = "UPDATE sync SET monicaFullName = ? WHERE monicaId = ?" - self.cursor.execute(insertSql, (monicaFullName, str(monicaId))) - self.connection.commit() - - def __updateFullNameByGoogleId(self, googleId: str, googleFullName: str) -> None: - insertSql = "UPDATE sync SET googleFullName = ? WHERE googleId = ?" - self.cursor.execute(insertSql, (googleFullName, googleId)) - self.connection.commit() - - def __updateMonicaLastChanged(self, monicaId: str, monicaLastChanged: str) -> None: - insertSql = "UPDATE sync SET monicaLastChanged = ? WHERE monicaId = ?" - self.cursor.execute(insertSql, (monicaLastChanged, str(monicaId))) - self.connection.commit() - - def __updateGoogleLastChanged(self, googleId: str, googleLastChanged: str) -> None: - insertSql = "UPDATE sync SET googleLastChanged = ? WHERE googleId = ?" - self.cursor.execute(insertSql, (googleLastChanged, googleId)) - self.connection.commit() - - def findById(self, googleId: str = None, monicaId: str = None) -> Union[tuple, None]: - '''Search for a contact row in the database. Returns None if not found. - Needs Google id OR Monica id''' - if monicaId: - row = self.__findByMonicaId(str(monicaId)) - elif googleId: - row = self.__findByGoogleId(googleId) - else: - self.log.error("Unknown database find arguments!") - if row: - gId, mId, googleFullName, monicaFullName, googleLastChanged, monicaLastChanged = row - return gId, str(mId), googleFullName, monicaFullName, googleLastChanged, monicaLastChanged - return None - - def getIdMapping(self) -> dict: - '''Returns a dictionary with the {monicaId:googleId} mapping from the database''' - findSql = "SELECT googleId,monicaId FROM sync" - self.cursor.execute(findSql) - mapping = {googleId: str(monicaId) - for googleId, monicaId in self.cursor.fetchall()} - return mapping - - def __findByMonicaId(self, monicaId: str) -> List[tuple]: - findSql = "SELECT * FROM sync WHERE monicaId=?" - self.cursor.execute(findSql, (str(monicaId),)) - return self.cursor.fetchone() - - def __findByGoogleId(self, googleId: str) -> List[tuple]: - findSql = "SELECT * FROM sync WHERE googleId=?" - self.cursor.execute(findSql, (googleId,)) - return self.cursor.fetchone() - - def delete(self, googleId: str, monicaId: str) -> None: - '''Deletes a row from the database. Needs Monica id AND Google id.''' - deleteSql = "DELETE FROM sync WHERE monicaId=? AND googleId=?" - self.cursor.execute(deleteSql, (str(monicaId), googleId)) - self.connection.commit() - - def getGoogleNextSyncToken(self) -> Union[str, None]: - '''Returns the next sync token.''' - findSql = "SELECT * FROM config WHERE ROWID=1" - self.cursor.execute(findSql) - row = self.cursor.fetchone() - if row: - return row[0] - return None - - def updateGoogleNextSyncToken(self, token: str) -> None: - '''Updates the given token in the database.''' - timestamp = datetime.now().strftime('%F %H:%M:%S') - deleteSql = "DELETE FROM config WHERE ROWID=1" - insertSql = "INSERT INTO config(googleNextSyncToken, tokenLastUpdated) VALUES(?,?)" - self.cursor.execute(deleteSql) - self.cursor.execute(insertSql, (token, timestamp)) - self.connection.commit() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c00ede2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# Deriving the latest base image from https://hub.docker.com/_/python +FROM python:alpine + +# Labels +LABEL Maintainer="antonplagemann" + +# Choose working directory +WORKDIR /usr/app + +# Copy all files to working dir +COPY . . + +# Add data and logs volume +VOLUME /usr/app/data +VOLUME /usr/app/logs + +# Install dependencies +RUN python -m pip install --upgrade pip && pip install -r requirements.txt + +# Creates a non-root user with an explicit UID and adds permission to access the /app folder +# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /usr/app +USER appuser diff --git a/GMSync.py b/GMSync.py index 75b9c00..6d4d749 100644 --- a/GMSync.py +++ b/GMSync.py @@ -1,111 +1,284 @@ -# pylint: disable=import-error import argparse import logging +import os import sys +from os.path import abspath, join -try: - from conf import (BASE_URL, CREATE_REMINDERS, DELETE_ON_SYNC, FIELDS, - GOOGLE_LABELS, MONICA_LABELS, STREET_REVERSAL, TOKEN) -except ImportError: - print("\nFailed to import config settings!\n" \ - "Please verify that you have the latest version of the conf.py file " \ - "available on GitHub and check for possible typos!") - sys.exit(1) - -from DatabaseHelper import Database -from GoogleHelper import Google -from MonicaHelper import Monica -from SyncHelper import Sync - -VERSION = "v3.2.1" -DATABASE_FILENAME = "syncState.db" -LOG_FILENAME = 'Sync.log' +from dotenv import dotenv_values, find_dotenv # type: ignore +from dotenv.main import set_key # type: ignore + +from helpers.ConfigHelper import Config +from helpers.DatabaseHelper import Database +from helpers.Exceptions import ConfigError +from helpers.GoogleHelper import Google +from helpers.MonicaHelper import Monica +from helpers.SyncHelper import Sync + +VERSION = "v4.0.0" +LOG_FOLDER = "logs" +LOG_FILENAME = "sync.log" +DEFAULT_CONFIG_FILEPATH = join("helpers", ".env.default") # Google -> Monica contact syncing script # Make sure you installed all requirements using 'pip install -r requirements.txt' -# Get module specific logger -log = logging.getLogger('GMSync') +class GMSync: + def main(self) -> None: + try: + # Create logger + self.create_logger() + self.log.info(f"Script started ({VERSION})") -def main() -> None: - try: - # Setup argument parser - parser = argparse.ArgumentParser(description='Syncs Google contacts to a Monica instance.') - parser.add_argument('-i', '--initial', action='store_true', - required=False, help="build the syncing database and do a full sync") - parser.add_argument('-sb', '--syncback', action='store_true', - required=False, help="sync new Monica contacts back to Google. " \ - "Can be combined with other arguments") - parser.add_argument('-d', '--delta', action='store_true', - required=False, help="do a delta sync of new or changed Google contacts") - parser.add_argument('-f', '--full', action='store_true', - required=False, help="do a full sync and request a new delta sync token") - parser.add_argument('-c', '--check', action='store_true', - required=False, help="check database consistency and report all errors. " \ - "Can be combined with other arguments") + # Create argument parser + self.create_argument_parser() - # Parse arguments - args = parser.parse_args() + # Convert environment if requested (-u) + if self.args.update: + self.update_environment() + + # Load config + self.load_config() + + # Create sync object + self.create_sync_helper() - # Logging configuration + # Print chosen sync arguments (optional ones first) + print("\nYour choice (unordered):") + if self.args.syncback: + print("- sync back") + if self.args.check: + print("- database check") + + # Start + if self.args.initial: + # Start initial sync (-i) + print("- initial sync\n") + self.sync.start_sync("initial") + elif self.args.delta: + # Start initial sync (-d) + print("- delta sync\n") + self.sync.start_sync("delta") + elif self.args.full: + # Start initial sync (-f) + print("- full sync\n") + self.sync.start_sync("full") + elif self.args.syncback: + # Start sync back from Monica to Google (-sb) + print("") + self.sync.start_sync("syncBack") + elif self.args.check: + # Start database error check (-c) + print("") + self.sync.check_database() + elif not self.args.update: + # Wrong arguments + print("Unknown sync arguments, check your input!\n") + self.parser.print_help() + sys.exit(2) + + # It's over now + self.log.info("Script ended\n") + + except Exception as e: + self.log.exception(e) + self.log.info("Script aborted") + print(f"\nScript aborted: {type(e).__name__}: {str(e)}") + print(f"See log file ({join(LOG_FOLDER, LOG_FILENAME)}) for all details") + raise SystemExit(1) from e + + def create_logger(self) -> None: + """Creates the logger object""" + # Set logging configuration + if not os.path.exists(LOG_FOLDER): + os.makedirs(LOG_FOLDER) + log = logging.getLogger("GMSync") + dotenv_log = logging.getLogger("dotenv.main") log.setLevel(logging.INFO) - loggingFormat = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - handler = logging.FileHandler(filename=LOG_FILENAME, mode='a', encoding="utf8") + logging_format = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + log_filepath = join(LOG_FOLDER, LOG_FILENAME) + handler = logging.FileHandler(filename=log_filepath, mode="a", encoding="utf8") handler.setLevel(logging.INFO) - handler.setFormatter(loggingFormat) + handler.setFormatter(logging_format) log.addHandler(handler) - log.info(f"Script started ({VERSION})") - - # Create sync object - database = Database(log, DATABASE_FILENAME) - google = Google(log, database, GOOGLE_LABELS) - monica = Monica(log, database, TOKEN, BASE_URL, CREATE_REMINDERS, MONICA_LABELS) - sync = Sync(log, database, monica, google, args.syncback, args.check, - DELETE_ON_SYNC, STREET_REVERSAL, FIELDS) - - # Print chosen sync arguments (optional ones first) - print("\nYour choice (unordered):") - if args.syncback: - print("- sync back") - if args.check: - print("- database check") - - # Start - if args.initial: - # Start initial sync - print("- initial sync\n") - sync.startSync('initial') - elif args.delta: - # Start initial sync - print("- delta sync\n") - sync.startSync('delta') - elif args.full: - # Start initial sync - print("- full sync\n") - sync.startSync('full') - elif args.syncback: - # Start sync back from Monica to Google - print("") - sync.startSync('syncBack') - elif args.check: - # Start database error check - print("") - sync.checkDatabase() + dotenv_log.addHandler(handler) + self.log = log + + def create_argument_parser(self) -> None: + """Creates the argument parser object""" + # Setup argument parser + parser = argparse.ArgumentParser(description="Syncs Google contacts to a Monica instance.") + parser.add_argument( + "-i", + "--initial", + action="store_true", + required=False, + help="build the syncing database and do a full sync", + ) + parser.add_argument( + "-d", + "--delta", + action="store_true", + required=False, + help="do a delta sync of new or changed Google contacts", + ) + parser.add_argument( + "-f", + "--full", + action="store_true", + required=False, + help="do a full sync and request a new delta sync token", + ) + parser.add_argument( + "-sb", + "--syncback", + action="store_true", + required=False, + help="sync new Monica contacts back to Google. Can be combined with other arguments", + ) + parser.add_argument( + "-c", + "--check", + action="store_true", + required=False, + help="check database consistency and report all errors. " + "Can be combined with other arguments", + ) + parser.add_argument( + "-e", "--env-file", type=str, required=False, help="custom path to your .env config file" + ) + parser.add_argument( + "-u", + "--update", + action="store_true", + required=False, + help="Updates the environment files from 3.x to v4.x scheme", + ) + + # Parse arguments + self.parser = parser + self.args = parser.parse_args() + + def load_config(self) -> None: + """Loads the config from file or environment variables""" + # Load default config + self.log.info("Loading config (last value wins)") + default_config = find_dotenv(DEFAULT_CONFIG_FILEPATH, raise_error_if_not_found=True) + self.log.info(f"Loading default config from {default_config}") + default_config_values = dotenv_values(default_config) + if self.args.env_file: + if not os.path.exists(self.args.env_file): + raise ConfigError("Could not find the custom user config file, check your input!") + # Load config from custom path + user_config = abspath(self.args.env_file) else: - # Wrong arguments - print("Unknown sync arguments, check your input!\n") - parser.print_help() - sys.exit(2) + # Search config path + user_config = find_dotenv() + if user_config: + # Load config from file + self.log.info(f"Loading file config from {user_config}") + file_config_values = dotenv_values(user_config) + else: + file_config_values = {} + + # Load config from environment vars + self.log.info("Loading os environment config") + environment_config_values = dict(os.environ) + self.log.info("Config loading complete") + raw_config = {**default_config_values, **file_config_values, **environment_config_values} + + # Parse config + self.conf = Config(self.log, raw_config) + self.log.info("Config successfully parsed") + + def create_sync_helper(self) -> None: + """Creates the main sync class object""" + # Create sync objects + database = Database(self.log, self.conf.DATABASE_FILE) + google = Google( + self.log, + database, + self.conf.GOOGLE_CREDENTIALS_FILE, + self.conf.GOOGLE_TOKEN_FILE, + self.conf.GOOGLE_LABELS_INCLUDE, + self.conf.GOOGLE_LABELS_EXCLUDE, + self.args.initial, + ) + monica = Monica( + self.log, + database, + self.conf.TOKEN, + self.conf.BASE_URL, + self.conf.CREATE_REMINDERS, + self.conf.MONICA_LABELS_INCLUDE, + self.conf.MONICA_LABELS_EXCLUDE, + ) + self.sync = Sync( + self.log, + database, + monica, + google, + self.args.syncback, + self.args.check, + self.conf.DELETE_ON_SYNC, + self.conf.STREET_REVERSAL, + self.conf.FIELDS, + ) + + def update_environment(self): + """Updates the config and other environment files to work with v4.x.x""" + self.log.info("Start updating environment") + + # Make 'data' folder + if not os.path.exists("data"): + os.makedirs("data") + msg = "'data' folder created" + self.log.info(msg) + print(msg) + + # Convert config to '.env' file + env_file = ".env" + open(env_file, "w").close() + from conf import ( # type: ignore + BASE_URL, + CREATE_REMINDERS, + DELETE_ON_SYNC, + FIELDS, + GOOGLE_LABELS, + MONICA_LABELS, + STREET_REVERSAL, + TOKEN, + ) + + set_key(env_file, "TOKEN", TOKEN) + set_key(env_file, "BASE_URL", BASE_URL) + set_key(env_file, "CREATE_REMINDERS", str(CREATE_REMINDERS)) + set_key(env_file, "DELETE_ON_SYNC", str(DELETE_ON_SYNC)) + set_key(env_file, "STREET_REVERSAL", str(STREET_REVERSAL)) + set_key(env_file, "FIELDS", ",".join([field for field, is_true in FIELDS.items() if is_true])) + set_key(env_file, "GOOGLE_LABELS_INCLUDE", ",".join(GOOGLE_LABELS["include"])) + set_key(env_file, "GOOGLE_LABELS_EXCLUDE", ",".join(GOOGLE_LABELS["exclude"])) + set_key(env_file, "MONICA_LABELS_INCLUDE", ",".join(MONICA_LABELS["include"])) + set_key(env_file, "MONICA_LABELS_EXCLUDE", ",".join(MONICA_LABELS["exclude"])) + msg = "'.env' file created, old 'conf.py' can be deleted now" + self.log.info(msg) + print(msg) - # Its over now - log.info("Script ended\n") + # Move token, credentials and database inside new 'data' folder + files = ["syncState.db", "token.pickle", "credentials.json"] + for filename in files: + try: + os.rename(filename, f"data/{filename}") + msg = f"'{filename}' moved to 'data/{filename}'" + self.log.info(msg) + print(msg) + except FileNotFoundError: + msg = f"Could not move {filename}, file not found!" + print(msg) + self.log.warning(msg) - except Exception as e: - msg = f"Script aborted: {type(e).__name__}: {str(e)}\n" - log.exception(e) - print("\n" + msg) - raise SystemExit(1) from e + # Finished + self.log.info("Finished updating environment") -if __name__ == '__main__': - main() +if __name__ == "__main__": + GMSync().main() diff --git a/GoogleHelper.py b/GoogleHelper.py deleted file mode 100644 index c1fe584..0000000 --- a/GoogleHelper.py +++ /dev/null @@ -1,492 +0,0 @@ -import os.path -import pickle -import sys -from logging import Logger -from typing import List, Tuple, Union -import time - -from google.auth.transport.requests import Request -from google_auth_oauthlib.flow import InstalledAppFlow -from googleapiclient.discovery import Resource, build -from googleapiclient.errors import HttpError - -from DatabaseHelper import Database - - -class Google(): - '''Handles all Google related (api) stuff.''' - - def __init__(self, log: Logger, databaseHandler: Database = None, - labelFilter: dict = None) -> None: - self.log = log - self.labelFilter = labelFilter or {"include": [], "exclude": []} - self.database = databaseHandler - self.apiRequests = 0 - self.service = self.__buildService() - self.labelMapping = self.__getLabelMapping() - self.reverseLabelMapping = {labelId: name for name, labelId in self.labelMapping.items()} - self.contacts = [] - self.dataAlreadyFetched = False - self.createdContacts = {} - self.syncFields = 'addresses,biographies,birthdays,emailAddresses,genders,' \ - 'memberships,metadata,names,nicknames,occupations,organizations,phoneNumbers' - self.updateFields = 'addresses,biographies,birthdays,clientData,emailAddresses,' \ - 'events,externalIds,genders,imClients,interests,locales,locations,memberships,' \ - 'miscKeywords,names,nicknames,occupations,organizations,phoneNumbers,relations,' \ - 'sipAddresses,urls,userDefined' - - def __buildService(self) -> Resource: - creds = None - FILENAME = 'token.pickle' - # The file token.pickle stores the user's access and refresh tokens, and is - # created automatically when the authorization flow completes for the first - # time. - if os.path.exists(FILENAME): - with open(FILENAME, 'rb') as token: - creds = pickle.load(token) - # If there are no (valid) credentials available, let the user log in. - if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - creds.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - 'credentials.json', scopes='https://www.googleapis.com/auth/contacts') - creds = flow.run_local_server(port=56411) - # Save the credentials for the next run - with open(FILENAME, 'wb') as token: - pickle.dump(creds, token) - - service = build('people', 'v1', credentials=creds) - return service - - def getLabelId(self, name:str, createOnError:bool = True) -> str: - '''Returns the Google label id for a given tag name. - Creates a new label if it has not been found.''' - if createOnError: - return self.labelMapping.get(name, self.createLabel(name)) - else: - return self.labelMapping.get(name, '') - - def getLabelName(self, labelString: str) -> str: - '''Returns the Google label name for a given label id.''' - labelId = labelString.split("/")[1] - return self.reverseLabelMapping.get(labelString, labelId) - - def __filterContactsByLabel(self, contactList: List[dict]) -> List[dict]: - '''Filters a contact list by include/exclude labels.''' - if self.labelFilter["include"]: - return [contact for contact in contactList - if any([contactLabel["contactGroupMembership"]["contactGroupId"] - in self.labelFilter["include"] - for contactLabel in contact["memberships"]]) - and all([contactLabel["contactGroupMembership"]["contactGroupId"] - not in self.labelFilter["exclude"] - for contactLabel in contact["memberships"]])] - elif self.labelFilter["exclude"]: - return [contact for contact in contactList - if all([contactLabel["contactGroupMembership"]["contactGroupId"] - not in self.labelFilter["exclude"] - for contactLabel in contact["memberships"]])] - else: - return contactList - - def __filterUnnamedContacts(self, contactList: List[dict]) -> List[dict]: - '''Exclude contacts without name.''' - filteredContactList = [] - for googleContact in contactList: - # Look for empty names but keep deleted contacts (they too don't have a name) - isDeleted = googleContact.get('metadata', {}).get('deleted', False) - isAnyName = any(self.getContactNames(googleContact)) - isNameKeyPresent = googleContact.get('names', False) - if (not isAnyName or not isNameKeyPresent) and not isDeleted: - self.log.info(f"Skipped the following unnamed google contact during sync:") - self.log.info(f"Contact details:\n{self.getContactAsString(googleContact)[2:-1]}") - else: - filteredContactList.append(googleContact) - if len(filteredContactList) != len(contactList): - print("\nSkipped one or more unnamed google contacts, see log for details") - - return filteredContactList - - def getContactNames(self, googleContact: dict) -> Tuple[str, str, str, str, str, str]: - '''Returns the given, family and display name of a Google contact.''' - names = googleContact.get('names', [{}])[0] - givenName = names.get("givenName", '') - familyName = names.get("familyName", '') - displayName = names.get("displayName", '') - middleName = names.get("middleName", '') - prefix = names.get("honorificPrefix", '') - suffix = names.get("honorificSuffix", '') - nickname = googleContact.get('nicknames', [{}])[0].get('value', '') - return givenName, middleName, familyName, displayName, prefix, suffix, nickname - - def getContactAsString(self, googleContact: dict) -> str: - '''Get some content from a Google contact to identify it as a user and return it as string.''' - string = f"\n\nContact id:\t{googleContact['resourceName']}\n" - for obj in googleContact.get('names', []): - for key, value in obj.items(): - if key == 'displayName': - string += f"Display name:\t{value}\n" - for obj in googleContact.get('birthdays', []): - for key, value in obj.items(): - if key == 'value': - string += f"Birthday:\t{value}\n" - for obj in googleContact.get('organizations', []): - for key, value in obj.items(): - if key == 'name': - string += f"Company:\t{value}\n" - if key == 'department': - string += f"Department:\t{value}\n" - if key == 'title': - string += f"Job title:\t{value}\n" - for obj in googleContact.get('addresses', []): - for key, value in obj.items(): - if key == 'formattedValue': - value = value.replace('\n', ' ') - string += f"Address:\t{value}\n" - for obj in googleContact.get('phoneNumbers', []): - for key, value in obj.items(): - if key == 'value': - string += f"Phone number:\t{value}\n" - for obj in googleContact.get('emailAddresses', []): - for key, value in obj.items(): - if key == 'value': - string += f"Email:\t\t{value}\n" - labels = [] - for obj in googleContact.get('memberships', []): - for key, value in obj.items(): - if key == 'contactGroupMembership': - name = self.getLabelName(value['contactGroupResourceName']) - labels.append(name) - if labels: - string += f"Labels:\t\t{', '.join(labels)}\n" - return string - - def removeContactFromList(self, googleContact: dict) -> None: - '''Removes a Google contact internally to avoid further processing - (e.g. if it has been deleted on both sides)''' - self.contacts.remove(googleContact) - - def getContact(self, googleId: str) -> dict: - '''Fetches a single contact by id from Google.''' - try: - # Check if contact is already fetched - if self.contacts: - googleContactList = [c for c in self.contacts if str(c['resourceName']) == str(googleId)] - if googleContactList: - return googleContactList[0] - - # Build GET parameters - parameters = { - 'resourceName': googleId, - 'personFields': self.syncFields, - } - - # Fetch contact - # pylint: disable=no-member - result = self.service.people().get(**parameters).execute() - self.apiRequests += 1 - - # Return contact - googleContact = self.__filterContactsByLabel([result])[0] - googleContact = self.__filterUnnamedContacts([result])[0] - self.contacts.append(googleContact) - return googleContact - - except HttpError as error: - if self.__isSlowDownError(error): - return self.getContact(googleId) - else: - msg = f"Failed to fetch Google contact '{googleId}': {str(error)}" - self.log.error(msg) - raise Exception(msg) from error - - except IndexError as error: - msg = f"Contact processing of '{googleId}' not allowed by label filter" - self.log.info(msg) - raise Exception(msg) from error - - except Exception as error: - msg = f"Failed to fetch Google contact '{googleId}': {str(error)}" - self.log.error(msg) - raise Exception(msg) from error - - def __isSlowDownError(self, error: HttpError) -> bool: - '''Checks if the error is an qoate exceeded error and slows down the requests if yes.''' - WAITING_TIME = 60 - if "Quota exceeded" in str(error): - print(f"\nToo many Google requests, waiting {WAITING_TIME} seconds...") - time.sleep(WAITING_TIME) - return True - else: - return False - - def getContacts(self, refetchData : bool = False, **params) -> List[dict]: - '''Fetches all contacts from Google if not already fetched.''' - # Build GET parameters - parameters = {'resourceName': 'people/me', - 'pageSize': 1000, - 'personFields': self.syncFields, - 'requestSyncToken': True, - **params} - - # Avoid multiple fetches - if self.dataAlreadyFetched and not refetchData: - return self.contacts - - # Start fetching - msg = "Fetching Google contacts..." - self.log.info(msg) - sys.stdout.write(f"\r{msg}") - sys.stdout.flush() - try: - self.__fetchContacts(parameters) - except HttpError as error: - if 'Sync token' in str(error): - msg = "Sync token expired or invalid. Fetching again without token (full sync)..." - self.log.warning(msg) - print("\n" + msg) - parameters.pop('syncToken') - self.__fetchContacts(parameters) - elif self.__isSlowDownError(error): - return self.getContacts(refetchData, **params) - else: - msg = "Failed to fetch Google contacts!" - self.log.error(msg) - raise Exception(str(error)) from error - msg = "Finished fetching Google contacts" - self.log.info(msg) - print("\n" + msg) - self.dataAlreadyFetched = True - return self.contacts - - def __fetchContacts(self, parameters: dict) -> None: - contacts = [] - while True: - # pylint: disable=no-member - result = self.service.people().connections().list(**parameters).execute() - self.apiRequests += 1 - nextPageToken = result.get('nextPageToken', False) - contacts += result.get('connections', []) - if nextPageToken: - parameters['pageToken'] = nextPageToken - else: - self.contacts = self.__filterContactsByLabel(contacts) - self.contacts = self.__filterUnnamedContacts(contacts) - break - - nextSyncToken = result.get('nextSyncToken', None) - if nextSyncToken and self.database: - self.database.updateGoogleNextSyncToken(nextSyncToken) - - def __getLabelMapping(self) -> dict: - '''Fetches all contact groups from Google (aka labels) and - returns a {name: id} mapping.''' - try: - # Get all contact groups - # pylint: disable=no-member - response = self.service.contactGroups().list().execute() - self.apiRequests += 1 - groups = response.get('contactGroups', []) - - # Initialize mapping for all user groups and allowed system groups - labelMapping = {group['name']: group['resourceName'] for group in groups - if group['groupType'] == 'USER_CONTACT_GROUP' - or group['name'] in ['myContacts', 'starred']} - - return labelMapping - except HttpError as error: - if self.__isSlowDownError(error): - return self.__getLabelMapping() - else: - msg = "Failed to fetch Google labels!" - self.log.error(msg) - raise Exception(str(error)) from error - - def deleteLabel(self, groupId) -> None: - '''Deletes a contact group from Google (aka label). Does not delete assigned contacts.''' - try: - # pylint: disable=no-member - response = self.service.contactGroups().delete(resourceName=groupId).execute() - self.apiRequests += 1 - except HttpError as error: - if self.__isSlowDownError(error): - self.deleteLabel(groupId) - else: - reason = str(error) - msg = f"Failed to delete Google contact group. Reason: {reason}" - self.log.warning(msg) - print("\n" + msg) - raise Exception(reason) from error - - if response: - msg = f"Non-empty response received, please check carefully: {response}" - self.log.warning(msg) - print("\n" + msg) - - def createLabel(self, labelName: str) -> str: - '''Creates a new Google contacts label and returns its id.''' - # Search label and return if found - if labelName in self.labelMapping: - return self.labelMapping[labelName] - - # Create group object - newGroup = { - "contactGroup": { - "name": labelName - } - } - - try: - # Upload group object - # pylint: disable=no-member - response = self.service.contactGroups().create(body=newGroup).execute() - self.apiRequests += 1 - - groupId = response.get('resourceName', 'contactGroups/myContacts') - self.labelMapping.update({labelName: groupId}) - return groupId - - except HttpError as error: - if self.__isSlowDownError(error): - return self.createLabel(labelName) - else: - msg = "Failed to create Google label!" - self.log.error(msg) - raise Exception(str(error)) from error - - def createContact(self, data) -> Union[dict, None]: - '''Creates a given Google contact via api call and returns the created contact.''' - # Upload contact - try: - # pylint: disable=no-member - result = self.service.people().createContact(personFields=self.syncFields, body=data).execute() - self.apiRequests += 1 - except HttpError as error: - if self.__isSlowDownError(error): - return self.createContact(data) - else: - reason = str(error) - msg = f"'{data['names'][0]}':Failed to create Google contact. Reason: {reason}" - self.log.error(msg) - print("\n" + msg) - raise Exception(reason) from error - - # Process result - googleId = result.get('resourceName', '-') - name = result.get('names', [{}])[0].get('displayName', 'error') - self.createdContacts[googleId] = True - self.contacts.append(result) - self.log.info( - f"'{name}': Contact with id '{googleId}' created successfully") - return result - - def updateContact(self, data) -> Union[dict, None]: - '''Updates a given Google contact via api call and returns the created contact.''' - # Upload contact - try: - # pylint: disable=no-member - result = self.service.people().updateContact(resourceName=data['resourceName'], updatePersonFields=self.updateFields, body=data).execute() - self.apiRequests += 1 - except HttpError as error: - if self.__isSlowDownError(error): - return self.updateContact(data) - else: - reason = str(error) - msg = f"'{data['names'][0]}':Failed to update Google contact. Reason: {reason}" - self.log.warning(msg) - print("\n" + msg) - raise Exception(reason) from error - - # Process result - googleId = result.get('resourceName', '-') - name = result.get('names', [{}])[0].get('displayName', 'error') - self.log.info('Contact has not been saved internally!') - self.log.info( - f"'{name}': Contact with id '{googleId}' updated successfully") - return result - - -class GoogleContactUploadForm(): - '''Creates json form for creating Google contacts.''' - - def __init__(self, firstName: str = '', lastName: str = '', - middleName: str = '', birthdate: dict = {}, - phoneNumbers: List[str] = [], career: dict = {}, - emailAdresses: List[str] = [], labelIds: List[str] = [], - addresses: List[dict] = {}) -> None: - self.data = { - "names": [ - { - "familyName": lastName, - "givenName": firstName, - "middleName": middleName - } - ] - } - - if birthdate: - self.data["birthdays"] = [ - { - "date": { - "year": birthdate.get('year', 0), - "month": birthdate.get('month', 0), - "day": birthdate.get('day', 0) - } - } - ] - - if career: - self.data["organizations"] = [ - { - "name": career.get('company', ''), - "title": career.get('job', '') - } - ] - - if addresses: - self.data["addresses"] = [ - { - 'type': address.get("name",''), - "streetAddress": address.get('street', ''), - "city": address.get('city', ''), - "region": address.get('province', ''), - "postalCode": address.get('postal_code', ''), - "country": address["country"].get("name", None) if address["country"] else None, - "countryCode": address["country"].get("iso", None) if address["country"] else None, - } - for address in addresses - ] - - if phoneNumbers: - self.data["phoneNumbers"] = [ - { - "value": number, - "type": "other", - } - for number in phoneNumbers - ] - - if emailAdresses: - self.data["emailAddresses"] = [ - { - "value": email, - "type": "other", - } - for email in emailAdresses - ] - - if labelIds: - self.data["memberships"] = [ - { - "contactGroupMembership": - { - "contactGroupResourceName": labelId - } - } - for labelId in labelIds - ] - - def getData(self) -> dict: - '''Returns the Google contact form data.''' - return self.data diff --git a/MonicaHelper.py b/MonicaHelper.py deleted file mode 100644 index 10504c3..0000000 --- a/MonicaHelper.py +++ /dev/null @@ -1,567 +0,0 @@ -import sys -from logging import Logger -from typing import List -import time - -import requests -from requests.models import Response - -from DatabaseHelper import Database - - -class Monica(): - '''Handles all Monica related (api) stuff.''' - - def __init__(self, log: Logger, databaseHandler: Database, token: str, base_url: str, createReminders: bool, labelFilter: dict, sampleData: list = None) -> None: - self.log = log - self.database = databaseHandler - self.base_url = base_url - self.labelFilter = labelFilter - self.header = {'Authorization': f'Bearer {token}'} - self.parameters = {'limit': 100} - self.dataAlreadyFetched = False - self.contacts = [] - self.genderMapping = {} - self.contactFieldTypeMapping = {} - self.updatedContacts = {} - self.createdContacts = {} - self.deletedContacts = {} - self.apiRequests = 0 - self.createReminders = createReminders - - def __filterContactsByLabel(self, contactList: List[dict]) -> List[dict]: - '''Filters a contact list by include/exclude labels.''' - if self.labelFilter["include"]: - return [contact for contact in contactList - if any([contactLabel["name"] - in self.labelFilter["include"] - for contactLabel in contact["tags"]]) - and all([contactLabel["name"] - not in self.labelFilter["exclude"] - for contactLabel in contact["tags"]])] - elif self.labelFilter["exclude"]: - return [contact for contact in contactList - if all([contactLabel["name"] - not in self.labelFilter["exclude"] - for contactLabel in contact["tags"]])] - else: - return contactList - - def updateStatistics(self) -> None: - '''Updates internal statistics for printing.''' - # A contact should only count as updated if it has not been created during sync - self.updatedContacts = {key: value for key, value in self.updatedContacts.items() - if key not in self.createdContacts} - - def getGenderMapping(self) -> dict: - '''Fetches all genders from Monica and saves them to a dictionary.''' - # Only fetch if not present yet - if self.genderMapping: - return self.genderMapping - - while True: - # Get genders - response = requests.get( - self.base_url + f"/genders", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - genders = response.json()['data'] - genderMapping = {gender['type']: gender['id'] for gender in genders} - self.genderMapping = genderMapping - return self.genderMapping - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.error(f"Failed to fetch genders from Monica: {error}") - raise Exception("Error fetching genders from Monica!") - - def updateContact(self, monicaId: str, data: dict) -> None: - '''Updates a given contact and its id via api call.''' - name = f"{data['first_name']} {data['last_name']}" - - # Remove Monica contact from contact list (add again after updated) - self.contacts = [c for c in self.contacts if str(c['id']) != str(monicaId)] - - while True: - # Update contact - response = requests.put(self.base_url + f"/contacts/{monicaId}", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - contact = response.json()['data'] - self.updatedContacts[monicaId] = True - self.contacts.append(contact) - name = contact["complete_name"] - self.log.info(f"'{name}' ('{monicaId}'): Contact updated successfully") - self.database.update( - monicaId=monicaId, monicaLastChanged=contact['updated_at'], monicaFullName=contact["complete_name"]) - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.error(f"'{name}' ('{monicaId}'): Error updating Monica contact: {error}. Does it exist?") - self.log.error(f"Monica form data: {data}") - raise Exception("Error updating Monica contact!") - - def deleteContact(self, monicaId: str, name: str) -> None: - '''Deletes the contact with the given id from Monica and removes it from the internal list.''' - - while True: - # Delete contact - response = requests.delete( - self.base_url + f"/contacts/{monicaId}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.contacts = [c for c in self.contacts if str(c['id']) != str(monicaId)] - self.deletedContacts[monicaId] = True - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.error(f"'{name}' ('{monicaId}'): Failed to complete delete request: {error}") - raise Exception("Error deleting Monica contact!") - - def createContact(self, data: dict, referenceId: str) -> dict: - '''Creates a given Monica contact via api call and returns the created contact.''' - name = f"{data['first_name']} {data['last_name']}".strip() - - while True: - # Create contact - response = requests.post(self.base_url + f"/contacts", - headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 201: - contact = response.json()['data'] - self.createdContacts[contact['id']] = True - self.contacts.append(contact) - self.log.info(f"'{referenceId}' ('{contact['id']}'): Contact created successfully") - return contact - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.info(f"'{referenceId}': Error creating Monica contact: {error}") - raise Exception("Error creating Monica contact!") - - def getContacts(self) -> List[dict]: - '''Fetches all contacts from Monica if not already fetched.''' - try: - # Avoid multiple fetches - if self.dataAlreadyFetched: - return self.contacts - - # Start fetching - maxPage = '?' - page = 1 - contacts = [] - self.log.info("Fetching all Monica contacts...") - while True: - sys.stdout.write(f"\rFetching all Monica contacts (page {page} of {maxPage})") - sys.stdout.flush() - response = requests.get( - self.base_url + f"/contacts?page={page}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - # If successful - if response.status_code == 200: - data = response.json() - contacts += data['data'] - maxPage = data['meta']['last_page'] - if page == maxPage: - self.contacts = self.__filterContactsByLabel(contacts) - break - page += 1 - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - msg = f"Error fetching Monica contacts: {error}" - self.log.error(msg) - raise Exception(msg) - self.dataAlreadyFetched = True - msg = "Finished fetching Monica contacts" - self.log.info(msg) - print("\n" + msg) - return self.contacts - - except Exception as e: - msg = f"Failed to fetch Monica contacts (maybe connection issue): {str(e)}" - print("\n" + msg) - self.log.error(msg) - raise Exception(msg) - - def getContact(self, monicaId: str) -> dict: - '''Fetches a single contact by id from Monica.''' - try: - # Check if contact is already fetched - if self.contacts: - monicaContactList = [c for c in self.contacts if str(c['id']) == str(monicaId)] - if monicaContactList: - return monicaContactList[0] - - while True: - # Fetch contact - response = requests.get( - self.base_url + f"/contacts/{monicaId}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - monicaContact = response.json()['data'] - monicaContact = self.__filterContactsByLabel([monicaContact])[0] - self.contacts.append(monicaContact) - return monicaContact - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(error) - - except IndexError: - msg = f"Contact processing of '{monicaId}' not allowed by label filter" - self.log.info(msg) - raise Exception(msg) - - except Exception as e: - msg = f"Failed to fetch Monica contact '{monicaId}': {str(e)}" - self.log.error(msg) - raise Exception(msg) - - def getNotes(self, monicaId: str, name: str) -> List[dict]: - '''Fetches all contact notes for a given Monica contact id via api call.''' - - while True: - # Get contact fields - response = requests.get(self.base_url + f"/contacts/{monicaId}/notes", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - monicaNotes = response.json()['data'] - return monicaNotes - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error fetching Monica notes: {error}") - - def addNote(self, data: dict, name: str) -> None: - '''Creates a new note for a given contact id via api call.''' - # Initialization - monicaId = data['contact_id'] - - while True: - # Create address - response = requests.post(self.base_url + f"/notes", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 201: - self.updatedContacts[monicaId] = True - note = response.json()['data'] - noteId = note["id"] - self.log.info(f"'{name}' ('{monicaId}'): Note '{noteId}' created successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error creating Monica note: {error}") - - def updateNote(self, noteId: str, data: dict, name: str) -> None: - '''Creates a new note for a given contact id via api call.''' - # Initialization - monicaId = data['contact_id'] - - while True: - # Create address - response = requests.put(self.base_url + f"/notes/{noteId}", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - note = response.json()['data'] - noteId = note["id"] - self.log.info(f"'{name}' ('{monicaId}'): Note '{noteId}' updated successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error updating Monica note: {error}") - - def deleteNote(self, noteId: str, monicaId: str, name: str) -> None: - '''Creates a new note for a given contact id via api call.''' - - while True: - # Create address - response = requests.delete(self.base_url + f"/notes/{noteId}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - self.log.info(f"'{name}' ('{monicaId}'): Note '{noteId}' deleted successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error deleting Monica note: {error}") - - def removeTags(self, data: dict, monicaId: str, name: str) -> None: - '''Removes all tags given by id from a given contact id via api call.''' - - while True: - # Create address - response = requests.post(self.base_url + f"/contacts/{monicaId}/unsetTag", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - self.log.info(f"'{name}' ('{monicaId}'): Label(s) with id {data['tags']} removed successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error removing Monica labels: {error}") - - def addTags(self, data: dict, monicaId: str, name: str) -> None: - '''Adds all tags given by name for a given contact id via api call.''' - - while True: - # Create address - response = requests.post(self.base_url + f"/contacts/{monicaId}/setTags", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - self.log.info(f"'{name}' ('{monicaId}'): Labels {data['tags']} assigned successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error assigning Monica labels: {error}") - - - def updateCareer(self, monicaId: str, data: dict) -> None: - '''Updates job title and company for a given contact id via api call.''' - # Initialization - contact = self.getContact(monicaId) - name = contact['complete_name'] - - while True: - # Update contact - response = requests.put(self.base_url + f"/contacts/{monicaId}/work", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - contact = response.json()['data'] - self.log.info(f"'{name}' ('{monicaId}'): Company and job title updated successfully") - self.database.update(monicaId=monicaId, monicaLastChanged=contact['updated_at']) - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.warning(f"'{name}' ('{monicaId}'): Error updating Monica contact career info: {error}") - - def deleteAddress(self, addressId: str, monicaId: str, name: str) -> None: - '''Deletes an address for a given address id via api call.''' - while True: - # Delete address - response = requests.delete(self.base_url + f"/addresses/{addressId}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - self.log.info(f"'{name}' ('{monicaId}'): Address '{addressId}' deleted successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error deleting address '{addressId}': {error}") - - def createAddress(self, data: dict, name: str) -> None: - '''Creates an address for a given contact id via api call.''' - # Initialization - monicaId = data['contact_id'] - - while True: - # Create address - response = requests.post(self.base_url + f"/addresses", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 201: - self.updatedContacts[monicaId] = True - address = response.json()['data'] - addressId = address["id"] - self.log.info(f"'{name}' ('{monicaId}'): Address '{addressId}' created successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error creating Monica address: {error}") - - def getContactFields(self, monicaId: str, name: str) -> List[dict]: - '''Fetches all contact fields (phone numbers, emails, etc.) - for a given Monica contact id via api call.''' - - while True: - # Get contact fields - response = requests.get(self.base_url + f"/contacts/{monicaId}/contactfields", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - fieldList = response.json()['data'] - return fieldList - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error fetching Monica contact fields: {error}") - - def getContactFieldId(self, typeName: str) -> str: - '''Returns the id for a Monica contact field.''' - # Fetch if not present yet - if not self.contactFieldTypeMapping: - self.__getContactFieldTypes() - - # Get contact field id - fieldId = self.contactFieldTypeMapping.get(typeName, None) - - # No id is a serious issue - if not fieldId: - raise Exception(f"Could not find an id for contact field type '{typeName}'") - - return fieldId - - def __getContactFieldTypes(self) -> dict: - '''Fetches all contact field types from Monica and saves them to a dictionary.''' - - while True: - # Get genders - response = requests.get( - self.base_url + f"/contactfieldtypes", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - contactFieldTypes = response.json()['data'] - contactFieldTypeMapping = {field['type']: field['id'] for field in contactFieldTypes} - self.contactFieldTypeMapping = contactFieldTypeMapping - return self.contactFieldTypeMapping - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - self.log.error(f"Failed to fetch contact field types from Monica: {error}") - raise Exception("Error fetching contact field types from Monica!") - - - def createContactField(self, monicaId: str, data: dict, name: str) -> None: - '''Creates a contact field (phone number, email, etc.) - for a given Monica contact id via api call.''' - - while True: - # Create contact field - response = requests.post(self.base_url + f"/contactfields", headers=self.header, params=self.parameters, json=data) - self.apiRequests += 1 - - # If successful - if response.status_code == 201: - self.updatedContacts[monicaId] = True - contactField = response.json()['data'] - fieldId = contactField["id"] - typeDesc = contactField["contact_field_type"]["type"] - self.log.info(f"'{name}' ('{monicaId}'): Contact field '{fieldId}' ({typeDesc}) created successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error creating Monica contact field: {error}") - - def deleteContactField(self, fieldId: str, monicaId: str, name: str) -> None: - '''Updates a contact field (phone number, email, etc.) - for a given Monica contact id via api call.''' - - while True: - # Delete contact field - response = requests.delete(self.base_url + f"/contactfields/{fieldId}", headers=self.header, params=self.parameters) - self.apiRequests += 1 - - # If successful - if response.status_code == 200: - self.updatedContacts[monicaId] = True - self.log.info(f"'{name}' ('{monicaId}'): Contact field '{fieldId}' deleted successfully") - return - else: - error = response.json()['error']['message'] - if self.__isSlowDownError(response, error): - continue - raise Exception(f"'{name}' ('{monicaId}'): Error deleting Monica contact field '{fieldId}': {error}") - - def __isSlowDownError(self, response: Response, error: str) -> bool: - '''Checks if the error is an rate limiter error and slows down the requests if yes.''' - if "Too many attempts, please slow down the request" in error: - sec = int(response.headers.get('Retry-After')) - print(f"\nToo many Monica requests, waiting {sec} seconds...") - time.sleep(sec) - return True - else: - return False - -class MonicaContactUploadForm(): - '''Creates json form for creating or updating Monica contacts.''' - - def __init__(self, monica: Monica, firstName: str, lastName: str = None, nickName: str = None, - middleName: str = None, genderType: str = 'O', birthdateDay: str = None, - birthdateMonth: str = None, birthdateYear: str = None, - birthdateAgeBased: bool = False, isBirthdateKnown: bool = False, - isDeceased: bool = False, isDeceasedDateKnown: bool = False, - deceasedDay: int = None, deceasedMonth: int = None, - deceasedYear: int = None, deceasedAgeBased: bool = None, - createReminders: bool = True) -> None: - genderId = monica.getGenderMapping()[genderType] - self.data = { - "first_name": firstName, - "last_name": lastName, - "nickname": nickName, - "middle_name": middleName, - "gender_id": genderId, - "birthdate_day": birthdateDay, - "birthdate_month": birthdateMonth, - "birthdate_year": birthdateYear, - "birthdate_is_age_based": birthdateAgeBased, - "deceased_date_add_reminder": createReminders, - "birthdate_add_reminder": createReminders, - "is_birthdate_known": isBirthdateKnown, - "is_deceased": isDeceased, - "is_deceased_date_known": isDeceasedDateKnown, - "deceased_date_day": deceasedDay, - "deceased_date_month": deceasedMonth, - "deceased_date_year": deceasedYear, - "deceased_date_is_age_based": deceasedAgeBased, - } diff --git a/README.md b/README.md index 6fdc240..06d02ad 100644 --- a/README.md +++ b/README.md @@ -2,31 +2,57 @@ # Google to Monica contact syncing script -## Introduction - -This script does a contact syncing from a Google account to a [Monica](https://github.com/monicahq/monica) account. This script is intended for personal use only. It can contain all kinds of bugs, errors, and unhandled exceptions, so please do a backup before using it. It was programmed very carefully not to do bad things and delete everything, but it's your own risk trusting my words. - -That being said: Be welcome to use it, fork it, copy it for your own projects, and file issues for bug fixes or improvements. +🤖 Automated CI/CD Pipelines + +[![CodeQL](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/codeql-analysis.yml) +[![Docker CD](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/docker-cd.yml/badge.svg)](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/docker-cd.yml) +[![Python CI](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/python-ci.yml/badge.svg)](https://github.com/antonplagemann/GoogleMonicaSync/actions/workflows/python_ci.yml) +[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![spellcheck: codespell](https://img.shields.io/badge/spellcheck-codespell-brightgreen.svg)](https://github.com/codespell-project/codespell) +[![security: bandit](https://img.shields.io/badge/security-bandit-success.svg)](https://github.com/PyCQA/bandit) +[![linter: flake8](https://img.shields.io/badge/linter-flake8-brightgreen.svg)](https://github.com/PyCQA/flake8) +[![imports: isort](https://img.shields.io/badge/imports-isort-blue.svg)](https://pycqa.github.io/isort/) +[![mypy: checked](https://img.shields.io/badge/mypy-checked-blue.svg)](https://github.com/python/mypy) + +🔒 SonarCloud monitored + +[![SonarCloud Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Bugs](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=bugs)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Code Smells](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=code_smells)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Duplicated Lines (%)](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=duplicated_lines_density)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=ncloc)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=sqale_rating)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=sqale_index)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=antonplagemann_GoogleMonicaSync&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=antonplagemann_GoogleMonicaSync) + +## Hey 👋 + +Using [Monica](https://github.com/monicahq/monica) and Google Contacts, but you're annoyed by keeping your data in sync? Then this python script is for you 😎 +It does one-way ➡ contact syncing of your Google Contacts to a Monica account 🎉 +But wait, before trying something new, please do not forget to **make a full backup 🔒** before entering the first command 😉 +I did my best to write clean and working code, but if I missed a bug 🐛 that you've found, please return it to me 🏡😊, I would be happy to fix it 🙏! +All contributions are welcome! Feel free to open an issue or pull request 🙌 ## Features - One-way sync (Google → Monica) - Syncs the following details: first name, last name, middle name, nickname, birthday, job title, company, addresses, phone numbers, email addresses, labels (tags), notes (see [limits](#limits)) - Advanced matching of already present Monica contacts (e.g. from earlier contact import) -- User choice prompt before any modification to your Monica data during initial sync (you can choose to abort before the script makes any change). +- User choice prompt before any modification to your Monica data during initial sync (you can choose to abort before the script makes any change) - Fast delta sync using Google sync tokens - Optional sync-back (Monica → Google) of new Monica contacts that do not have a corresponding Google contact yet - Syncs the following details: first name, last name, middle name, birthday, job title, company, addresses, phone numbers, email addresses, labels (tags) -- Extensive logging of every change, warning, or error including the affected contact ID and name (File: `Sync.log`) +- Extensive logging of every change, warning, or error including the affected contact ID and name (file: `logs/sync.log`) ## Limits -- **Do not update [*synced*](#features) details at Monica.** As this is a one-way sync, it will overwrite all Monica changes to these fields! Of course, you can continue to use activities, notes, journal, relationships and almost any other Monica feature. Just don't update [name, birthday, job info, ...](#features) at Monica. +- **Do not update [*synced*](#features) details at Monica.** As this is a one-way ➡ sync, it will overwrite all Monica changes to these fields! Of course, you can continue to use activities, notes, journal, relationships and almost any other Monica feature. Just don't update [name, birthday, job info, ...](#features) at Monica. - **Do not delete synced contacts at Monica.** This will cause a sync error which you can resolve by doing initial sync again. -- Monica limits API usage to 60 calls per minute. As every contact needs *at least* 2 API calls, **the script can not sync more than 30 contacts per minute** (thus affecting primarily initial and full sync). -**UPDATE:** If you are using the hosted version of Monica you can [configure this rate limit](https://github.com/monicahq/monica/pull/5489) starting from Monica v3.3.0. -- Delta sync will fail if there are more than 7 days between the last sync (Google restriction). In this case, the script will automatically do full sync instead -- No support for gender sync (support can be added, file an issue if you want it). +- The Monica public instance limits API usage to 60 calls per minute. As every contact needs *at least* 2 API calls, **the script can not sync more than 30 contacts per minute** (thus affecting primarily initial and full sync). + > If you are hosting your own instance of Monica you can configure this rate limit starting from Monica v3.3.0. +- Delta sync will fail if there are more than 7 days between the last sync (Google restriction). In this case, the script will automatically do (fast) full sync instead - A label itself won't be deleted automatically if it has been removed from the last contact. - If there is a Google note it will be synced with exactly one Monica note. To this end, a small text will be added to the synced note at Monica. This makes it easy for you to distinguish synced and Monica-only notes. This means **you can update and create as many *additional* notes as you want at Monica**, they will not be overwritten. - Monica contacts require a first name. If a Google contact does not have any name, it will be skipped. @@ -34,19 +60,46 @@ That being said: Be welcome to use it, fork it, copy it for your own projects, a ## Known bugs - Sometimes the Google API returns more contacts than necessary. This is not an issue because the sync will match the last known update timestamps and skip the contact if nothing has changed. -- Birthdays on 29. Feb will be synced as 01. March :-) -- Pay attention when you *merge* Google contacts on the web GUI. In this case the contact will get recreated at Monica during sync if `DELETE_ON_SYNC` is set to `True`(because Google assigns a new contact ID). That means all Monica-specific data will be deleted. You can avoid this by merging them manually (copy over the details by hand) or doing initial sync `-i` again afterwards. +- Birthdays on 29. Feb will be synced as 01. March 😊 +- Pay attention when you *merge* Google contacts on the web GUI. In this case the contact will get recreated at Monica during sync if `DELETE_ON_SYNC` is set to `True` (default: `False`), because Google assigns a new contact ID. That means all Monica-specific data will be deleted. You can avoid this by merging them manually (copy over the details by hand) or doing initial sync `-i` again afterwards. ## Get started -0. Install Python 3.9 or newer -1. Get the [official Python Quickstart script from Google](https://developers.google.com/people/quickstart/python) working. -2. Copy `credentials.json` and `token.pickle` inside the main repository directory. -3. Create a new `conf.py` file inside the main repository directory with [this content](#Config). -4. Do a `pip install -r requirements.txt` inside the main repository directory. -5. Run `python GMSync.py -i` +The setup is a bit involving process as you have to create a Google Cloud Platform project to access your contacts via the Google People API. +[Please follow this guide](./Setup.md) to complete the required steps and do an initial sync. Once you did that you can use one of the delta sync commands from the next section to update your contacts regularly. + +## Delta sync (without docker) -## All sync commands +Run the following command inside the main folder: + +```bash +python GMSync.py -d +``` + +## Delta sync (with docker) + +Run the following command inside the main folder (add `-d` for detached mode): + +```bash +docker run -v "$(pwd)/data":/usr/app/data -v "$(pwd)/logs":/usr/app/logs --env-file .env antonplagemann/google-monica-sync sh -c "python -u GMSync.py -d" +``` + +Alternatively you can download and configure the [docker-compose.yml](./docker-compose.yml) to your main directory and use `docker-compose up` (Windows & Linux). + +### Background script with docker + +Here's a sample script which you can use to automate syncing e.g. with a crontab schedule. + +```bash +#!/bin/bash + +cd /path/to/folder-with-docker-compose-yaml +set -e +export COMPOSE_INTERACTIVE_NO_CLI=1 +docker-compose up -d +``` + +## All commands Usage: @@ -54,13 +107,14 @@ Usage: python GMSync.py [arguments] ``` -| Argument | Description | -| :------- | :--------------------------------------------------------------------------------------- | -| `-i` | Database rebuild (interactive) and full sync | -| `-d` | Delta sync (unattended) | -| `-f` | Full sync (unattended) | -| `-sb` | Sync back new Monica contacts (unattended). Can be combined with all other arguments | -| `-c` | Check syncing database for errors (unattended). Can be combined with all other arguments | +| Argument | Description | +| :-------- | :--------------------------------------------------------------------------------------- | +| `-i` | Database rebuild (interactive) and full sync | +| `-d` | Delta sync (unattended) | +| `-f` | Full sync (unattended) | +| `-sb` | Sync back new Monica contacts (unattended). Can be combined with all other arguments | +| `-c` | Check syncing database for errors (unattended). Can be combined with all other arguments | +| `-e PATH` | Custom .env configuration file path (relative or absolute) | **Remark**: Full sync, database check and sync back require heavy API use (e.g. fetching of all Monica and Google contacts). So use wisely and consider the load you're producing with those operations (especially if you use the public hosted Monica instance). @@ -95,57 +149,6 @@ All progress will be printed at running time and logged in the `Sync.log` file. If you think something has gone wrong, you miss some contacts or just want a pretty database statistic, you can do a database check. This will check if every Google contact has its Monica counterpart and vice versa. It will also report orphaned database entries that do not have a contact on both sides. -## Config - -This is the config file. -Copy the content below and create a new `conf.py` file inside the main repository directory. -Then fill in your desired settings (hint: a Monica token can be retrieved in your account settings, no OAuth client needed). - -```python -# General: -# String values need to be in single or double quotes -# Boolean values need to be True or False -# List Elements need to be are seperated by commas (e.g. ["a", "b"]) - -# Your Monica api token (without 'Bearer ') -TOKEN = 'YOUR_TOKEN_HERE' -# Your Monica base url -BASE_URL = 'https://app.monicahq.com/api' -# Create reminders for birthdays and deceased days? -CREATE_REMINDERS = True -# Delete Monica contact if the corresponding Google contact has been deleted? -DELETE_ON_SYNC = True -# Do a street reversal in address sync if the first character is a number? -# (e.g. from '13 Auenweg' to 'Auenweg 13') -STREET_REVERSAL = False - -# What fields should be synced? (both directions) -# Names and birthday are mandatory -FIELDS = { - "career": True, # Company and job title - "address": True, - "phone": True, - "email": True, - "labels": True, - "notes": True -} - -# Define contact labels/tags/groups you want to include or exclude from sync. -# Exclude labels have the higher priority. -# Both lists empty means every contact is included -# Example: "include": ["Family"] will only process contacts labeled as Family. -GOOGLE_LABELS = { - # Applies for Google -> Monica sync - "include": [], - "exclude": [] -} -MONICA_LABELS = { - # Applies for Monica -> Google sync back - "include": [], - "exclude": [] -} -``` - ## Feature roadmap - ~~Add more sync fields:~~ @@ -164,5 +167,7 @@ MONICA_LABELS = { - [x] Extend config to allow user choice of synced fields - [ ] ~~Think about two-way sync~~ (too involving, not really needed) - [x] Database consistency check function -- [ ] Think about a pip package -- [ ] ~~Implement sync procedure using python threads: propably much faster with multithreading~~ (not much faster because the Monica API rate limit is the bottleneck here) +- [ ] ~~Think about a pip package~~ +- [ ] ~~Implement sync procedure using python threads: probably much faster with multithreading~~ (not much faster because the Monica API rate limit is the bottleneck here) +- [x] Add docker container +- [x] Add automated testing of changes diff --git a/Setup.md b/Setup.md new file mode 100644 index 0000000..f58e52a --- /dev/null +++ b/Setup.md @@ -0,0 +1,98 @@ +[comment]: <> "LTeX: language=en-US" + +# Google OAuth Application Setup Instructions + +Use these instructions to set up API credentials for use with the syncing script. +The following instructions are based on the [Create a project and enable the API](https://developers.google.com/workspace/guides/create-project) and [Create credentials](https://developers.google.com/workspace/guides/create-credentials) articles. + +## Create a new Google Cloud Platform (GCP) project + +To use the Google People API (formerly Contacts API), you need a Google Cloud Platform project. This project forms the basis for creating, enabling, and using all GCP services, including managing APIs, adding and removing collaborators, and managing permissions. + +1. Open the [Google Cloud Console](https://console.cloud.google.com/). +2. Next to `Google Cloud Platform`, click the down arrow 🔽. A dialog listing current projects appears. +3. Click `New Project`. The New Project screen appears. +4. In the `Project Name` field, enter a descriptive name for your project. For example, enter "Google Monica Sync". +5. Click `Create`. The console navigates to the Dashboard page and your project is created within a few minutes. + +## Enable the Google People API + +1. Next to "Google Cloud Platform," click the down arrow 🔽 and open your newly created project. +2. In the top-left corner, click `Menu` > `APIs & Services`. +3. Click `Enable APIs and Services`. The `Welcome to API Library` page appears. +4. In the search field, enter `People API` and press enter. +5. Click `Google People API`. The API page appears. +6. Click `Enable`. The "Overview" page appears. + +## Create credentials + +Credentials are used to obtain an access token from a Google authorization server. This token is then used to call the Google People API. All Google Workspace APIs access data are owned by end-users. + +### Configure the OAuth consent screen + +The python script uses the OAuth protocol. When you start the script, it requests authorizations for `read/write contacts` from a Google Account. Google then displays a consent screen to you including a summary of your created project and the requested scopes of access. You must configure this consent screen for the authentication to work. + +To configure the OAuth consent screen: + +1. On the "Overview" page click `Credentials`. The credential page for your project appears. +2. Click `Configure Consent Screen`. The "OAuth consent screen" screen appears. +3. Select `External` as the user type for your app. +4. Click `Create`. A second "OAuth consent screen" screen appears. +5. Fill out the required form field (leave others blank): + - Enter "GMSync" in the `App name` field. + - Enter your personal email address in the `User support email` field. + - Enter your personal email address in the `Developer contact information` field. +6. Click `Save and Continue`. The "Scopes" page appears. +7. Click `Add or Remove Scopes`. The "Update selected scopes" page appears. +8. Filter for "People API" and select the scope ".../auth/contacts". +9. Click `Update`. A list of scopes for your app appears. Check if the scope ".../auth/contacts" is now listed in "Your sensitive scopes". +10. Click `Save and Continue`. The "Edit app registration" page appears. +11. At the "Test user" section click `Add Users`, enter the email of the Google Account whose contacts you want to sync, and click `Add`. + > Hint: If you get a `403 forbidden` during consent later, you may have entered the wrong email here. +12. Click `Save and Continue`. The "OAuth consent screen" appears. +13. Click `Back to Dashboard`. + +### Create an OAuth client ID credential + +1. In the left-hand navigation, click `Credentials`. The "Credentials" page appears. +2. Click `Create Credentials` and select `OAuth client ID`. The "Create OAuth client ID" page appears. +3. Click the Application type drop-down list and select "Desktop application". +4. In the name field, type a name for the credential. For example type "Python Syncing Script" +5. Click `Create`. The "OAuth client created" screen appears. This screen shows the Client ID and Client Secret. +6. Click `DOWNLOAD JSON`. This copies a client secret JSON file to your desktop. Note the location of this file. +7. Rename the client secret JSON file to `credentials.json`. + +## Get the sync token and run initial sync (**without** docker) + +0. Install Python 3.9 or newer +1. Copy `credentials.json` inside the `data` folder of the repository +2. In the main repository folder rename `.env.example` to `.env` and fill in your desired settings (a Monica token can be retrieved in your account settings). +3. Do a `pip install -r requirements.txt` inside the main repository directory. +4. Open a command prompt inside the main repository directory and run `python GMSync.py -i`. +5. On the first run the script will print a Google consent URL in the console. Copy this URL and open it in a browser on the host machine. +6. In your browser window, log in to your target Google account (the Google Account whose contacts you want to sync). +7. At "Google hasn’t verified this app" click `Continue`. +8. At "GMSync wants access to your Google Account" click `Continue`. +9. You should see now an authorization code. Copy this code, and switch back to your terminal window. +10. Paste the authorization code, press enter, and follow the prompts to complete the initial sync. + +## Get the sync token and run initial sync (**with** docker) + +0. Install docker +1. In your chosen main folder, create two folders named `data` and `logs`. +2. Copy `credentials.json` inside a `data` folder of your main directory. +3. [Download](https://github.com/antonplagemann/GoogleMonicaSync/blob/main/.env.example) the `.env.example` file, rename to `.env`, put it in your main folder, and fill in your desired settings (a Monica token can be retrieved in your account settings). + > This project is using a **non-root** container, so `data` and `logs` must have read-write permissions for UID 5678 (container user). + > For example, you can use `sudo chown -R 5678 data logs` or `sudo chmod -R 755 data logs` inside your main directory to set the appropriate permissions. +4. Open a command prompt inside the main directory run initial sync using the following command (on Windows replace `$(pwd)` with `%cd%`) + + ```bash + docker run -v "$(pwd)/data":/usr/app/data -v "$(pwd)/logs":/usr/app/logs --env-file .env -it antonplagemann/google-monica-sync sh -c "python -u GMSync.py -i" + ``` + +5. On the first run the script will print a Google consent URL in the console. Copy this URL and open it in a browser on the host machine. +6. In your browser window, log in to your target Google account (the Google Account whose contacts you want to sync). +7. At "Google hasn’t verified this app" click `Continue`. +8. At "GMSync wants access to your Google Account" click `Continue`. +9. You should see now an authorization code. Copy this code, and switch back to your terminal window. +10. Paste the authorization code, press enter, and follow the prompts to complete the initial sync. diff --git a/SyncHelper.py b/SyncHelper.py deleted file mode 100644 index f185982..0000000 --- a/SyncHelper.py +++ /dev/null @@ -1,1131 +0,0 @@ -# pylint: disable=import-error -import sys -from datetime import datetime -from logging import Logger -from typing import Tuple, Union, List - -from DatabaseHelper import Database, DatabaseEntry -from GoogleHelper import Google, GoogleContactUploadForm -from MonicaHelper import Monica, MonicaContactUploadForm - - -class Sync(): - '''Handles all syncing and merging issues with Google, Monica and the database.''' - - def __init__(self, log: Logger, databaseHandler: Database, - monicaHandler: Monica, googleHandler: Google, syncBackToGoogle: bool, - checkDatabase: bool, deleteMonicaContactsOnSync: bool, - streetReversalOnAddressSync: bool, syncingFields: dict) -> None: - self.log = log - self.startTime = datetime.now() - self.monica = monicaHandler - self.google = googleHandler - self.database = databaseHandler - self.mapping = self.database.getIdMapping() - self.reverseMapping = {monicaId: googleId for googleId, monicaId in self.mapping.items()} - self.nextSyncToken = self.database.getGoogleNextSyncToken() - self.syncBack = syncBackToGoogle - self.check = checkDatabase - self.deleteMonicaContacts = deleteMonicaContactsOnSync - self.streetReversal = streetReversalOnAddressSync - self.syncingFields = syncingFields - self.skipCreationPrompt = False - - def __updateMapping(self, googleId: str, monicaId: str) -> None: - '''Updates the internal Google <-> Monica id mapping dictionary.''' - self.mapping.update({googleId: monicaId}) - self.reverseMapping.update({monicaId: googleId}) - - def startSync(self, syncType: str = '') -> None: - '''Starts the next sync cycle depending on the requested type - and the database data.''' - if syncType == 'initial': - # Initial sync requested - self.__initialSync() - elif not self.mapping: - # There is no sync database. Initial sync is needed for all other sync types - msg = "No sync database found, please do a initial sync first!" - self.log.info(msg) - print(msg + "\n") - raise Exception("Initial sync needed!") - elif syncType == 'full': - # As this is a full sync, get all contacts at once to save time - self.monica.getContacts() - # Full sync requested so dont use database timestamps here - self.__sync('full', dateBasedSync=False) - elif syncType == 'delta' and not self.nextSyncToken: - # Delta sync requested but no sync token found - msg = "No sync token found, delta sync not possible. Doing (fast) full sync instead..." - self.log.info(msg) - print(msg + "\n") - # Do a full sync with database timestamp comparison (fast) - self.__sync('full') - elif syncType == 'delta': - # Delta sync requested - self.__sync('delta') - elif syncType == 'syncBack': - # Sync back to Google requested - self.__syncBack() - - # Print statistics - self.__printSyncStatistics() - - if self.check: - # Database check requested - self.checkDatabase() - - def __initialSync(self) -> None: - '''Builds the syncing database and starts a full sync. Needs user interaction!''' - self.database.deleteAndInitialize() - self.mapping.clear() - self.__buildSyncDatabase() - print("\nThe following fields will be overwritten with Google data:") - for field, choice in list(self.syncingFields.items())[:-1]: - if choice: - print(f"- {field}") - print("Start full sync now?") - print("\t0: No (abort initial sync)") - print("\t1: Yes") - choice = int(input("Enter your choice (number only): ")) - if not choice: - raise Exception("Sync aborted by user choice") - self.__sync('full', dateBasedSync=False) - - def __deleteMonicaContact(self, googleContact: dict) -> None: - '''Deletes a Monica contact given a corresponding Google contact.''' - # Initialization - googleId = googleContact["resourceName"] - gContactDisplayName = self.google.getContactNames(googleContact)[3] - msg = f"'{gContactDisplayName}' ('{googleId}'): Found deleted Google contact. Deleting Monica contact..." - self.log.info(msg) - - try: - # Try to delete the corresponding contact - monicaId = self.database.findById(googleId=googleId)[1] - self.monica.deleteContact(monicaId, gContactDisplayName) - self.database.delete(googleId, monicaId) - self.mapping.pop(googleId) - self.google.removeContactFromList(googleContact) - msg = f"'{gContactDisplayName}' ('{monicaId}'): Monica contact deleted successfully" - self.log.info(msg) - except Exception: - msg = f"'{gContactDisplayName}' ('{googleId}'): Failed deleting corresponding Monica contact! Please delete manually!" - self.google.removeContactFromList(googleContact) - self.log.error(msg) - print(msg) - - def __sync(self, syncType: str, dateBasedSync: bool = True) -> None: - '''Fetches every contact from Google and Monica and does a full sync.''' - # Initialization - msg = f"Starting {syncType} sync..." - self.log.info(msg) - print("\n" + msg) - if syncType == 'delta': - googleContacts = self.google.getContacts(syncToken=self.nextSyncToken) - else: - googleContacts = self.google.getContacts() - contactCount = len(googleContacts) - - # If Google hasnt returned some data - if not googleContacts: - msg = f"No (changed) Google contacts found!" - self.log.info(msg) - print("\n" + msg) - - # Process every Google contact - for num, googleContact in enumerate(googleContacts): - sys.stdout.write( - f"\rProcessing Google contact {num+1} of {contactCount}") - sys.stdout.flush() - - # Delete Monica contact if Google contact was deleted (if chosen by user; delta sync only) - isDeleted = googleContact.get('metadata', {}).get('deleted', False) - if isDeleted and self.deleteMonicaContacts: - self.__deleteMonicaContact(googleContact) - # Skip further processing - continue - - # Skip all contacts which have not changed according to the database lastChanged date (if present) - try: - # Get Monica id and timestamp - monicaId, _, _, databaseTimestamp = self.database.findById(googleId=googleContact["resourceName"])[1:5] - if dateBasedSync and not databaseTimestamp == 'NULL': - databaseDate = self.__convertGoogleTimestamp(databaseTimestamp) - contactTimestamp = googleContact['metadata']['sources'][0]["updateTime"] - contactDate = self.__convertGoogleTimestamp(contactTimestamp) - - # Skip if nothing has changed - if databaseDate == contactDate: - continue - except Exception: - # Create a new Google contact if there's nothing in the database yet - googleId = googleContact['resourceName'] - gContactDisplayName = self.google.getContactNames(googleContact)[3] - msg = f"'{gContactDisplayName}' ('{googleId}'): No Monica id found': Creating new Monica contact..." - self.log.info(msg) - print("\n" + msg) - - # Create new Monica contact - monicaContact = self.__createMonicaContact(googleContact) - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): New Monica contact created" - self.log.info(msg) - print(msg) - - # Update database and mapping - databaseEntry = DatabaseEntry(googleContact['resourceName'], - monicaContact['id'], - gContactDisplayName, - monicaContact['complete_name'], - googleContact['metadata']['sources'][0]['updateTime'], - monicaContact['updated_at']) - self.database.insertData(databaseEntry) - self.__updateMapping(googleContact['resourceName'], str(monicaContact['id'])) - msg = f"'{googleContact['resourceName']}' <-> '{monicaContact['id']}': New sync connection added" - self.log.info(msg) - - # Sync additional details - self.__syncDetails(googleContact, monicaContact) - - # Proceed with next contact - continue - - try: - # Get Monica contact by id - monicaContact = self.monica.getContact(monicaId) - except Exception as e: - msg = f"'{monicaId}': Failed to fetch Monica contact: {str(e)}" - self.log.error(msg) - print("\n" + msg) - print("Please do not delete Monica contacts manually!") - raise Exception("Could't connect to Monica api or Database not consistent, consider doing initial sync to rebuild.") - # Merge name, birthday and deceased date and update them - self.__mergeAndUpdateNBD(monicaContact, googleContact) - - # Update Google contact last changed date in the database - self.database.update(googleId=googleContact['resourceName'], - googleFullName=self.google.getContactNames(googleContact)[3], - googleLastChanged=googleContact['metadata']['sources'][0]['updateTime']) - - # Refresh Monica data (could have changed) - monicaContact = self.monica.getContact(monicaId) - - # Sync additional details - self.__syncDetails(googleContact, monicaContact) - - # Finished - msg = f"{syncType.capitalize()} sync finished!" - self.log.info(msg) - print("\n" + msg) - - # Sync lonely Monica contacts back to Google if chosen by user - if self.syncBack: - self.__syncBack() - - def __syncDetails(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs additional details, such as company, jobtitle, labels, - address, phone numbers, emails, notes, contact picture, etc.''' - if self.syncingFields["career"]: - # Sync career info - self.__syncCareerInfo(googleContact, monicaContact) - - if self.syncingFields["address"]: - # Sync address info - self.__syncAddress(googleContact, monicaContact) - - if self.syncingFields["phone"] or self.syncingFields["email"]: - # Sync phone and email - self.__syncPhoneEmail(googleContact, monicaContact) - - if self.syncingFields["labels"]: - # Sync labels - self.__syncLabels(googleContact, monicaContact) - - if self.syncingFields["notes"]: - # Sync notes if not existent at Monica - self.__syncNotes(googleContact, monicaContact) - - def __syncNotes(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs Google contact notes if there is no note present at Monica.''' - monicaNotes = self.monica.getNotes(monicaContact["id"], monicaContact["complete_name"]) - try: - identifier = "\n\n*This note is synced from your Google contacts. Do not edit here.*" - if googleContact.get("biographies", []): - # Get Google note - googleNote = { - "body": googleContact["biographies"][0].get("value", "").strip(), - "contact_id": monicaContact["id"], - "is_favorited": False - } - # Convert normal newlines to markdown newlines - googleNote["body"] = googleNote["body"].replace("\n", " \n") - - if not monicaNotes: - # If there is no Monica note sync the Google note - googleNote["body"] += identifier - self.monica.addNote(googleNote, monicaContact["complete_name"]) - else: - updated = False - for monicaNote in monicaNotes: - if monicaNote["body"] == googleNote["body"]: - # If there is a note with the same content update it and add the identifier - googleNote["body"] += identifier - self.monica.updateNote(monicaNote["id"], googleNote, monicaContact["complete_name"]) - updated = True - break - elif identifier in monicaNote["body"]: - # Found identifier, update this note if changed - googleNote["body"] += identifier - if monicaNote["body"] != googleNote["body"]: - self.monica.updateNote(monicaNote["id"], googleNote, monicaContact["complete_name"]) - updated = True - break - if not updated: - # No note with same content or identifier found so create a new one - googleNote["body"] += identifier - self.monica.addNote(googleNote, monicaContact["complete_name"]) - elif monicaNotes: - for monicaNote in monicaNotes: - if identifier in monicaNote["body"]: - # Found identifier, delete this note - self.monica.deleteNote(monicaNote["id"], monicaNote["contact"]["id"], monicaContact["complete_name"]) - break - - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error creating Monica note: {str(e)}" - self.log.warning(msg) - - def __syncLabels(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs Google contact labels/groups/tags.''' - try: - # Get google labels information - googleLabels = [ - self.google.getLabelName( - label["contactGroupMembership"]["contactGroupResourceName"]) - for label in googleContact.get("memberships", []) - ] - - # Remove tags if not present in Google contact - removeList = [label["id"] for label in monicaContact["tags"] - if label["name"] not in googleLabels] - if removeList: - self.monica.removeTags({"tags": removeList}, monicaContact["id"], monicaContact["complete_name"]) - - # Update labels if neccessary - monicaLabels = [label["name"] - for label in monicaContact["tags"] if label["name"] in googleLabels] - if sorted(googleLabels) != sorted(monicaLabels): - self.monica.addTags({"tags": googleLabels}, monicaContact["id"], monicaContact["complete_name"]) - - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error updating Monica contact labels: {str(e)}" - self.log.warning(msg) - - def __syncPhoneEmail(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs phone and email fields.''' - monicaContactFields = self.monica.getContactFields(monicaContact['id'], monicaContact['complete_name']) - if self.syncingFields["email"]: - self.__syncEmail(googleContact, monicaContact, monicaContactFields) - if self.syncingFields["phone"]: - self.__syncPhone(googleContact, monicaContact, monicaContactFields) - - def __syncEmail(self, googleContact: dict, monicaContact: dict, monicaContactFields: dict) -> None: - '''Syncs email fields.''' - try: - # Email processing - monicaContactEmails = [ - field for field in monicaContactFields - if field["contact_field_type"]["type"] == "email"] - googleContactEmails = googleContact.get("emailAddresses", []) - - if googleContactEmails: - googleEmails = [ - { - "contact_field_type_id": self.monica.getContactFieldId('email'), - "data": email["value"].strip(), - "contact_id": monicaContact["id"] - } - for email in googleContactEmails - ] - if monicaContactEmails: - # There is Google and Monica data: Check and recreate emails - for monicaEmail in monicaContactEmails: - # Check if there are emails to be deleted - if monicaEmail["content"] in [googleEmail["data"] for googleEmail in googleEmails]: - continue - else: - self.monica.deleteContactField(monicaEmail["id"], monicaContact["id"], monicaContact["complete_name"]) - for googleEmail in googleEmails: - # Check if there are emails to be created - if googleEmail["data"] in [monicaEmail["content"] for monicaEmail in monicaContactEmails]: - continue - else: - self.monica.createContactField(monicaContact["id"], googleEmail, monicaContact["complete_name"]) - else: - # There is only Google data: Create emails - for googleEmail in googleEmails: - self.monica.createContactField(monicaContact["id"], googleEmail, monicaContact["complete_name"]) - - elif monicaContactEmails: - # Delete Monica contact emails - for monicaEmail in monicaContactEmails: - self.monica.deleteContactField(monicaEmail["id"], monicaContact["id"], monicaContact["complete_name"]) - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error updating Monica contact email: {str(e)}" - self.log.warning(msg) - - def __syncPhone(self, googleContact: dict, monicaContact: dict, monicaContactFields: dict) -> None: - '''Syncs phone fields.''' - try: - # Phone number processing - monicaContactPhones = [ - field for field in monicaContactFields - if field["contact_field_type"]["type"] == "phone"] - googleContactPhones = googleContact.get("phoneNumbers", []) - - if googleContactPhones: - googlePhones = [ - { - "contact_field_type_id": self.monica.getContactFieldId('phone'), - "data": number["value"].strip(), - "contact_id": monicaContact["id"] - } - for number in googleContactPhones - ] - if monicaContactPhones: - # There is Google and Monica data: Check and recreate phone numbers - for monicaPhone in monicaContactPhones: - # Check if there are phone numbers to be deleted - if monicaPhone["content"] in [googlePhone["data"] for googlePhone in googlePhones]: - continue - else: - self.monica.deleteContactField(monicaPhone["id"], monicaContact["id"], monicaContact["complete_name"]) - for googlePhone in googlePhones: - # Check if there are phone numbers to be created - if googlePhone["data"] in [monicaPhone["content"] for monicaPhone in monicaContactPhones]: - continue - else: - self.monica.createContactField(monicaContact["id"], googlePhone, monicaContact["complete_name"]) - else: - # There is only Google data: Create phone numbers - for googlePhone in googlePhones: - self.monica.createContactField(monicaContact["id"], googlePhone, monicaContact["complete_name"]) - - elif monicaContactPhones: - # Delete Monica contact phone numbers - for monicaPhone in monicaContactPhones: - self.monica.deleteContactField(monicaPhone["id"], monicaContact["id"], monicaContact["complete_name"]) - - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error updating Monica contact phone: {str(e)}" - self.log.warning(msg) - - def __syncCareerInfo(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs company and job title fields.''' - try: - monicaDataPresent = bool(monicaContact["information"]["career"]["job"] or - monicaContact["information"]["career"]["company"]) - googleDataPresent = bool(googleContact.get("organizations", False)) - if googleDataPresent or monicaDataPresent: - # Get google career information - company = googleContact.get("organizations", [{}])[0].get("name", "").strip() - department = googleContact.get("organizations", [{}])[0].get("department", "").strip() - if department: - department = f"; {department}" - job = googleContact.get("organizations", [{}])[0].get("title", None) - googleData = { - "job": job.strip() if job else None, - "company": company + department if company or department else None - } - # Get monica career information - monicaData = { - "job": monicaContact['information']['career'].get('job', None), - "company": monicaContact['information']['career'].get('company', None) - } - - # Compare and update if neccessary - if googleData != monicaData: - self.monica.updateCareer(monicaContact["id"], googleData) - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error updating Monica contact career: {str(e)}" - self.log.warning(msg) - - def __syncAddress(self, googleContact: dict, monicaContact: dict) -> None: - '''Syncs all address fields.''' - try: - monicaDataPresent = bool(monicaContact.get("addresses", False)) - googleDataPresent = bool(googleContact.get("addresses", False)) - if googleDataPresent: - # Get Google data - googleAddressList = [] - for addr in googleContact.get("addresses", []): - # None type is important for comparison, empty string won't work here - name = None - street = None - city = None - province = None - postalCode = None - countryCode = None - street = addr.get("streetAddress", "").replace("\n", " ").strip() or None - if self.streetReversal: - # Street reversal: from '13 Auenweg' to 'Auenweg 13' - try: - if street and street[0].isdigit(): - street = f'{street[street.index(" ")+1:]} {street[:street.index(" ")]}'.strip() - except Exception: - pass - - # Get (extended) city - city = f'{addr.get("city", "")} {addr.get("extendedAddress", "")}'.strip() or None - # Get other details - province = addr.get("region", None) - postalCode = addr.get("postalCode", None) - countryCode = addr.get("countryCode", None) - # Name can not be empty - name = addr.get("formattedType", None) or "Other" - # Do not sync empty addresses - if not any([street, city, province, postalCode, countryCode]): - continue - googleAddressList.append({ - 'name': name, - 'street': street, - 'city': city, - 'province': province, - 'postal_code': postalCode, - 'country': countryCode, - 'contact_id': monicaContact['id'] - }) - - if monicaDataPresent: - # Get Monica data - monicaAddressList = [] - for addr in monicaContact.get("addresses", []): - monicaAddressList.append({addr["id"]: { - 'name': addr["name"], - 'street': addr["street"], - 'city': addr["city"], - 'province': addr["province"], - 'postal_code': addr["postal_code"], - 'country': addr["country"].get("iso", None) if addr["country"] else None, - 'contact_id': monicaContact['id'] - }}) - - if googleDataPresent and monicaDataPresent: - monicaPlainAddressList = [monicaAddress for item in monicaAddressList for monicaAddress in item.values()] - # Do a complete comparison - if all([googleAddress in monicaPlainAddressList for googleAddress in googleAddressList]): - # All addresses are equal, nothing to do - return - else: - # Delete all Monica addresses and create new ones afterwards - # Safest way, I don't want to code more deeper comparisons and update functions - for element in monicaAddressList: - for addressId, _ in element.items(): - self.monica.deleteAddress(addressId, monicaContact["id"], monicaContact["complete_name"]) - elif not googleDataPresent and monicaDataPresent: - # Delete all Monica addresses - for element in monicaAddressList: - for addressId, _ in element.items(): - self.monica.deleteAddress(addressId, monicaContact["id"], monicaContact["complete_name"]) - - if googleDataPresent: - # All old Monica data (if existed) have been cleaned now, proceed with address creation - for googleAddress in googleAddressList: - self.monica.createAddress(googleAddress, monicaContact["complete_name"]) - - except Exception as e: - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): Error updating Monica addresses: {str(e)}" - self.log.warning(msg) - - def __buildSyncDatabase(self) -> None: - '''Builds a Google <-> Monica 1:1 contact id mapping and saves it to the database.''' - # Initialization - conflicts = [] - googleContacts = self.google.getContacts() - self.monica.getContacts() - contactCount = len(googleContacts) - msg = "Building sync database..." - self.log.info(msg) - print("\n" + msg) - - # Process every Google contact - for num, googleContact in enumerate(googleContacts): - sys.stdout.write(f"\rProcessing Google contact {num+1} of {contactCount}") - sys.stdout.flush() - # Try non-interactive search first - monicaId = self.__simpleMonicaIdSearch(googleContact) - if not monicaId: - # Non-interactive search failed, try interactive search next - conflicts.append(googleContact) - - # Process all conflicts - if len(conflicts): - msg = f"Found {len(conflicts)} possible conflicts, starting resolving procedure..." - self.log.info(msg) - print("\n" + msg) - for googleContact in conflicts: - # Do a interactive search with user interaction next - monicaId = self.__interactiveMonicaIdSearch(googleContact) - assert monicaId, "Could not create a Monica contact. Sync aborted." - - # Finished - msg = "Sync database built!" - self.log.info(msg) - print("\n" + msg) - - def __syncBack(self) -> None: - '''Sync lonely Monica contacts back to Google by creating a new contact there.''' - msg = "Starting sync back..." - self.log.info(msg) - print("\n" + msg) - monicaContacts = self.monica.getContacts() - contactCount = len(monicaContacts) - - # Process every Monica contact - for num, monicaContact in enumerate(monicaContacts): - sys.stdout.write(f"\rProcessing Monica contact {num+1} of {contactCount}") - sys.stdout.flush() - - # If there the id isnt in the database: create a new Google contact and upload - if str(monicaContact['id']) not in self.mapping.values(): - # Create Google contact - googleContact = self.__createGoogleContact(monicaContact) - if not googleContact: - msg = f"'{monicaContact['complete_name']}': Error encountered at creating new Google contact. Skipping..." - self.log.warning(msg) - print(msg) - continue - gContactDisplayName = self.google.getContactNames(googleContact)[3] - - # Update database and mapping - databaseEntry = DatabaseEntry(googleContact['resourceName'], - monicaContact['id'], - gContactDisplayName, - monicaContact['complete_name']) - self.database.insertData(databaseEntry) - msg = f"'{gContactDisplayName}' ('{googleContact['resourceName']}'): New google contact created (sync back)" - print("\n" + msg) - self.log.info(msg) - self.__updateMapping(googleContact['resourceName'], str(monicaContact['id'])) - msg = f"'{googleContact['resourceName']}' <-> '{monicaContact['id']}': New sync connection added" - self.log.info(msg) - - if not self.google.createdContacts: - msg = "No contacts for sync back found" - self.log.info(msg) - print("\n" + msg) - - # Finished - msg = "Sync back finished!" - self.log.info(msg) - print(msg) - - def __printSyncStatistics(self) -> None: - '''Prints and logs a pretty sync statistic of the last sync.''' - self.monica.updateStatistics() - tme = str(datetime.now() - self.startTime).split(".")[0] + "h" - gAp = str(self.google.apiRequests) + (8-len(str(self.google.apiRequests))) * ' ' - mAp = str(self.monica.apiRequests) + (8-len(str(self.monica.apiRequests))) * ' ' - mCC = str(len(self.monica.createdContacts)) + (8-len(str(len(self.monica.createdContacts)))) * ' ' - mCU = str(len(self.monica.updatedContacts)) + (8-len(str(len(self.monica.updatedContacts)))) * ' ' - mCD = str(len(self.monica.deletedContacts)) + (8-len(str(len(self.monica.deletedContacts)))) * ' ' - gCC = str(len(self.google.createdContacts)) + (8-len(str(len(self.google.createdContacts)))) * ' ' - msg = "\n" \ - f"Sync statistics: \n" \ - f"+-------------------------------------+\n" \ - f"| Syncing time: {tme } |\n" \ - f"| Google api calls used: {gAp } |\n" \ - f"| Monica api calls used: {mAp } |\n" \ - f"| Monica contacts created: {mCC } |\n" \ - f"| Monica contacts updated: {mCU } |\n" \ - f"| Monica contacts deleted: {mCD } |\n" \ - f"| Google contacts created: {gCC } |\n" \ - f"+-------------------------------------+" - print(msg) - self.log.info(msg) - - def __createGoogleContact(self, monicaContact: dict) -> dict: - '''Creates a new Google contact from a given Monica contact and returns it.''' - # Get names (no nickname) - firstName = monicaContact['first_name'] or '' - lastName = monicaContact['last_name'] or '' - fullName = monicaContact['complete_name'] or '' - nickname = monicaContact['nickname'] or '' - middleName = self.__getMonicaMiddleName(firstName, lastName, nickname, fullName) - - # Get birthday details (age based birthdays are not supported by Google) - birthday = {} - birthdayTimestamp = monicaContact['information']["dates"]["birthdate"]["date"] - ageBased = monicaContact['information']["dates"]["birthdate"]["is_age_based"] - if birthdayTimestamp and not ageBased: - yearUnknown = monicaContact['information']["dates"]["birthdate"]["is_year_unknown"] - date = self.__convertMonicaTimestamp(birthdayTimestamp) - if not yearUnknown: - birthday.update({ - 'year': date.year - }) - birthday.update({ - 'month': date.month, - 'day': date.day - }) - - # Get addresses - addresses = monicaContact["addresses"] if self.syncingFields["address"] else [] - - # Get career info if exists - career = {key: value for key, value in monicaContact['information']["career"].items() - if value and self.syncingFields["career"]} - - # Get phone numbers and email addresses - if self.syncingFields["phone"] or self.syncingFields["email"]: - monicaContactFields = self.monica.getContactFields(monicaContact['id'], monicaContact['complete_name']) - # Get email addresses - emails = [field["content"] for field in monicaContactFields - if field["contact_field_type"]["type"] == "email" - and self.syncingFields["email"]] - # Get phone numbers - phoneNumbers = [field["content"] for field in monicaContactFields - if field["contact_field_type"]["type"] == "phone" - and self.syncingFields["phone"]] - - # Get tags/labels and create them if neccessary - labelIds = [self.google.getLabelId(tag['name']) - for tag in monicaContact["tags"] - if self.syncingFields["labels"]] - - # Create contact upload form - form = GoogleContactUploadForm(firstName=firstName, lastName=lastName, - middleName=middleName, birthdate=birthday, - phoneNumbers=phoneNumbers, career=career, - emailAdresses=emails, labelIds=labelIds, - addresses=addresses) - - # Upload contact - contact = self.google.createContact(data=form.getData()) - - return contact - - def __getMonicaMiddleName(self, firstName: str, lastName: str, nickname: str, fullName: str) -> str: - '''Monica contacts have for some reason a hidden field middlename that can be set (creation/update) - but sadly can not retrieved later. This function computes it by using the complete_name field.''' - try: - # If there is a nickname it will be parenthesized with a space - nicknameLength = len(nickname) + 3 if nickname else 0 - middleName = fullName[len(firstName):len(fullName) - (len(lastName) + nicknameLength)].strip() - return middleName - except Exception: - return '' - - def checkDatabase(self) -> None: - '''Checks if there are orphaned database entries which need to be resolved. - The following checks and assumptions will be made: - 1. Google contact id NOT IN database - -> Info: contact is currently not in sync - 2. Google contact id IN database BUT Monica contact not found - -> Error: deleted Monica contact or wrong id - 3. Monica contact id NOT IN database - -> Info: contact is currently not in sync - 4. Monica contact id IN database BUT Google contact not found - -> Error: deleted Google contact or wrong id - 5. Google contact id IN database BUT Monica AND Google contact not found - -> Warning: orphaned database entry''' - # Initialization - startTime = datetime.now() - googleContactsNotSynced = [] - googleContactsSynced = [] - monicaContactsNotSynced = [] - monicaContactsSynced = [] - errors = 0 - msg = f"Starting database check..." - self.log.info(msg) - print("\n" + msg) - - # Get contacts - googleContacts = self.google.getContacts(refetchData=True, requestSyncToken=False) - googleContactsCount = len(googleContacts) - monicaContacts = self.monica.getContacts() - monicaContactsCount = len(monicaContacts) - - # Check every Google contact - for num, googleContact in enumerate(googleContacts): - sys.stdout.write( - f"\rProcessing Google contact {num+1} of {googleContactsCount}") - sys.stdout.flush() - - # Get monica id - monicaId = self.mapping.get(googleContact['resourceName'], None) - if not monicaId: - googleContactsNotSynced.append(googleContact) - continue - - # Get monica contact - try: - monicaContact = self.monica.getContact(monicaId) - assert monicaContact - monicaContactsSynced.append(monicaContact) - except Exception: - errors += 1 - msg = f"'{self.google.getContactNames(googleContact)[3]}' ('{googleContact['resourceName']}'): " \ - f"Wrong id or missing Monica contact for id '{monicaId}'." - self.log.error(msg) - print("\nError: " + msg) - - # Print a newline to avoid overwriting console output - print("") - - # Check every Monica contact - for num, monicaContact in enumerate(monicaContacts): - sys.stdout.write( - f"\rProcessing Monica contact {num+1} of {monicaContactsCount}") - sys.stdout.flush() - - # Get monica id - googleId = self.reverseMapping.get(str(monicaContact["id"]), None) - if not googleId: - monicaContactsNotSynced.append(monicaContact) - continue - - # Get Monica contact - try: - googleContact = self.google.getContact(googleId) - assert googleContact - googleContactsSynced.append(googleContact) - except Exception: - errors += 1 - msg = f"'{monicaContact['complete_name']}' ('{monicaContact['id']}'): " \ - f"Wrong id or missing Google contact for id '{googleId}'." - self.log.error(msg) - print("\nError: " + msg) - - # Check for orphaned database entrys - googleIds = [c['resourceName'] for c in googleContacts] - monicaIds = [str(c['id']) for c in monicaContacts] - orphanedEntrys = [googleId for googleId, monicaId in self.mapping.items() - if googleId not in googleIds - and monicaId not in monicaIds] - - # Log results - if orphanedEntrys: - self.log.info("The following database entrys are orphaned:") - for googleId in orphanedEntrys: - monicaId, googleFullName, monicaFullName = self.database.findById(googleId)[1:4] - self.log.warning(f"'{googleId}' <-> '{monicaId}' ('{googleFullName}' <-> '{monicaFullName}')") - self.log.info("This doesn't cause sync errors, but you can fix it doing initial sync '-i'") - if not monicaContactsNotSynced and not googleContactsNotSynced: - self.log.info("All contacts are currently in sync") - elif monicaContactsNotSynced: - self.log.info("The following Monica contacts are currently not in sync:") - for monicaContact in monicaContactsNotSynced: - self.log.info(f"'{monicaContact['complete_name']}' ('{monicaContact['id']}')") - self.log.info("You can do a sync back '-sb' to fix that") - if googleContactsNotSynced: - self.log.info("The following Google contacts are currently not in sync:") - for googleContact in googleContactsNotSynced: - googleId = googleContact['resourceName'] - gContactDisplayName = self.google.getContactNames(googleContact)[3] - self.log.info(f"'{gContactDisplayName}' ('{googleId}')") - self.log.info("You can do a full sync '-f' to fix that") - - # Finished - if errors: - msg = f"Database check failed. Consider doing initial sync '-i' again!" - else: - msg = f"Database check finished, no errors found!" - self.log.info(msg) - print("\n" + msg) - - # Print and log statistics - self.__printCheckStatistics(startTime, errors, len(orphanedEntrys), - len(monicaContactsNotSynced), len(googleContactsNotSynced), - monicaContactsCount, googleContactsCount) - - def __printCheckStatistics(self, startTime: str, errors: int, orphaned: int, - monicaContactsNotSyncedCount: int, googleContactsNotSyncedCount: int, - monicaContactsCount: int, googleContactsCount: int) -> None: - '''Prints and logs a pretty check statistic of the last database check.''' - tme = str(datetime.now() - startTime).split(".")[0] + "h" - err = str(errors) + (8-len(str(errors))) * ' ' - oph = str(orphaned) + (8-len(str(orphaned))) * ' ' - mNS = str(monicaContactsNotSyncedCount) + (8-len(str(monicaContactsNotSyncedCount))) * ' ' - gNS = str(googleContactsNotSyncedCount) + (8-len(str(googleContactsNotSyncedCount))) * ' ' - cMC = str(monicaContactsCount) + (8-len(str(monicaContactsCount))) * ' ' - cGC = str(googleContactsCount) + (8-len(str(googleContactsCount))) * ' ' - msg = "\n" \ - f"Check statistics: \n" \ - f"+-----------------------------------------+\n" \ - f"| Check time: {tme } |\n" \ - f"| Errors: {err } |\n" \ - f"| Orphaned database entrys: {oph } |\n" \ - f"| Monica contacts not in sync: {mNS } |\n" \ - f"| Google contacts not in sync: {gNS } |\n" \ - f"| Checked Monica contacts: {cMC } |\n" \ - f"| Checked Google contacts: {cGC } |\n" \ - f"+-----------------------------------------+" - print(msg) - self.log.info(msg) - - def __mergeAndUpdateNBD(self, monicaContact: dict, googleContact: dict) -> None: - '''Updates names, birthday and deceased date by merging an existing Monica contact with - a given Google contact.''' - # Get names - firstName, lastName = self.__getMonicaNamesFromGoogleContact(googleContact) - middleName = self.google.getContactNames(googleContact)[1] - displayName = self.google.getContactNames(googleContact)[3] - nickName = self.google.getContactNames(googleContact)[6] - # First name is required for Monica - if not firstName: - firstName = displayName - lastName = '' - - # Get birthday - birthday = googleContact.get("birthdays", None) - birthdateYear, birthdateMonth, birthdateDay = None, None, None - if birthday: - birthdateYear = birthday[0].get("date", {}).get("year", None) - birthdateMonth = birthday[0].get("date", {}).get("month", None) - birthdateDay = birthday[0].get("date", {}).get("day", None) - - # Get deceased info - deceasedDate = monicaContact["information"]["dates"]["deceased_date"]["date"] - deceasedDateIsAgeBased = monicaContact["information"]["dates"]["deceased_date"]["is_age_based"] - deceasedYear, deceasedMonth, deceasedDay = None, None, None - if deceasedDate: - date = self.__convertMonicaTimestamp(deceasedDate) - deceasedYear = date.year - deceasedMonth = date.month - deceasedDay = date.day - - # Assemble form object - googleForm = MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, nickName=nickName, - middleName=middleName, genderType=monicaContact["gender_type"], - birthdateDay=birthdateDay, birthdateMonth=birthdateMonth, - birthdateYear=birthdateYear, isBirthdateKnown=bool(birthday), - isDeceased=monicaContact["is_dead"], isDeceasedDateKnown=bool(deceasedDate), - deceasedYear=deceasedYear, deceasedMonth=deceasedMonth, - deceasedDay=deceasedDay, deceasedAgeBased=deceasedDateIsAgeBased, - createReminders=self.monica.createReminders) - - # Check if contacts are already equal - monicaForm = self.__getMonicaForm(monicaContact) - #if all([googleForm.data[key] == monicaForm.data[key] for key in googleForm.data.keys() if key != 'birthdate_year']): - if googleForm.data == monicaForm.data: - return - - # Upload contact - self.monica.updateContact(monicaId=monicaContact["id"], data=googleForm.data) - - def __getMonicaForm(self, monicaContact: dict) -> MonicaContactUploadForm: - '''Creates a Monica contact upload form from a given Monica contact for comparison.''' - # Get names - firstName = monicaContact['first_name'] or '' - lastName = monicaContact['last_name'] or '' - fullName = monicaContact['complete_name'] or '' - nickname = monicaContact['nickname'] or '' - middleName = self.__getMonicaMiddleName(firstName, lastName, nickname, fullName) - - # Get birthday details - birthdayTimestamp = monicaContact['information']["dates"]["birthdate"]["date"] - birthdateYear, birthdateMonth, birthdateDay = None, None, None - if birthdayTimestamp: - yearUnknown = monicaContact['information']["dates"]["birthdate"]["is_year_unknown"] - date = self.__convertMonicaTimestamp(birthdayTimestamp) - birthdateYear = date.year if not yearUnknown else None - birthdateMonth = date.month - birthdateDay = date.day - - - # Get deceased info - deceasedDate = monicaContact["information"]["dates"]["deceased_date"]["date"] - deceasedDateIsAgeBased = monicaContact["information"]["dates"]["deceased_date"]["is_age_based"] - deceasedYear, deceasedMonth, deceasedDay = None, None, None - if deceasedDate: - date = self.__convertMonicaTimestamp(deceasedDate) - deceasedYear = date.year - deceasedMonth = date.month - deceasedDay = date.day - - # Assemble form object - return MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, nickName=nickname, - middleName=middleName, genderType=monicaContact["gender_type"], - birthdateDay=birthdateDay, birthdateMonth=birthdateMonth, - birthdateYear=birthdateYear, isBirthdateKnown=bool(birthdayTimestamp), - isDeceased=monicaContact["is_dead"], isDeceasedDateKnown=bool(deceasedDate), - deceasedYear=deceasedYear, deceasedMonth=deceasedMonth, - deceasedDay=deceasedDay, deceasedAgeBased=deceasedDateIsAgeBased, - createReminders=self.monica.createReminders) - - def __createMonicaContact(self, googleContact: dict) -> dict: - '''Creates a new Monica contact from a given Google contact and returns it.''' - # Get names - firstName, lastName = self.__getMonicaNamesFromGoogleContact(googleContact) - middleName = self.google.getContactNames(googleContact)[1] - displayName = self.google.getContactNames(googleContact)[3] - nickName = self.google.getContactNames(googleContact)[6] - # First name is required for Monica - if not firstName: - firstName = displayName - lastName = '' - - # Get birthday - birthday = googleContact.get("birthdays", None) - birthdateYear, birthdateMonth, birthdateDay = None, None, None - if birthday: - birthdateYear = birthday[0].get("date", {}).get("year", None) - birthdateMonth = birthday[0].get("date", {}).get("month", None) - birthdateDay = birthday[0].get("date", {}).get("day", None) - - # Assemble form object - form = MonicaContactUploadForm(firstName=firstName, monica=self.monica, lastName=lastName, middleName=middleName, - nickName=nickName, birthdateDay=birthdateDay, - birthdateMonth=birthdateMonth, birthdateYear=birthdateYear, - isBirthdateKnown=bool(birthday), - createReminders=self.monica.createReminders) - # Upload contact - monicaContact = self.monica.createContact(data=form.data, referenceId=googleContact['resourceName']) - return monicaContact - - def __convertGoogleTimestamp(self, timestamp: str) -> datetime: - '''Converts Google timestamp to a datetime object.''' - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') - - def __convertMonicaTimestamp(self, timestamp: str) -> datetime: - '''Converts Monica timestamp to a datetime object.''' - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ') - - def __interactiveMonicaIdSearch(self, googleContact: dict) -> str: - '''Advanced search by first and last name for a given Google contact. - Tries to find a matching Monica contact and asks for user choice if - at least one candidate has been found. Creates a new Monica contact - if neccessary or chosen by User. Returns Monica contact id.''' - # Initialization - candidates = [] - gContactGivenName = self.google.getContactNames(googleContact)[0] - gContactFamilyName = self.google.getContactNames(googleContact)[2] - gContactDisplayName = self.google.getContactNames(googleContact)[3] - monicaContact = None - - # Process every Monica contact - for mContact in self.monica.getContacts(): - if (str(mContact['id']) not in self.mapping.values() - and (gContactGivenName == mContact['first_name'] - or gContactFamilyName == mContact['last_name'])): - # If the id isnt in the database and first or last name matches add potential candidate to list - candidates.append(mContact) - - # If there is at least one candidate let the user choose - choice = None - if candidates: - print("\nPossible syncing conflict, please choose your alternative by number:") - print(f"\tWhich Monica contact should be connected to '{gContactDisplayName}'?") - for num, monicaContact in enumerate(candidates): - print(f"\t{num}: {monicaContact['complete_name']}") - print(f"\t{num+1}: Create a new Monica contact") - choice = int(input("Enter your choice (number only): ")) - # Created a sublist with the selected candidate or an empty list if user votes for a new contact - candidates = candidates[choice:choice+1] - - # If there are no candidates (user vote or nothing found) create a new Monica contact - if not candidates: - # Ask user if not done before - if not choice and not self.skipCreationPrompt: - print(f"\nNo Monica contact has been found for '{gContactDisplayName}'") - print(f"\tCreate a new Monica contact?") - print("\t0: No (abort initial sync)") - print("\t1: Yes") - print("\t2: Yes to all") - choice = int(input("Enter your choice (number only): ")) - if not choice: - raise Exception("Sync aborted by user choice") - if choice == 2: - # Skip further contact creation prompts - self.skipCreationPrompt = True - - # Create new Monica contact - monicaContact = self.__createMonicaContact(googleContact) - - # Print success - msg = f"'{gContactDisplayName}' ('{monicaContact['id']}'): New Monica contact created" - self.log.info(msg) - print("Conflict resolved: " + msg) - - # There must be exactly one candidate from user vote - else: - monicaContact = candidates[0] - - # Update database and mapping - databaseEntry = DatabaseEntry(googleContact['resourceName'], - monicaContact['id'], - gContactDisplayName, - monicaContact['complete_name']) - self.database.insertData(databaseEntry) - self.__updateMapping(googleContact['resourceName'], str(monicaContact['id'])) - - # Print success - msg = f"'{googleContact['resourceName']}' <-> '{monicaContact['id']}': New sync connection added" - self.log.info(msg) - print("Conflict resolved: " + msg) - - return str(monicaContact['id']) - - # pylint: disable=unsubscriptable-object - def __simpleMonicaIdSearch(self, googleContact: dict) -> Union[str, None]: - '''Simple search by displayname for a given Google contact. - Tries to find a matching Monica contact and returns its id or None if not found''' - # Initialization - gContactGivenName = self.google.getContactNames(googleContact)[0] - gContactMiddleName = self.google.getContactNames(googleContact)[1] - gContactFamilyName = self.google.getContactNames(googleContact)[2] - gContactDisplayName = self.google.getContactNames(googleContact)[3] - candidates = [] - - # Process every Monica contact - for monicaContact in self.monica.getContacts(): - # Get monica data - mContactId = str(monicaContact['id']) - mContactFirstName = monicaContact['first_name'] or '' - mContactLastName = monicaContact['last_name'] or '' - mContactFullName = monicaContact['complete_name'] or '' - mContactNickname = monicaContact['nickname'] or '' - mContactMiddleName = self.__getMonicaMiddleName(mContactFirstName, mContactLastName, mContactNickname, mContactFullName) - # Check if the Monica contact is already assigned to a Google contact - isMonicaContactAssigned = mContactId in self.mapping.values() - # Check if display names match - isDisplayNameMatch = (gContactDisplayName == mContactFullName) - # Pre-check that the Google contact has a given and a family name - hasNames = gContactGivenName and gContactFamilyName - # Check if names match when ignoring honorifix prefixes - isWithoutPrefixMatch = hasNames and (' '.join([gContactGivenName, gContactFamilyName]) == mContactFullName) - # Check if first, middle and last name matches - isFirstLastMiddleNameMatch = (mContactFirstName == gContactGivenName - and mContactMiddleName == gContactMiddleName - and mContactLastName == gContactFamilyName) - # Assemble all conditions - matches = [isDisplayNameMatch, isWithoutPrefixMatch, isFirstLastMiddleNameMatch] - if not isMonicaContactAssigned and any(matches): - # Add possible candidate - candidates.append(monicaContact) - - # If there is only one candidate - if len(candidates) == 1: - monicaContact = candidates[0] - - # Update database and mapping - databaseEntry = DatabaseEntry(googleContact['resourceName'], - monicaContact['id'], - gContactDisplayName, - monicaContact['complete_name']) - self.database.insertData(databaseEntry) - self.__updateMapping(googleContact['resourceName'], str(monicaContact['id'])) - return str(monicaContact['id']) - - # Simple search failed - return None - - def __getMonicaNamesFromGoogleContact(self, googleContact: dict) -> Tuple[str, str]: - '''Creates first and last name from a Google contact with respect to honoric - suffix/prefix.''' - givenName, _, familyName, _, prefix, suffix, _ = self.google.getContactNames(googleContact) - if prefix: - givenName = f"{prefix} {givenName}".strip() - if suffix: - familyName = f"{familyName} {suffix}".strip() - return givenName, familyName diff --git a/__init__.py b/data/.empty similarity index 100% rename from __init__.py rename to data/.empty diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c2e4845 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: "3.8" + +services: + python: + image: antonplagemann/google-monica-sync:latest + + environment: + # Your Monica api token (without 'Bearer ') + - TOKEN=YOUR_TOKEN_HERE + # Your Monica base url (ends with /api) + # - BASE_URL=https://app.monicahq.com/api + # For more config options see .env.example + + # You can also specify a .env file to load from + # env_file: + # - .env + + # Put credentials, sync database and token files in ./data + volumes: + - ./data:/usr/app/data + # Remove the next line if you do not want to access the logs + - ./logs:/usr/app/logs + + # Adjust command if needed (-u needed for getting console output) + command: python -u GMSync.py -d diff --git a/helpers/.env.default b/helpers/.env.default new file mode 100644 index 0000000..9a0d727 --- /dev/null +++ b/helpers/.env.default @@ -0,0 +1,33 @@ +# Default values, do not download or modify this file! +# Instead define your own .env file using .env.example + +# Your Monica base url +BASE_URL=https://app.monicahq.com/api + +# Create reminders for birthdays and deceased days? +CREATE_REMINDERS=True +# Delete Monica contact if the corresponding Google contact has been deleted? +DELETE_ON_SYNC=True +# Do a street reversal in address sync if the first character is a number? +# (e.g. from '13 Auenweg' to 'Auenweg 13') +STREET_REVERSAL=false + +# What fields should be synced? (both directions) +# Names and birthday are mandatory +FIELDS=career,address,phone,email,labels,notes + +# Define contact labels/tags/groups you want to include or exclude from sync. +# Exclude labels have the higher priority. +# Both lists empty means every contact is included +# Example: 'GOOGLE_LABELS_INCLUDE=Family,My Friends' will only process contacts labeled as 'Family' or 'My Friends'. +# Applies for Google -> Monica sync +GOOGLE_LABELS_INCLUDE= +GOOGLE_LABELS_EXCLUDE= +# Applies for Monica -> Google sync back +MONICA_LABELS_INCLUDE= +MONICA_LABELS_EXCLUDE= + +# Define custom file paths +DATABASE_FILE=data/syncState.db +GOOGLE_TOKEN_FILE=data/token.pickle +GOOGLE_CREDENTIALS_FILE=data/credentials.json diff --git a/helpers/ConfigHelper.py b/helpers/ConfigHelper.py new file mode 100644 index 0000000..603e24b --- /dev/null +++ b/helpers/ConfigHelper.py @@ -0,0 +1,45 @@ +from logging import Logger +from os.path import abspath +from typing import Dict, Union + +from helpers.Exceptions import ConfigError + + +class Config: + """Class for parsing config .env files""" + + def __init__(self, log: Logger, raw_config: Dict[str, Union[str, None]]) -> None: + self._log = log + self._values = {key: value or "" for key, value in raw_config.items()} + try: + self.TOKEN = self._values.get("TOKEN", "") + self.BASE_URL = self._values["BASE_URL"] + if not self.TOKEN or self.TOKEN == "YOUR_TOKEN_HERE": + msg = "Missing required monica token config value!" + self._log.error(msg) + raise ConfigError(msg) + self.CREATE_REMINDERS = self.__get_boolean("CREATE_REMINDERS") + self.DELETE_ON_SYNC = self.__get_boolean("DELETE_ON_SYNC") + self.STREET_REVERSAL = self.__get_boolean("STREET_REVERSAL") + self.FIELDS = self.__get_array("FIELDS") + self.GOOGLE_LABELS_INCLUDE = self.__get_array("GOOGLE_LABELS_INCLUDE") + self.GOOGLE_LABELS_EXCLUDE = self.__get_array("GOOGLE_LABELS_EXCLUDE") + self.MONICA_LABELS_INCLUDE = self.__get_array("MONICA_LABELS_INCLUDE") + self.MONICA_LABELS_EXCLUDE = self.__get_array("MONICA_LABELS_EXCLUDE") + self.DATABASE_FILE = abspath(self._values["DATABASE_FILE"]) + self.GOOGLE_TOKEN_FILE = abspath(self._values["GOOGLE_TOKEN_FILE"]) + self.GOOGLE_CREDENTIALS_FILE = abspath(self._values["GOOGLE_CREDENTIALS_FILE"]) + except Exception as e: + raise ConfigError("Error parsing config, check syntax and required args!") from e + + def __get_boolean(self, key: str) -> bool: + """Get a boolean value from config as Python boolean""" + value_str = self._values[key].lower() + return value_str in ["true", "1", "t", "y"] + + def __get_array(self, key: str) -> list: + """Get an array from config as Python list""" + values_str = self._values.get(key, "") + if not values_str: + return [] + return [v.strip() for v in values_str.split(",")] diff --git a/helpers/DatabaseHelper.py b/helpers/DatabaseHelper.py new file mode 100644 index 0000000..5882ad6 --- /dev/null +++ b/helpers/DatabaseHelper.py @@ -0,0 +1,210 @@ +import sqlite3 +from datetime import datetime +from logging import Logger +from typing import Dict, List, Tuple, Union + +from helpers.Exceptions import DatabaseError + + +class DatabaseEntry: + """Represents a database row.""" + + def __init__( + self, + google_id: str = "", + monica_id: Union[str, int] = "", + google_full_name: str = "NULL", + monica_full_name: str = "NULL", + google_last_changed: str = "NULL", + monica_last_changed: str = "NULL", + ) -> None: + self.google_id = google_id + self.monica_id = str(monica_id) + self.google_full_name = google_full_name + self.monica_full_name = monica_full_name + self.google_last_changed = google_last_changed + self.monica_last_changed = monica_last_changed + + def __repr__(self) -> str: + """Returns the database entry as string""" + return ( + f"google_id: '{self.google_id}', " + f"monica_id: '{self.monica_id}', " + f"google_full_name: '{self.google_full_name}', " + f"monica_full_name: '{self.monica_full_name}', " + f"google_last_changed: '{self.google_last_changed}', " + f"monica_last_changed: '{self.monica_last_changed}'" + ) + + def get_insert_statement(self) -> Tuple[str, tuple]: + insert_sql = """ + INSERT INTO sync(googleId, monicaId, googleFullName, monicaFullName, + googleLastChanged, monicaLastChanged) + VALUES(?,?,?,?,?,?) + """ + return ( + insert_sql, + ( + self.google_id, + self.monica_id, + self.google_full_name, + self.monica_full_name, + self.google_last_changed, + self.monica_last_changed, + ), + ) + + +class Database: + """Handles all database related stuff.""" + + def __init__(self, log: Logger, filename: str) -> None: + self.log = log + self.connection = sqlite3.connect(filename) + self.cursor = self.connection.cursor() + self.__initialize_database() + + def delete_and_initialize(self) -> None: + """Deletes all tables from the database and creates new ones.""" + delete_sync_table_sql = """ + DROP TABLE IF EXISTS sync; + """ + delete_config_table_sql = """ + DROP TABLE IF EXISTS config; + """ + self.cursor.execute(delete_sync_table_sql) + self.cursor.execute(delete_config_table_sql) + self.connection.commit() + self.__initialize_database() + + def __initialize_database(self): + """Initializes the database with all tables.""" + create_sync_table_sql = """ + CREATE TABLE IF NOT EXISTS sync ( + googleId VARCHAR(50) NOT NULL UNIQUE, + monicaId VARCHAR(10) NOT NULL UNIQUE, + googleFullName VARCHAR(50) NULL, + monicaFullName VARCHAR(50) NULL, + googleLastChanged DATETIME NULL, + monicaLastChanged DATETIME NULL); + """ + create_config_table_sql = """ + CREATE TABLE IF NOT EXISTS config ( + googleNextSyncToken VARCHAR(100) NULL UNIQUE, + tokenLastUpdated DATETIME NULL); + """ + self.cursor.execute(create_sync_table_sql) + self.cursor.execute(create_config_table_sql) + self.connection.commit() + + def insert_data(self, database_entry: DatabaseEntry) -> None: + """Inserts the given data into the database.""" + self.cursor.execute(*database_entry.get_insert_statement()) + self.connection.commit() + + def update(self, database_entry: DatabaseEntry) -> None: + """Updates a dataset in the database. + Needs at least a Monica id OR a Google id and the related data.""" + unknown_arguments = "Unknown database update arguments!" + if database_entry.monica_id: + if database_entry.monica_full_name: + self.__update_full_name_by_monica_id( + database_entry.monica_id, database_entry.monica_full_name + ) + if database_entry.monica_last_changed: + self.__update_monica_last_changed( + database_entry.monica_id, database_entry.monica_last_changed + ) + else: + self.log.error(f"Failed to update database: {database_entry}") + raise DatabaseError(unknown_arguments) + if database_entry.google_id: + if database_entry.google_full_name: + self.__update_full_name_by_google_id( + database_entry.google_id, database_entry.google_full_name + ) + if database_entry.google_last_changed: + self.__update_google_last_changed( + database_entry.google_id, database_entry.google_last_changed + ) + else: + self.log.error(f"Failed to update database: {database_entry}") + raise DatabaseError(unknown_arguments) + if not database_entry.monica_id and not database_entry.google_id: + self.log.error(f"Failed to update database: {database_entry}") + raise DatabaseError(unknown_arguments) + + def __update_full_name_by_monica_id(self, monica_id: str, monica_full_name: str) -> None: + insert_sql = "UPDATE sync SET monicaFullName = ? WHERE monicaId = ?" + self.cursor.execute(insert_sql, (monica_full_name, str(monica_id))) + self.connection.commit() + + def __update_full_name_by_google_id(self, google_id: str, google_full_name: str) -> None: + insert_sql = "UPDATE sync SET googleFullName = ? WHERE googleId = ?" + self.cursor.execute(insert_sql, (google_full_name, google_id)) + self.connection.commit() + + def __update_monica_last_changed(self, monica_id: str, monica_last_changed: str) -> None: + insert_sql = "UPDATE sync SET monicaLastChanged = ? WHERE monicaId = ?" + self.cursor.execute(insert_sql, (monica_last_changed, str(monica_id))) + self.connection.commit() + + def __update_google_last_changed(self, google_id: str, google_last_changed: str) -> None: + insert_sql = "UPDATE sync SET googleLastChanged = ? WHERE googleId = ?" + self.cursor.execute(insert_sql, (google_last_changed, google_id)) + self.connection.commit() + + def find_by_id(self, google_id: str = None, monica_id: str = None) -> Union[DatabaseEntry, None]: + """Search for a contact row in the database. Returns None if not found. + Needs Google id OR Monica id""" + if monica_id: + row = self.__find_by_monica_id(str(monica_id)) + elif google_id: + row = self.__find_by_google_id(google_id) + else: + self.log.error(f"Unknown database find arguments: '{google_id}', '{monica_id}'") + raise DatabaseError("Unknown database find arguments") + if row: + return DatabaseEntry(*row) + return None + + def get_id_mapping(self) -> Dict[str, str]: + """Returns a dictionary with the { googleId : monicaId } mapping from the database""" + find_sql = "SELECT googleId,monicaId FROM sync" + self.cursor.execute(find_sql) + mapping = {google_id: str(monica_id) for google_id, monica_id in self.cursor.fetchall()} + return mapping + + def __find_by_monica_id(self, monica_id: str) -> List[str]: + find_sql = "SELECT * FROM sync WHERE monicaId=?" + self.cursor.execute(find_sql, (str(monica_id),)) + return self.cursor.fetchone() + + def __find_by_google_id(self, google_id: str) -> List[str]: + find_sql = "SELECT * FROM sync WHERE googleId=?" + self.cursor.execute(find_sql, (google_id,)) + return self.cursor.fetchone() + + def delete(self, google_id: str, monica_id: str) -> None: + """Deletes a row from the database. Needs Monica id AND Google id.""" + delete_sql = "DELETE FROM sync WHERE monicaId=? AND googleId=?" + self.cursor.execute(delete_sql, (str(monica_id), google_id)) + self.connection.commit() + + def get_google_next_sync_token(self) -> Union[str, None]: + """Returns the next sync token.""" + find_sql = "SELECT * FROM config WHERE ROWID=1" + self.cursor.execute(find_sql) + row = self.cursor.fetchone() + if row: + return row[0] + return None + + def update_google_next_sync_token(self, token: str) -> None: + """Updates the given token in the database.""" + timestamp = datetime.now().strftime("%F %H:%M:%S") + delete_sql = "DELETE FROM config WHERE ROWID=1" + insert_sql = "INSERT INTO config(googleNextSyncToken, tokenLastUpdated) VALUES(?,?)" + self.cursor.execute(delete_sql) + self.cursor.execute(insert_sql, (token, timestamp)) + self.connection.commit() diff --git a/helpers/Exceptions.py b/helpers/Exceptions.py new file mode 100644 index 0000000..69b0890 --- /dev/null +++ b/helpers/Exceptions.py @@ -0,0 +1,49 @@ +class SyncError(Exception): + """Exception class from which every exception in this library will derive. + It enables other projects using this library to catch all errors coming + from the library with a single "except" statement + """ + + pass + + +class MonicaFetchError(SyncError): + """The fetching of an outside Monica resource failed""" + + pass + + +class GoogleFetchError(SyncError): + """The fetching of an outside Google resource failed""" + + pass + + +class BadUserInput(SyncError): + """Wrong command switch used or otherwise wrong user input""" + + pass + + +class ConfigError(SyncError): + """Error reading config files or environment""" + + pass + + +class UserChoice(SyncError): + """Intended exit chosen by the user""" + + pass + + +class DatabaseError(SyncError): + """Something went wrong with the database (e.g. entry not found)""" + + pass + + +class InternalError(SyncError): + """An internal error that should not happen (fail save error)""" + + pass diff --git a/helpers/GoogleHelper.py b/helpers/GoogleHelper.py new file mode 100644 index 0000000..679f8fc --- /dev/null +++ b/helpers/GoogleHelper.py @@ -0,0 +1,660 @@ +import codecs +import os.path +import pickle +import time +from logging import Logger +from typing import Any, Dict, List, Tuple + +from google.auth.transport.requests import Request # type: ignore +from google.oauth2.credentials import Credentials # type: ignore +from google_auth_oauthlib.flow import InstalledAppFlow # type: ignore +from googleapiclient.discovery import Resource, build # type: ignore +from googleapiclient.errors import HttpError # type: ignore + +from helpers.DatabaseHelper import Database +from helpers.Exceptions import ConfigError, GoogleFetchError, InternalError + + +class Google: + """Handles all Google related (api) stuff.""" + + def __init__( + self, + log: Logger, + database_handler: Database, + credentials_file: str, + token_file: str, + include_labels: list, + exclude_labels: list, + is_interactive_sync: bool, + ) -> None: + self.log = log + self.credentials_file = credentials_file + self.token_file = token_file + self.include_labels = include_labels + self.exclude_labels = exclude_labels + self.is_interactive = is_interactive_sync + self.database = database_handler + self.api_requests = 0 + self.service = self.__build_service() + self.label_mapping = self.__get_label_mapping() + self.reverse_label_mapping = {label_id: name for name, label_id in self.label_mapping.items()} + self.contacts: List[dict] = [] + self.data_already_fetched = False + self.created_contacts: Dict[str, bool] = {} + self.sync_fields = ( + "addresses,biographies,birthdays,emailAddresses,genders," + "memberships,metadata,names,nicknames,occupations,organizations,phoneNumbers" + ) + self.update_fields = ( + "addresses,biographies,birthdays,clientData,emailAddresses," + "events,externalIds,genders,imClients,interests,locales,locations,memberships," + "miscKeywords,names,nicknames,occupations,organizations,phoneNumbers,relations," + "sipAddresses,urls,userDefined" + ) + + def __build_service(self) -> Resource: + creds: Credentials = None + # The file token.pickle stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + try: + if os.path.exists(self.token_file): + with open(self.token_file, "r") as base64_token: + creds_pickled = base64_token.read() + creds = pickle.loads(codecs.decode(creds_pickled.encode(), "base64")) + else: + raise ConfigError("Google token file not found!") + except UnicodeDecodeError: + # Maybe old pickling file, try to update + with open(self.token_file, "rb") as binary_token: + creds = pickle.load(binary_token) + creds_str = codecs.encode(pickle.dumps(creds), "base64").decode() + with open(self.token_file, "w") as token: + token.write(creds_str) + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + elif self.is_interactive: + prompt = "\nPlease visit this URL to authorize this application:\n{url}\n" + flow = InstalledAppFlow.from_client_secrets_file( + self.credentials_file, + scopes=["https://www.googleapis.com/auth/contacts"], + ) + creds = flow.run_console(prompt="consent", authorization_prompt_message=prompt) + else: + self.log.error("The 'token.pickle' file was not found or invalid!") + self.log.info( + "Please run the script using '-i' to acquire a new token (needs user input)." + ) + self.log.info( + f"Debug info: creds={bool(creds)}, valid={creds.valid}, " + f"expired={creds.expired}, refresh_token={bool(creds.refresh_token)}" + ) + print( + "Google token not found or invalid!\n" + "Please run '-i' to acquire a new one! (interactive)" + ) + raise ConfigError("Google token not found or invalid!") + # Save the credentials for the next run + creds_str = codecs.encode(pickle.dumps(creds), "base64").decode() + with open(self.token_file, "w") as token: + token.write(creds_str) + + service = build("people", "v1", credentials=creds) + return service + + def get_label_id(self, name: str, create_on_error: bool = True) -> str: + """Returns the Google label id for a given tag name. + Creates a new label if it has not been found.""" + if create_on_error: + return self.label_mapping.get(name, self.create_label(name)) + else: + return self.label_mapping.get(name, "") + + def get_label_name(self, label_string: str) -> str: + """Returns the Google label name for a given label id.""" + label_id = label_string.split("/")[1] + return self.reverse_label_mapping.get(label_string, label_id) + + def __filter_contacts_by_label(self, contact_list: List[dict]) -> List[dict]: + """Filters a contact list by include/exclude labels.""" + if self.include_labels: + return [ + contact + for contact in contact_list + if any( + [ + contact_label["contactGroupMembership"]["contactGroupId"] in self.include_labels + for contact_label in contact["memberships"] + ] + ) + and all( + [ + contact_label["contactGroupMembership"]["contactGroupId"] + not in self.exclude_labels + for contact_label in contact["memberships"] + ] + ) + ] + elif self.exclude_labels: + return [ + contact + for contact in contact_list + if all( + [ + contact_label["contactGroupMembership"]["contactGroupId"] + not in self.exclude_labels + for contact_label in contact["memberships"] + ] + ) + ] + else: + return contact_list + + def __filter_unnamed_contacts(self, contact_list: List[dict]) -> List[dict]: + """Exclude contacts without name.""" + filtered_contact_list = [] + for google_contact in contact_list: + # Look for empty names but keep deleted contacts (they too don't have a name) + is_deleted = google_contact.get("metadata", {}).get("deleted", False) + is_any_name = any(self.get_contact_names(google_contact)) + is_name_key_present = google_contact.get("names", False) + if (not is_any_name or not is_name_key_present) and not is_deleted: + self.log.info("Skipped the following unnamed google contact during sync:") + self.log.info(f"Contact details:\n{self.get_contact_as_string(google_contact)[2:-1]}") + else: + filtered_contact_list.append(google_contact) + if len(filtered_contact_list) != len(contact_list): + print("\nSkipped one or more unnamed google contacts, see log for details") + + return filtered_contact_list + + def get_contact_names( + self, google_contact: Dict[str, List[dict]] + ) -> Tuple[str, str, str, str, str, str, str]: + """Returns the given, family and display name of a Google contact.""" + names = google_contact.get("names", [{}])[0] + given_name: str = names.get("givenName", "") + family_name: str = names.get("familyName", "") + display_name: str = names.get("displayName", "") + middle_name: str = names.get("middleName", "") + prefix: str = names.get("honorificPrefix", "") + suffix: str = names.get("honorificSuffix", "") + nickname: str = google_contact.get("nicknames", [{}])[0].get("value", "") + return given_name, middle_name, family_name, display_name, prefix, suffix, nickname + + def get_contact_as_string(self, google_contact: dict) -> str: + """Get some content from a Google contact to identify it as a user + and return it as string.""" + search_keys = { + "names": ["displayName"], + "birthdays": ["value"], + "organizations": ["name", "department", "title"], + "addresses": ["formattedValue"], + "phoneNumbers": ["value"], + "emailAddresses": ["value"], + "memberships": ["contactGroupMembership"], + } + + contact_string = f"\n\nContact id: {google_contact['resourceName']}\n" + + for key, values in google_contact.items(): + if key not in search_keys: + continue + sub_string = "" + for value in values: + for sub_key, sub_value in value.items(): + if sub_key not in search_keys[key]: + continue + if sub_key == "contactGroupMembership": + sub_value = self.get_label_name(sub_value["contactGroupResourceName"]) + sub_value = sub_value.replace("\n", " ") + sub_string += f" {sub_key}: {sub_value}\n" + if sub_string: + contact_string += f"{key}:\n" + sub_string + + return contact_string + + def remove_contact_from_list(self, google_contact: dict) -> None: + """Removes a Google contact internally to avoid further processing + (e.g. if it has been deleted on both sides)""" + self.contacts.remove(google_contact) + + def get_contact(self, google_id: str) -> dict: + """Fetches a single contact by id from Google.""" + try: + # Check if contact is already fetched + if self.contacts: + google_contact_list = [ + c for c in self.contacts if str(c["resourceName"]) == str(google_id) + ] + if google_contact_list: + return google_contact_list[0] + + # Build GET parameters + parameters = { + "resourceName": google_id, + "personFields": self.sync_fields, + } + + # Fetch contact + result = self.service.people().get(**parameters).execute() + self.api_requests += 1 + + # Return contact + google_contact = self.__filter_contacts_by_label([result])[0] + google_contact = self.__filter_unnamed_contacts([result])[0] + self.contacts.append(google_contact) + return google_contact + + except HttpError as error: + if self.__is_slow_down_error(error): + return self.get_contact(google_id) + else: + msg = f"Failed to fetch Google contact '{google_id}': {str(error)}" + self.log.error(msg) + raise GoogleFetchError(msg) from error + + except IndexError as error: + msg = f"Contact processing of '{google_id}' not allowed by label filter" + self.log.info(msg) + raise InternalError(msg) from error + + except Exception as error: + msg = f"Failed to fetch Google contact '{google_id}': {str(error)}" + self.log.error(msg) + raise GoogleFetchError(msg) from error + + def __is_slow_down_error(self, error: HttpError) -> bool: + """Checks if the error is an quota exceeded error and slows down the requests if yes.""" + waiting_time = 60 + if "Quota exceeded" in str(error): + print(f"\nToo many Google requests, waiting {waiting_time} seconds...") + time.sleep(waiting_time) + return True + else: + return False + + def get_contacts(self, refetch_data: bool = False, **params) -> List[dict]: + """Fetches all contacts from Google if not already fetched.""" + # Build GET parameters + parameters = { + "resourceName": "people/me", + "pageSize": 1000, + "personFields": self.sync_fields, + "requestSyncToken": True, + **params, + } + + # Avoid multiple fetches + if self.data_already_fetched and not refetch_data: + return self.contacts + + # Start fetching + msg = "Fetching Google contacts..." + self.log.info(msg) + print(msg) + try: + self.__fetch_contacts(parameters) + except HttpError as error: + if "Sync token" in str(error): + msg = "Sync token expired or invalid. Fetching again without token (full sync)..." + self.log.warning(msg) + print("\n" + msg) + parameters.pop("syncToken") + self.__fetch_contacts(parameters) + elif self.__is_slow_down_error(error): + return self.get_contacts(refetch_data, **params) + else: + msg = "Failed to fetch Google contacts!" + self.log.error(msg) + raise GoogleFetchError(str(error)) from error + msg = "Finished fetching Google contacts" + self.log.info(msg) + print("\n" + msg) + self.data_already_fetched = True + return self.contacts + + def __fetch_contacts(self, parameters: dict) -> None: + contacts = [] + while True: + result = self.service.people().connections().list(**parameters).execute() + self.api_requests += 1 + next_page_token = result.get("nextPageToken", False) + contacts += result.get("connections", []) + if next_page_token: + parameters["pageToken"] = next_page_token + else: + self.contacts = self.__filter_contacts_by_label(contacts) + self.contacts = self.__filter_unnamed_contacts(contacts) + break + + next_sync_token = result.get("nextSyncToken", None) + if next_sync_token and self.database: + self.database.update_google_next_sync_token(next_sync_token) + + def __get_label_mapping(self) -> dict: + """Fetches all contact groups from Google (aka labels) and + returns a {name: id} mapping.""" + try: + # Get all contact groups + response = self.service.contactGroups().list().execute() + self.api_requests += 1 + groups = response.get("contactGroups", []) + + # Initialize mapping for all user groups and allowed system groups + label_mapping = { + group["name"]: group["resourceName"] + for group in groups + if group["groupType"] == "USER_CONTACT_GROUP" + or group["name"] in ["myContacts", "starred"] + } + + return label_mapping + except HttpError as error: + if self.__is_slow_down_error(error): + return self.__get_label_mapping() + else: + msg = "Failed to fetch Google labels!" + self.log.error(msg) + raise GoogleFetchError(str(error)) from error + + def delete_label(self, group_id) -> None: + """Deletes a contact group from Google (aka label). Does not delete assigned contacts.""" + try: + response = self.service.contactGroups().delete(resourceName=group_id).execute() + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + self.delete_label(group_id) + else: + reason = str(error) + msg = f"Failed to delete Google contact group. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + if response: + msg = f"Non-empty response received, please check carefully: {response}" + self.log.warning(msg) + print("\n" + msg) + + def create_label(self, label_name: str) -> str: + """Creates a new Google contacts label and returns its id.""" + # Search label and return if found + if label_name in self.label_mapping: + return self.label_mapping[label_name] + + # Create group object + new_group = {"contactGroup": {"name": label_name}} + + try: + # Upload group object + response = self.service.contactGroups().create(body=new_group).execute() + self.api_requests += 1 + + group_id = response.get("resourceName", "contactGroups/myContacts") + self.label_mapping.update({label_name: group_id}) + return group_id + + except HttpError as error: + if self.__is_slow_down_error(error): + return self.create_label(label_name) + else: + msg = "Failed to create Google label!" + self.log.error(msg) + raise GoogleFetchError(str(error)) from error + + def create_contact(self, data: dict) -> dict: + """Creates a given Google contact via api call and returns the created contact.""" + # Upload contact + try: + result = ( + self.service.people().createContact(personFields=self.sync_fields, body=data).execute() + ) + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.create_contact(data) + else: + reason = str(error) + msg = f"'{data['names'][0]}':Failed to create Google contact. Reason: {reason}" + self.log.error(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Process result + google_id = result["resourceName"] + name = result["names"][0]["displayName"] + self.created_contacts[google_id] = True + self.contacts.append(result) + self.log.info(f"'{name}': Contact with id '{google_id}' created successfully") + return result + + def update_contacts(self, data: List[dict]) -> List[dict]: + """Updates a given Google contact list via api call and returns the updated contacts.""" + assert len(data) < 200, "Too many contacts for batch update!" + if not data: + return [] + # Prepare body + body = { + "contacts": {contact["resourceName"]: contact for contact in data}, + "updateMask": self.update_fields, + "readMask": self.update_fields, + } + # Upload contacts + try: + results = self.service.people().batchUpdateContacts(body=body).execute() + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.update_contacts(data) + else: + reason = str(error) + msg = f"Failed to update Google contacts. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Process result + results = results["updateResult"].values() + contacts = [] + for item in results: + contact = item["person"] + google_id = contact.get("resourceName", "-") + name = contact.get("names", [{}])[0].get("displayName", "error") + if item["httpStatusCode"] != 200: + self.log.error(f"'{name}': Failed to update contact with id '{google_id}'!") + continue + self.log.info(f"'{name}': Contact with id '{google_id}' updated successfully") + contacts.append(contact) + return contacts + + def delete_contacts(self, data: Dict[str, str]) -> None: + """Deletes all given Google contacts list via api call.""" + assert len(data) < 500, "Too many contacts for batch delete!" + if not data: + return + # Prepare body + body = {"resourceNames": list(data)} + # Delete contacts + try: + self.service.people().batchDeleteContacts(body=body).execute() + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.delete_contacts(data) + else: + reason = str(error) + msg = f"Failed to delete Google contacts. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Finished + for google_id, display_name in data.items(): + self.log.info(f"'{display_name}': Contact with id '{google_id}' deleted successfully") + + def create_contacts(self, data: List[dict]) -> List[dict]: + """Creates a given Google contact list via api call and returns the created contacts.""" + assert len(data) < 200, "Too many contacts for batch create!" + if not data: + return [] + # Prepare body + body = { + "contacts": [{"contactPerson": contact} for contact in data], + "readMask": self.update_fields, + } + # Upload contacts + try: + results = self.service.people().batchCreateContacts(body=body).execute() + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.create_contacts(data) + else: + reason = str(error) + msg = f"Failed to create Google contacts. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Process result + contacts = [] + for item in results["createdPeople"]: + contact = item["person"] + google_id = contact.get("resourceName", "-") + name = contact.get("names", [{}])[0].get("displayName", "error") + if item["httpStatusCode"] != 200: + self.log.error(f"'{name}': Failed to create contact with id '{google_id}'!") + continue + self.log.info(f"'{name}': Contact with id '{google_id}' created successfully") + contacts.append(contact) + return contacts + + def update_contact(self, data: dict) -> dict: + """Updates a given Google contact via api call and returns the updated contact.""" + # Upload contact + try: + result = ( + self.service.people() + .updateContact( + resourceName=data["resourceName"], updatePersonFields=self.update_fields, body=data + ) + .execute() + ) + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.update_contact(data) + else: + reason = str(error) + msg = f"'{data['names'][0]}':Failed to update Google contact. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Process result + google_id = result.get("resourceName", "-") + name = result.get("names", [{}])[0].get("displayName", "error") + self.log.info(f"'{name}': Contact with id '{google_id}' updated successfully") + return result + + def delete_contact(self, google_id: str, display_name: str) -> None: + """Deletes a given Google contact via api call.""" + # Upload contact + try: + self.service.people().deleteContact(resourceName=google_id).execute() + self.api_requests += 1 + except HttpError as error: + if self.__is_slow_down_error(error): + return self.delete_contact(google_id, display_name) + else: + reason = str(error) + msg = f"'{display_name}':Failed to delete Google contact. Reason: {reason}" + self.log.warning(msg) + print("\n" + msg) + raise GoogleFetchError(reason) from error + + # Finished + self.log.info(f"'{display_name}': Contact with id '{google_id}' deleted successfully") + + +class GoogleContactUploadForm: + """Creates json form for creating Google contacts.""" + + def __init__( + self, + first_name: str = "", + last_name: str = "", + middle_name: str = "", + birthdate: dict = {}, + phone_numbers: List[str] = [], + career: dict = {}, + email_adresses: List[str] = [], + label_ids: List[str] = [], + addresses: List[dict] = [], + ) -> None: + self.data: Dict[str, List[Dict[str, Any]]] = { + "names": [{"familyName": last_name, "givenName": first_name, "middleName": middle_name}] + } + + if birthdate: + self.data["birthdays"] = [ + { + "date": { + "year": birthdate.get("year", 0), + "month": birthdate.get("month", 0), + "day": birthdate.get("day", 0), + } + } + ] + + if career: + self.data["organizations"] = [ + {"name": career.get("company", ""), "title": career.get("job", "")} + ] + + if addresses: + self.data["addresses"] = [ + { + "type": address.get("name", ""), + "streetAddress": address.get("street", ""), + "city": address.get("city", ""), + "region": address.get("province", ""), + "postalCode": address.get("postal_code", ""), + "country": address["country"].get("name", None) if address["country"] else None, + "countryCode": address["country"].get("iso", None) if address["country"] else None, + } + for address in addresses + ] + + if phone_numbers: + self.data["phoneNumbers"] = [ + { + "value": number, + "type": "other", + } + for number in phone_numbers + ] + + if email_adresses: + self.data["emailAddresses"] = [ + { + "value": email, + "type": "other", + } + for email in email_adresses + ] + + if label_ids: + self.data["memberships"] = [ + {"contactGroupMembership": {"contactGroupResourceName": label_id}} + for label_id in label_ids + ] + + def get_data(self) -> dict: + """Returns the Google contact form data.""" + return self.data diff --git a/helpers/MonicaHelper.py b/helpers/MonicaHelper.py new file mode 100644 index 0000000..bdc2cce --- /dev/null +++ b/helpers/MonicaHelper.py @@ -0,0 +1,685 @@ +import time +from logging import Logger +from typing import Dict, List + +import requests +from requests.models import Response + +from helpers.DatabaseHelper import Database, DatabaseEntry +from helpers.Exceptions import InternalError, MonicaFetchError + + +class Monica: + """Handles all Monica related (api) stuff.""" + + def __init__( + self, + log: Logger, + database_handler: Database, + token: str, + base_url: str, + create_reminders: bool, + include_labels: list, + exclude_labels: list, + ) -> None: + self.log = log + self.database = database_handler + self.base_url = base_url + self.include_labels = include_labels + self.exclude_labels = exclude_labels + self.header = {"Authorization": f"Bearer {token}"} + self.parameters = {"limit": 100} + self.is_data_already_fetched = False + self.contacts: List[dict] = [] + self.gender_mapping: Dict[str, str] = {} + self.contact_field_type_mapping: Dict[str, str] = {} + self.updated_contacts: Dict[str, bool] = {} + self.created_contacts: Dict[str, bool] = {} + self.deleted_contacts: Dict[str, bool] = {} + self.api_requests = 0 + self.create_reminders = create_reminders + + def __filter_contacts_by_label(self, contact_list: List[dict]) -> List[dict]: + """Filters a contact list by include/exclude labels.""" + if self.include_labels: + return [ + contact + for contact in contact_list + if any( + [contact_label["name"] in self.include_labels for contact_label in contact["tags"]] + ) + and all( + [ + contact_label["name"] not in self.exclude_labels + for contact_label in contact["tags"] + ] + ) + ] + elif self.exclude_labels: + return [ + contact + for contact in contact_list + if all( + [ + contact_label["name"] not in self.exclude_labels + for contact_label in contact["tags"] + ] + ) + ] + else: + return contact_list + + def update_statistics(self) -> None: + """Updates internal statistics for printing.""" + # A contact should only count as updated if it has not been created during sync + self.updated_contacts = { + key: value + for key, value in self.updated_contacts.items() + if key not in self.created_contacts + } + + def get_gender_mapping(self) -> dict: + """Fetches all genders from Monica and saves them to a dictionary.""" + # Only fetch if not present yet + if self.gender_mapping: + return self.gender_mapping + try: + while True: + # Get genders + response = requests.get( + self.base_url + "/genders", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + genders = response.json()["data"] + gender_mapping = {gender["type"]: gender["id"] for gender in genders} + self.gender_mapping = gender_mapping + return self.gender_mapping + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.error(f"Failed to fetch genders from Monica: {error}") + raise MonicaFetchError("Error fetching genders from Monica!") + + except Exception as e: + msg = f"Failed to fetch Monica genders (maybe connection issue): {str(e)}" + print("\n" + msg) + self.log.error(msg) + if response: + self.log.info(response.text) + raise MonicaFetchError(msg) from e + + def update_contact(self, monica_id: str, data: dict) -> None: + """Updates a given contact and its id via api call.""" + name = f"{data['first_name']} {data['last_name']}" + + # Remove Monica contact from contact list (add again after updated) + self.contacts = [c for c in self.contacts if str(c["id"]) != str(monica_id)] + + while True: + # Update contact + response = requests.put( + self.base_url + f"/contacts/{monica_id}", + headers=self.header, + params=self.parameters, + json=data, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + contact = response.json()["data"] + self.updated_contacts[monica_id] = True + self.contacts.append(contact) + name = contact["complete_name"] + self.log.info(f"'{name}' ('{monica_id}'): Contact updated successfully") + entry = DatabaseEntry( + monica_id=monica_id, + monica_last_changed=contact["updated_at"], + monica_full_name=contact["complete_name"], + ) + self.database.update(entry) + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.error( + f"'{name}' ('{monica_id}'): " + f"Error updating Monica contact: {error}. Does it exist?" + ) + self.log.error(f"Monica form data: {data}") + raise MonicaFetchError("Error updating Monica contact!") + + def delete_contact(self, monica_id: str, name: str) -> None: + """Deletes the contact with the given id from Monica and removes it from the internal list.""" + + while True: + # Delete contact + response = requests.delete( + self.base_url + f"/contacts/{monica_id}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.contacts = [c for c in self.contacts if str(c["id"]) != str(monica_id)] + self.deleted_contacts[monica_id] = True + self.log.info(f"'{name}' ('{monica_id}'): Contact deleted successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.error(f"'{name}' ('{monica_id}'): Failed to complete delete request: {error}") + raise MonicaFetchError("Error deleting Monica contact!") + + def create_contact(self, data: dict, reference_id: str) -> dict: + """Creates a given Monica contact via api call and returns the created contact.""" + while True: + # Create contact + response = requests.post( + self.base_url + "/contacts", headers=self.header, params=self.parameters, json=data + ) + self.api_requests += 1 + + # If successful + if response.status_code == 201: + contact = response.json()["data"] + self.created_contacts[contact["id"]] = True + self.contacts.append(contact) + self.log.info(f"'{reference_id}' ('{contact['id']}'): Contact created successfully") + return contact + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.info(f"'{reference_id}': Error creating Monica contact: {error}") + raise MonicaFetchError("Error creating Monica contact!") + + def get_contacts(self) -> List[dict]: + """Fetches all contacts from Monica if not already fetched.""" + try: + # Avoid multiple fetches + if self.is_data_already_fetched: + return self.contacts + + # Start fetching + max_page = "?" + page = 1 + contacts = [] + self.log.info("Fetching all Monica contacts...") + while True: + print(f"Fetching all Monica contacts (page {page} of {max_page})") + response = requests.get( + self.base_url + f"/contacts?page={page}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + # If successful + if response.status_code == 200: + data = response.json() + contacts += data["data"] + max_page = data["meta"]["last_page"] + if page == max_page: + self.contacts = self.__filter_contacts_by_label(contacts) + break + page += 1 + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + msg = f"Error fetching Monica contacts: {error}" + self.log.error(msg) + raise MonicaFetchError(msg) + self.is_data_already_fetched = True + msg = "Finished fetching Monica contacts" + self.log.info(msg) + print("\n" + msg) + return self.contacts + + except Exception as e: + msg = f"Failed to fetch Monica contacts (maybe connection issue): {str(e)}" + print("\n" + msg) + self.log.error(msg) + if response: + self.log.info(response.text) + raise MonicaFetchError(msg) from e + + def get_contact(self, monica_id: str) -> dict: + """Fetches a single contact by id from Monica.""" + try: + # Check if contact is already fetched + if self.contacts: + monica_contact_list = [c for c in self.contacts if str(c["id"]) == str(monica_id)] + if monica_contact_list: + return monica_contact_list[0] + + while True: + # Fetch contact + response = requests.get( + self.base_url + f"/contacts/{monica_id}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + monica_contact = response.json()["data"] + monica_contact = self.__filter_contacts_by_label([monica_contact])[0] + self.contacts.append(monica_contact) + return monica_contact + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError(error) + + except IndexError as e: + msg = f"Contact processing of '{monica_id}' not allowed by label filter" + self.log.error(msg) + print("\n" + msg) + raise InternalError(msg) from e + + except Exception as e: + msg1 = f"Failed to fetch Monica contact '{monica_id}': {str(e)}" + msg2 = "Database may be inconsistent, did you delete a Monica contact?" + self.log.error(msg1) + self.log.warning(msg2) + print("\n" + msg1 + "\n" + msg2) + raise MonicaFetchError(msg1) from e + + def get_notes(self, monica_id: str, name: str) -> List[dict]: + """Fetches all contact notes for a given Monica contact id via api call.""" + + while True: + # Get contact fields + response = requests.get( + self.base_url + f"/contacts/{monica_id}/notes", + headers=self.header, + params=self.parameters, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + monica_notes = response.json()["data"] + return monica_notes + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError(f"'{name}' ('{monica_id}'): Error fetching Monica notes: {error}") + + def add_note(self, data: dict, name: str) -> None: + """Creates a new note for a given contact id via api call.""" + # Initialization + monica_id = data["contact_id"] + + while True: + # Create address + response = requests.post( + self.base_url + "/notes", headers=self.header, params=self.parameters, json=data + ) + self.api_requests += 1 + + # If successful + if response.status_code == 201: + self.updated_contacts[monica_id] = True + note = response.json()["data"] + note_id = note["id"] + self.log.info(f"'{name}' ('{monica_id}'): Note '{note_id}' created successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError(f"'{name}' ('{monica_id}'): Error creating Monica note: {error}") + + def update_note(self, note_id: str, data: dict, name: str) -> None: + """Creates a new note for a given contact id via api call.""" + # Initialization + monica_id = data["contact_id"] + + while True: + # Create address + response = requests.put( + self.base_url + f"/notes/{note_id}", + headers=self.header, + params=self.parameters, + json=data, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + note = response.json()["data"] + note_id = note["id"] + self.log.info(f"'{name}' ('{monica_id}'): Note '{note_id}' updated successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError(f"'{name}' ('{monica_id}'): Error updating Monica note: {error}") + + def delete_note(self, note_id: str, monica_id: str, name: str) -> None: + """Creates a new note for a given contact id via api call.""" + + while True: + # Create address + response = requests.delete( + self.base_url + f"/notes/{note_id}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + self.log.info(f"'{name}' ('{monica_id}'): Note '{note_id}' deleted successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError(f"'{name}' ('{monica_id}'): Error deleting Monica note: {error}") + + def remove_tags(self, data: dict, monica_id: str, name: str) -> None: + """Removes all tags given by id from a given contact id via api call.""" + + while True: + # Create address + response = requests.post( + self.base_url + f"/contacts/{monica_id}/unsetTag", + headers=self.header, + params=self.parameters, + json=data, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + self.log.info( + f"'{name}' ('{monica_id}'): Label(s) with id {data['tags']} removed successfully" + ) + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error removing Monica labels: {error}" + ) + + def add_tags(self, data: dict, monica_id: str, name: str) -> None: + """Adds all tags given by name for a given contact id via api call.""" + + while True: + # Create address + response = requests.post( + self.base_url + f"/contacts/{monica_id}/setTags", + headers=self.header, + params=self.parameters, + json=data, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + self.log.info(f"'{name}' ('{monica_id}'): Labels {data['tags']} assigned successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error assigning Monica labels: {error}" + ) + + def update_career(self, monica_id: str, data: dict) -> None: + """Updates job title and company for a given contact id via api call.""" + # Initialization + contact = self.get_contact(monica_id) + name = contact["complete_name"] + + while True: + # Update contact + response = requests.put( + self.base_url + f"/contacts/{monica_id}/work", + headers=self.header, + params=self.parameters, + json=data, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + contact = response.json()["data"] + self.log.info(f"'{name}' ('{monica_id}'): Company and job title updated successfully") + entry = DatabaseEntry(monica_id=monica_id, monica_last_changed=contact["updated_at"]) + self.database.update(entry) + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.warning( + f"'{name}' ('{monica_id}'): Error updating Monica contact career info: {error}" + ) + + def delete_address(self, address_id: str, monica_id: str, name: str) -> None: + """Deletes an address for a given address id via api call.""" + while True: + # Delete address + response = requests.delete( + self.base_url + f"/addresses/{address_id}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + self.log.info(f"'{name}' ('{monica_id}'): Address '{address_id}' deleted successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error deleting address '{address_id}': {error}" + ) + + def create_address(self, data: dict, name: str) -> None: + """Creates an address for a given contact id via api call.""" + # Initialization + monica_id = data["contact_id"] + + while True: + # Create address + response = requests.post( + self.base_url + "/addresses", headers=self.header, params=self.parameters, json=data + ) + self.api_requests += 1 + + # If successful + if response.status_code == 201: + self.updated_contacts[monica_id] = True + address = response.json()["data"] + address_id = address["id"] + self.log.info(f"'{name}' ('{monica_id}'): Address '{address_id}' created successfully") + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error creating Monica address: {error}" + ) + + def get_contact_fields(self, monica_id: str, name: str) -> List[dict]: + """Fetches all contact fields (phone numbers, emails, etc.) + for a given Monica contact id via api call.""" + + while True: + # Get contact fields + response = requests.get( + self.base_url + f"/contacts/{monica_id}/contactfields", + headers=self.header, + params=self.parameters, + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + field_list = response.json()["data"] + return field_list + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error fetching Monica contact fields: {error}" + ) + + def get_contact_field_id(self, type_name: str) -> str: + """Returns the id for a Monica contact field.""" + # Fetch if not present yet + if not self.contact_field_type_mapping: + self.__get_contact_field_types() + + # Get contact field id + field_id = self.contact_field_type_mapping.get(type_name, None) + + # No id is a serious issue + if not field_id: + raise InternalError(f"Could not find an id for contact field type '{type_name}'") + + return field_id + + def __get_contact_field_types(self) -> dict: + """Fetches all contact field types from Monica and saves them to a dictionary.""" + + while True: + # Get genders + response = requests.get( + self.base_url + "/contactfieldtypes", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + contact_field_types = response.json()["data"] + contact_field_type_mapping = { + field["type"]: field["id"] for field in contact_field_types + } + self.contact_field_type_mapping = contact_field_type_mapping + return self.contact_field_type_mapping + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + self.log.error(f"Failed to fetch contact field types from Monica: {error}") + raise MonicaFetchError("Error fetching contact field types from Monica!") + + def create_contact_field(self, monica_id: str, data: dict, name: str) -> None: + """Creates a contact field (phone number, email, etc.) + for a given Monica contact id via api call.""" + + while True: + # Create contact field + response = requests.post( + self.base_url + "/contactfields", headers=self.header, params=self.parameters, json=data + ) + self.api_requests += 1 + + # If successful + if response.status_code == 201: + self.updated_contacts[monica_id] = True + contact_field = response.json()["data"] + field_id = contact_field["id"] + type_desc = contact_field["contact_field_type"]["type"] + self.log.info( + f"'{name}' ('{monica_id}'): " + f"Contact field '{field_id}' ({type_desc}) created successfully" + ) + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error creating Monica contact field: {error}" + ) + + def delete_contact_field(self, field_id: str, monica_id: str, name: str) -> None: + """Updates a contact field (phone number, email, etc.) + for a given Monica contact id via api call.""" + + while True: + # Delete contact field + response = requests.delete( + self.base_url + f"/contactfields/{field_id}", headers=self.header, params=self.parameters + ) + self.api_requests += 1 + + # If successful + if response.status_code == 200: + self.updated_contacts[monica_id] = True + self.log.info( + f"'{name}' ('{monica_id}'): Contact field '{field_id}' deleted successfully" + ) + return + else: + error = response.json()["error"]["message"] + if self.__is_slow_down_error(response, error): + continue + raise MonicaFetchError( + f"'{name}' ('{monica_id}'): Error deleting Monica contact field" + f" '{field_id}': {error}" + ) + + def __is_slow_down_error(self, response: Response, error: str) -> bool: + """Checks if the error is an rate limiter error and slows down the requests if yes.""" + + if "Too many attempts, please slow down the request" in error: + sec_str = str(response.headers.get("Retry-After")) + sec = int(sec_str) + print(f"\nToo many Monica requests, waiting {sec} seconds...") + time.sleep(sec) + return True + else: + return False + + +class MonicaContactUploadForm: + """Creates json form for creating or updating Monica contacts.""" + + def __init__(self, monica: Monica, first_name: str, **form_data) -> None: + gender_type = form_data.get("gender_type", "O") + gender_mapping = monica.get_gender_mapping() + gender_id = gender_mapping.get(gender_type, None) + self.data = { + "first_name": first_name, + "last_name": form_data.get("last_name", None), + "nickname": form_data.get("nick_name", None), + "middle_name": form_data.get("middle_name", None), + "gender_id": gender_id, + "birthdate_day": form_data.get("birthdate_day", None), + "birthdate_month": form_data.get("birthdate_month", None), + "birthdate_year": form_data.get("birthdate_year", None), + "birthdate_is_age_based": form_data.get("is_birthdate_age_based", False), + "deceased_date_add_reminder": form_data.get("create_reminders", True), + "birthdate_add_reminder": form_data.get("create_reminders", True), + "is_birthdate_known": form_data.get("is_birthdate_known", False), + "is_deceased": form_data.get("is_deceased", False), + "is_deceased_date_known": form_data.get("is_deceased_date_known", False), + "deceased_date_day": form_data.get("deceased_day", None), + "deceased_date_month": form_data.get("deceased_month", None), + "deceased_date_year": form_data.get("deceased_year", None), + "deceased_date_is_age_based": form_data.get("deceased_age_based", None), + } diff --git a/helpers/SyncHelper.py b/helpers/SyncHelper.py new file mode 100644 index 0000000..db318e1 --- /dev/null +++ b/helpers/SyncHelper.py @@ -0,0 +1,1386 @@ +import os +from datetime import datetime +from logging import Logger +from typing import Any, Dict, List, Tuple, Union + +from helpers.DatabaseHelper import Database, DatabaseEntry +from helpers.Exceptions import BadUserInput, DatabaseError, UserChoice +from helpers.GoogleHelper import Google, GoogleContactUploadForm +from helpers.MonicaHelper import Monica, MonicaContactUploadForm + + +class Sync: + """Handles all syncing and merging issues with Google, Monica and the database.""" + + def __init__( + self, + log: Logger, + database_handler: Database, + monica_handler: Monica, + google_handler: Google, + is_sync_back_to_google: bool, + is_check_database: bool, + is_delete_monica_contacts_on_sync: bool, + is_street_reversal_on_address_sync: bool, + syncing_fields: list, + ) -> None: + self.log = log + self.start_time = datetime.now() + self.monica = monica_handler + self.google = google_handler + self.database = database_handler + self.mapping = self.database.get_id_mapping() + self.reverse_mapping = {monica_id: google_id for google_id, monica_id in self.mapping.items()} + self.next_sync_token = self.database.get_google_next_sync_token() + self.is_sync_back = is_sync_back_to_google + self.is_check = is_check_database + self.is_delete_monica_contacts = is_delete_monica_contacts_on_sync + self.is_street_reversal = is_street_reversal_on_address_sync + self.syncing_fields = set(syncing_fields) + self.skip_creation_prompt = False + + def __update_mapping(self, google_id: str, monica_id: str) -> None: + """Updates the internal Google <-> Monica id mapping dictionary.""" + self.mapping.update({google_id: monica_id}) + self.reverse_mapping.update({monica_id: google_id}) + + def start_sync(self, sync_type: str = "") -> None: + """Starts the next sync cycle depending on the requested type + and the database data.""" + if sync_type == "initial": + # Initial sync requested + self.__initial_sync() + elif not self.mapping: + # There is no sync database. Initial sync is needed for all other sync types + msg = "No sync database found, please do a initial sync first!" + self.log.info(msg) + print(msg + "\n") + raise BadUserInput("Initial sync needed!") + elif sync_type == "full": + # As this is a full sync, get all contacts at once to save time + self.monica.get_contacts() + # Full sync requested so dont use database timestamps here + self.__sync("full", is_date_based_sync=False) + elif sync_type == "delta" and not self.next_sync_token: + # Delta sync requested but no sync token found + msg = "No sync token found, delta sync not possible. Doing (fast) full sync instead..." + self.log.info(msg) + print(msg + "\n") + # Do a full sync with database timestamp comparison (fast) + self.__sync("full") + elif sync_type == "delta": + # Delta sync requested + self.__sync("delta") + elif sync_type == "syncBack": + # Sync back to Google requested + self.__sync_back() + + # Print statistics + self.__print_sync_statistics() + + if self.is_check: + # Database check requested + self.check_database() + + def __initial_sync(self) -> None: + """Builds the syncing database and starts a full sync. Needs user interaction!""" + self.database.delete_and_initialize() + self.mapping.clear() + self.__build_sync_database() + print("\nThe following fields will be overwritten with Google data:") + for field in self.syncing_fields - {"notes"}: + print(f"- {field}") + print("Start full sync now?") + print("\t0: No (abort initial sync)") + print("\t1: Yes") + choice = self.__get_user_input(allowed_nums=[0, 1]) + if not choice: + raise UserChoice("Sync aborted by user choice") + self.__sync("full", is_date_based_sync=False) + + def __delete_monica_contact(self, google_contact: dict) -> None: + """Removes a Monica contact given a corresponding Google contact.""" + try: + # Initialization + google_id = google_contact["resourceName"] + entry = self.database.find_by_id(google_id=google_id) + if not entry: + raise DatabaseError(f"No database entry for deleted google contact '{google_id}' found!") + + self.log.info( + f"'{entry.google_full_name}' ('{google_id}'): " + "Found deleted Google contact. Removing database entry..." + ) + + # Try to delete the corresponding contact + if self.is_delete_monica_contacts: + self.log.info( + f"'{entry.monica_full_name}' ('{entry.monica_id}'): Deleting Monica contact..." + ) + self.monica.delete_contact(entry.monica_id, entry.monica_full_name) + self.database.delete(google_id, entry.monica_id) + self.mapping.pop(google_id) + msg = f"'{entry.google_full_name}' ('{google_id}'): database entry removed successfully" + self.log.info(msg) + except Exception: + name = entry.google_full_name if entry else "" + msg = f"'{name}' ('{google_id}'): Failed removing Monica contact or database entry!" + self.log.error(msg) + print(msg) + + def __sync(self, sync_type: str, is_date_based_sync: bool = True) -> None: + """Fetches every contact from Google and Monica and does a full sync.""" + # Initialization + msg = f"Starting {sync_type} sync..." + self.log.info(msg) + print("\n" + msg) + if sync_type == "delta": + google_contacts = self.google.get_contacts(syncToken=self.next_sync_token) + else: + google_contacts = self.google.get_contacts() + contact_count = len(google_contacts) + + # If Google hasn't returned some data + if not google_contacts: + msg = "No (changed) Google contacts found!" + self.log.info(msg) + print("\n" + msg) + + # Process every Google contact + for num, google_contact in enumerate(google_contacts): + print(f"Processing Google contact {num+1} of {contact_count}") + + # Delete Monica contact if Google contact was deleted (if chosen by user; delta sync only) + is_deleted = google_contact.get("metadata", {}).get("deleted", False) + if is_deleted: + self.__delete_monica_contact(google_contact) + # Skip further processing + continue + + entry = self.database.find_by_id(google_id=google_contact["resourceName"]) + + # Create a new Google contact in the database if there's nothing yet + if not entry: + # Create a new Google contact in the database if there's nothing yet + google_id = google_contact["resourceName"] + g_contact_display_name = self.google.get_contact_names(google_contact)[3] + msg = ( + f"'{g_contact_display_name}' ('{google_id}'): " + "No Monica id found': Creating new Monica contact..." + ) + self.log.info(msg) + print("\n" + msg) + + # Create new Monica contact + monica_contact = self.create_monica_contact(google_contact) + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + "New Monica contact created" + ) + self.log.info(msg) + print(msg) + + # Update database and mapping + new_database_entry = DatabaseEntry( + google_contact["resourceName"], + monica_contact["id"], + g_contact_display_name, + monica_contact["complete_name"], + google_contact["metadata"]["sources"][0]["updateTime"], + monica_contact["updated_at"], + ) + self.database.insert_data(new_database_entry) + self.__update_mapping(google_contact["resourceName"], str(monica_contact["id"])) + msg = ( + f"'{google_contact['resourceName']}' <-> '{monica_contact['id']}': " + "New sync connection added" + ) + self.log.info(msg) + + # Sync additional details + self.__sync_details(google_contact, monica_contact) + + # Proceed with next contact + continue + + # Skip all contacts which have not changed + # according to the database lastChanged date (if present) + contact_timestamp = google_contact["metadata"]["sources"][0]["updateTime"] + database_date = self.__convert_google_timestamp(entry.google_last_changed) + contact_date = self.__convert_google_timestamp(contact_timestamp) + if is_date_based_sync and database_date == contact_date: + continue + + # Get Monica contact by id + monica_contact = self.monica.get_contact(entry.monica_id) + # Merge name, birthday and deceased date and update them + self.__merge_and_update_nbd(monica_contact, google_contact) + + # Update Google contact last changed date in the database + google_last_changed = google_contact["metadata"]["sources"][0]["updateTime"] + updated_entry = DatabaseEntry( + google_id=google_contact["resourceName"], + google_full_name=self.google.get_contact_names(google_contact)[3], + google_last_changed=google_last_changed, + ) + self.database.update(updated_entry) + + # Refresh Monica data (could have changed) + monica_contact = self.monica.get_contact(entry.monica_id) + + # Sync additional details + self.__sync_details(google_contact, monica_contact) + + # Finished + msg = f"{sync_type.capitalize()} sync finished!" + self.log.info(msg) + print("\n" + msg) + + # Sync lonely Monica contacts back to Google if chosen by user + if self.is_sync_back: + self.__sync_back() + + def __sync_details(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs additional details, such as company, jobtitle, labels, + address, phone numbers, emails, notes, contact picture, etc.""" + if "career" in self.syncing_fields: + # Sync career info + self.__sync_career_info(google_contact, monica_contact) + + if "address" in self.syncing_fields: + # Sync address info + self.__sync_address(google_contact, monica_contact) + + if "phone" in self.syncing_fields or "email" in self.syncing_fields: + # Sync phone and email + self.__sync_phone_email(google_contact, monica_contact) + + if "labels" in self.syncing_fields: + # Sync labels + self.__sync_labels(google_contact, monica_contact) + + if "notes" in self.syncing_fields: + # Sync notes if not existent at Monica + self.__sync_notes(google_contact, monica_contact) + + def __sync_notes(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs Google contact notes if there is no note present at Monica.""" + monica_notes = self.monica.get_notes(monica_contact["id"], monica_contact["complete_name"]) + try: + identifier = "\n\n*This note is synced from your Google contacts. Do not edit here.*" + if google_contact.get("biographies", []): + # Get Google note + google_note = { + "body": google_contact["biographies"][0].get("value", "").strip(), + "contact_id": monica_contact["id"], + "is_favorited": False, + } + # Convert normal newlines to markdown newlines + google_note["body"] = google_note["body"].replace("\n", " \n") + + # Update or create the Monica note + self.__update_or_create_note(monica_notes, google_note, identifier, monica_contact) + + elif monica_notes: + for monica_note in monica_notes: + if identifier in monica_note["body"]: + # Found identifier, delete this note + self.monica.delete_note( + monica_note["id"], + monica_note["contact"]["id"], + monica_contact["complete_name"], + ) + break + + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error creating Monica note: {str(e)}" + ) + self.log.warning(msg) + + def __update_or_create_note( + self, + monica_notes: List[dict], + google_note: Dict[str, Any], + identifier: str, + monica_contact: dict, + ) -> None: + """Updates a note at Monica or creates it if it does not exist""" + updated = False + if monica_notes: + for monica_note in monica_notes: + if monica_note["body"] == google_note["body"]: + # If there is a note with the same content update it and add the identifier + google_note["body"] += identifier + self.monica.update_note( + monica_note["id"], google_note, monica_contact["complete_name"] + ) + updated = True + break + elif identifier in monica_note["body"]: + # Found identifier, update this note if changed + google_note["body"] += identifier + if monica_note["body"] != google_note["body"]: + self.monica.update_note( + monica_note["id"], google_note, monica_contact["complete_name"] + ) + updated = True + break + if not updated: + # No note with same content or identifier found so create a new one + google_note["body"] += identifier + self.monica.add_note(google_note, monica_contact["complete_name"]) + + def __sync_labels(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs Google contact labels/groups/tags.""" + try: + # Get google labels information + google_labels = [ + self.google.get_label_name(label["contactGroupMembership"]["contactGroupResourceName"]) + for label in google_contact.get("memberships", []) + ] + + # Remove tags if not present in Google contact + remove_list = [ + label["id"] for label in monica_contact["tags"] if label["name"] not in google_labels + ] + if remove_list: + self.monica.remove_tags( + {"tags": remove_list}, monica_contact["id"], monica_contact["complete_name"] + ) + + # Update labels if necessary + monica_labels = [ + label["name"] for label in monica_contact["tags"] if label["name"] in google_labels + ] + if sorted(google_labels) != sorted(monica_labels): + self.monica.add_tags( + {"tags": google_labels}, monica_contact["id"], monica_contact["complete_name"] + ) + + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error updating Monica contact labels: {str(e)}" + ) + self.log.warning(msg) + + def __sync_phone_email(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs phone and email fields.""" + monica_contact_fields = self.monica.get_contact_fields( + monica_contact["id"], monica_contact["complete_name"] + ) + if "email" in self.syncing_fields: + self.__sync_email(google_contact, monica_contact, monica_contact_fields) + if "phone" in self.syncing_fields: + self.__sync_phone(google_contact, monica_contact, monica_contact_fields) + + def __sync_email( + self, google_contact: dict, monica_contact: dict, monica_contact_fields: List[dict] + ) -> None: + """Syncs email fields.""" + try: + # Email processing + monica_contact_emails = [ + field + for field in monica_contact_fields + if field["contact_field_type"]["type"] == "email" + ] + google_contact_emails = google_contact.get("emailAddresses", []) + + if not google_contact_emails: + # There may be only Monica data: Delete emails + for monica_email in monica_contact_emails: + self.monica.delete_contact_field( + monica_email["id"], monica_contact["id"], monica_contact["complete_name"] + ) + return + + google_emails = [ + { + "contact_field_type_id": self.monica.get_contact_field_id("email"), + "data": email["value"].strip(), + "contact_id": monica_contact["id"], + } + for email in google_contact_emails + ] + + if not monica_contact_emails: + # There is only Google data: Create emails + for google_email in google_emails: + self.monica.create_contact_field( + monica_contact["id"], google_email, monica_contact["complete_name"] + ) + return + + # There is Google and Monica data: Check and recreate emails + for monica_email in monica_contact_emails: + # Check if there are emails to be deleted + if monica_email["content"] in [google_email["data"] for google_email in google_emails]: + continue + else: + self.monica.delete_contact_field( + monica_email["id"], monica_contact["id"], monica_contact["complete_name"] + ) + for google_email in google_emails: + # Check if there are emails to be created + if google_email["data"] in [ + monica_email["content"] for monica_email in monica_contact_emails + ]: + continue + else: + self.monica.create_contact_field( + monica_contact["id"], google_email, monica_contact["complete_name"] + ) + + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error updating Monica contact email: {str(e)}" + ) + self.log.warning(msg) + + def __sync_phone( + self, google_contact: dict, monica_contact: dict, monica_contact_fields: List[dict] + ) -> None: + """Syncs phone fields.""" + try: + # Phone number processing + monica_contact_phones = [ + field + for field in monica_contact_fields + if field["contact_field_type"]["type"] == "phone" + ] + google_contact_phones = google_contact.get("phoneNumbers", []) + + if not google_contact_phones: + # No Google data: Delete all Monica contact phone numbers + for monica_phone in monica_contact_phones: + self.monica.delete_contact_field( + monica_phone["id"], monica_contact["id"], monica_contact["complete_name"] + ) + return + + google_phones = [ + { + "contact_field_type_id": self.monica.get_contact_field_id("phone"), + "data": number["value"].strip(), + "contact_id": monica_contact["id"], + } + for number in google_contact_phones + ] + if not monica_contact_phones: + # There is only Google data: Create Monica phone numbers + for google_phone in google_phones: + self.monica.create_contact_field( + monica_contact["id"], google_phone, monica_contact["complete_name"] + ) + return + + # There is Google and Monica data: Check and recreate phone numbers + for monica_phone in monica_contact_phones: + # Check if there are phone numbers to be deleted + if monica_phone["content"] in [google_phone["data"] for google_phone in google_phones]: + continue + else: + self.monica.delete_contact_field( + monica_phone["id"], monica_contact["id"], monica_contact["complete_name"] + ) + for google_phone in google_phones: + # Check if there are phone numbers to be created + if google_phone["data"] in [ + monica_phone["content"] for monica_phone in monica_contact_phones + ]: + continue + else: + self.monica.create_contact_field( + monica_contact["id"], google_phone, monica_contact["complete_name"] + ) + + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error updating Monica contact phone: {str(e)}" + ) + self.log.warning(msg) + + def __sync_career_info(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs company and job title fields.""" + try: + is_monica_data_present = bool( + monica_contact["information"]["career"]["job"] + or monica_contact["information"]["career"]["company"] + ) + is_google_data_present = bool(google_contact.get("organizations", False)) + if is_google_data_present or is_monica_data_present: + # Get google career information + company = google_contact.get("organizations", [{}])[0].get("name", "").strip() + department = google_contact.get("organizations", [{}])[0].get("department", "").strip() + if department: + department = f"; {department}" + job = google_contact.get("organizations", [{}])[0].get("title", None) + google_data = { + "job": job.strip() if job else None, + "company": company + department if company or department else None, + } + # Get monica career information + monica_data = { + "job": monica_contact["information"]["career"].get("job", None), + "company": monica_contact["information"]["career"].get("company", None), + } + + # Compare and update if necessary + if google_data != monica_data: + self.monica.update_career(monica_contact["id"], google_data) + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error updating Monica contact career: {str(e)}" + ) + self.log.warning(msg) + + def __sync_address(self, google_contact: dict, monica_contact: dict) -> None: + """Syncs all address fields.""" + try: + google_address_list = self.__get_google_addresses(google_contact, monica_contact["id"]) + monica_address_list = self.__get_monica_addresses(monica_contact) + + if not google_address_list: + # Delete all Monica addresses + for element in monica_address_list: + for address_id, _ in element.items(): + self.monica.delete_address( + address_id, monica_contact["id"], monica_contact["complete_name"] + ) + return + + # Create list for comparison + monica_plain_address_list = [ + monica_address for item in monica_address_list for monica_address in item.values() + ] + # Do a complete comparison + addresses_are_equal = [ + google_address in monica_plain_address_list for google_address in google_address_list + ] + if all(addresses_are_equal): + # All addresses are equal, nothing to do + return + + # Delete all Monica addresses and create new ones afterwards + # Safest way, I don't want to code more deeper comparisons and update functions + for element in monica_address_list: + for address_id, _ in element.items(): + self.monica.delete_address( + address_id, monica_contact["id"], monica_contact["complete_name"] + ) + + # All old Monica data (if existed) have been cleaned now, proceed with address creation + for google_address in google_address_list: + self.monica.create_address(google_address, monica_contact["complete_name"]) + + except Exception as e: + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Error updating Monica addresses: {str(e)}" + ) + self.log.warning(msg) + + def __get_monica_addresses(self, monica_contact: dict) -> List[dict]: + """Get all addresses from a Monica contact""" + if not monica_contact.get("addresses", False): + return [] + # Get Monica data + monica_address_list = [] + for addr in monica_contact.get("addresses", []): + monica_address_list.append( + { + addr["id"]: { + "name": addr["name"], + "street": addr["street"], + "city": addr["city"], + "province": addr["province"], + "postal_code": addr["postal_code"], + "country": addr["country"].get("iso", None) if addr["country"] else None, + "contact_id": monica_contact["id"], + } + } + ) + return monica_address_list + + def __get_google_addresses(self, google_contact: dict, monica_id: str) -> List[dict]: + """Get all addresses from a Google contact""" + if not google_contact.get("addresses", False): + return [] + # Get Google data + google_address_list = [] + for addr in google_contact.get("addresses", []): + # None type is important for comparison, empty string won't work here + name = None + street = None + city = None + province = None + postal_code = None + country_code = None + street = addr.get("streetAddress", "").replace("\n", " ").strip() or None + if self.is_street_reversal: + # Street reversal: from '13 Auenweg' to 'Auenweg 13' + try: + if street and street[0].isdigit(): + street = (f'{street[street.index(" ")+1:]} {street[:street.index(" ")]}').strip() + except Exception: + msg = f"Street reversal failed for '{street}'" + self.log.warning(msg) + print(msg) + + # Get (extended) city + city = f'{addr.get("city", "")} {addr.get("extendedAddress", "")}'.strip() or None + # Get other details + province = addr.get("region", None) + postal_code = addr.get("postalCode", None) + country_code = addr.get("countryCode", None) + # Name can not be empty + name = addr.get("formattedType", None) or "Other" + # Do not sync empty addresses + if not any([street, city, province, postal_code, country_code]): + continue + google_address_list.append( + { + "name": name, + "street": street, + "city": city, + "province": province, + "postal_code": postal_code, + "country": country_code, + "contact_id": monica_id, + } + ) + return google_address_list + + def __build_sync_database(self) -> None: + """Builds a Google <-> Monica 1:1 contact id mapping and saves it to the database.""" + # Initialization + conflicts = [] + google_contacts = self.google.get_contacts() + self.monica.get_contacts() + contact_count = len(google_contacts) + msg = "Building sync database..." + self.log.info(msg) + print("\n" + msg) + + # Process every Google contact + for num, google_contact in enumerate(google_contacts): + print(f"Processing Google contact {num+1} of {contact_count}") + # Try non-interactive search first + monica_id = self.__simple_monica_id_search(google_contact) + if not monica_id: + # Non-interactive search failed, try interactive search next + conflicts.append(google_contact) + + # Process all conflicts + if len(conflicts): + msg = f"Found {len(conflicts)} possible conflicts, starting resolving procedure..." + self.log.info(msg) + print("\n" + msg) + for google_contact in conflicts: + # Do a interactive search with user interaction next + monica_id = self.__interactive_monica_id_search(google_contact) + assert monica_id, "Could not create a Monica contact. Sync aborted." + + # Finished + msg = "Sync database built!" + self.log.info(msg) + print("\n" + msg) + + def __sync_back(self) -> None: + """Sync lonely Monica contacts back to Google by creating a new contact there.""" + msg = "Starting sync back..." + self.log.info(msg) + print("\n" + msg) + monica_contacts = self.monica.get_contacts() + contact_count = len(monica_contacts) + + # Process every Monica contact + for num, monica_contact in enumerate(monica_contacts): + print(f"Processing Monica contact {num+1} of {contact_count}") + + # If there the id isn't in the database: create a new Google contact and upload + if str(monica_contact["id"]) not in self.mapping.values(): + # Create Google contact + google_contact = self.create_google_contact(monica_contact) + if not google_contact: + msg = ( + f"'{monica_contact['complete_name']}': " + "Error encountered at creating new Google contact. Skipping..." + ) + self.log.warning(msg) + print(msg) + continue + g_contact_display_name = self.google.get_contact_names(google_contact)[3] + + # Update database and mapping + database_entry = DatabaseEntry( + google_contact["resourceName"], + monica_contact["id"], + g_contact_display_name, + monica_contact["complete_name"], + ) + self.database.insert_data(database_entry) + msg = ( + f"'{g_contact_display_name}' ('{google_contact['resourceName']}'): " + "New google contact created (sync back)" + ) + print("\n" + msg) + self.log.info(msg) + self.__update_mapping(google_contact["resourceName"], str(monica_contact["id"])) + msg = ( + f"'{google_contact['resourceName']}' <-> '{monica_contact['id']}': " + "New sync connection added" + ) + self.log.info(msg) + + if not self.google.created_contacts: + msg = "No contacts for sync back found" + self.log.info(msg) + print("\n" + msg) + + # Finished + msg = "Sync back finished!" + self.log.info(msg) + print(msg) + + def __print_sync_statistics(self) -> None: + """Prints and logs a pretty sync statistic of the last sync.""" + self.monica.update_statistics() + tme = str(datetime.now() - self.start_time).split(".")[0] + "h" + gac = str(self.google.api_requests) + (8 - len(str(self.google.api_requests))) * " " + mac = str(self.monica.api_requests) + (8 - len(str(self.monica.api_requests))) * " " + mcc = ( + str(len(self.monica.created_contacts)) + + (8 - len(str(len(self.monica.created_contacts)))) * " " + ) + mcu = ( + str(len(self.monica.updated_contacts)) + + (8 - len(str(len(self.monica.updated_contacts)))) * " " + ) + mcd = ( + str(len(self.monica.deleted_contacts)) + + (8 - len(str(len(self.monica.deleted_contacts)))) * " " + ) + gcc = ( + str(len(self.google.created_contacts)) + + (8 - len(str(len(self.google.created_contacts)))) * " " + ) + msg = ( + "\n" + f"Sync statistics: \n" + f"+-------------------------------------+\n" + f"| Syncing time: {tme } |\n" + f"| Google api calls used: {gac } |\n" + f"| Monica api calls used: {mac } |\n" + f"| Monica contacts created: {mcc } |\n" + f"| Monica contacts updated: {mcu } |\n" + f"| Monica contacts deleted: {mcd } |\n" + f"| Google contacts created: {gcc } |\n" + f"+-------------------------------------+" + ) + print(msg) + self.log.info(msg) + + def create_google_contact(self, monica_contact: dict) -> dict: + """Creates a new Google contact from a given Monica contact and returns it.""" + # Get names (no nickname) + first_name = monica_contact["first_name"] or "" + last_name = monica_contact["last_name"] or "" + full_name = monica_contact["complete_name"] or "" + nickname = monica_contact["nickname"] or "" + middle_name = self.__get_monica_middle_name(first_name, last_name, nickname, full_name) + + # Get birthday details (age based birthdays are not supported by Google) + birthday = {} + birthday_timestamp = monica_contact["information"]["dates"]["birthdate"]["date"] + is_age_based = monica_contact["information"]["dates"]["birthdate"]["is_age_based"] + if birthday_timestamp and not is_age_based: + is_year_unknown = monica_contact["information"]["dates"]["birthdate"]["is_year_unknown"] + date = self.__convert_monica_timestamp(birthday_timestamp) + if not is_year_unknown: + birthday.update({"year": date.year}) + birthday.update({"month": date.month, "day": date.day}) + + # Get addresses + addresses = monica_contact["addresses"] if "address" in self.syncing_fields else [] + + # Get career info if exists + career = { + key: value + for key, value in monica_contact["information"]["career"].items() + if value and "career" in self.syncing_fields + } + + # Get phone numbers and email addresses + if "phone" in self.syncing_fields or "email" in self.syncing_fields: + monica_contact_fields = self.monica.get_contact_fields( + monica_contact["id"], monica_contact["complete_name"] + ) + # Get email addresses + emails = [ + field["content"] + for field in monica_contact_fields + if field["contact_field_type"]["type"] == "email" and "email" in self.syncing_fields + ] + # Get phone numbers + phone_numbers = [ + field["content"] + for field in monica_contact_fields + if field["contact_field_type"]["type"] == "phone" and "phone" in self.syncing_fields + ] + + # Get tags/labels and create them if necessary + label_ids = [ + self.google.get_label_id(tag["name"]) + for tag in monica_contact["tags"] + if "labels" in self.syncing_fields + ] + + # Create contact upload form + form = GoogleContactUploadForm( + first_name=first_name, + last_name=last_name, + middle_name=middle_name, + birthdate=birthday, + phone_numbers=phone_numbers, + career=career, + email_adresses=emails, + label_ids=label_ids, + addresses=addresses, + ) + + # Upload contact + contact = self.google.create_contact(data=form.get_data()) + + return contact + + def __get_monica_middle_name( + self, first_name: str, last_name: str, nickname: str, full_name: str + ) -> str: + """Monica contacts have for some reason a hidden field middlename that can be set (creation/update) + but sadly can not retrieved later. This function computes it by using the complete_name field.""" + try: + # If there is a nickname it will be parenthesized with a space + nickname_length = len(nickname) + 3 if nickname else 0 + middle_name = full_name[ + len(first_name) : len(full_name) - (len(last_name) + nickname_length) + ].strip() + return middle_name + except Exception: + return "" + + def __check_google_contacts(self, google_contacts: List[dict]) -> Tuple[List[dict], int]: + """Checks every Google contact if it is currently in sync""" + errors = 0 + google_contacts_not_synced = [] + google_contacts_count = len(google_contacts) + # Check every Google contact + for num, google_contact in enumerate(google_contacts): + print(f"Processing Google contact {num+1} of {google_contacts_count}") + + # Get monica id + monica_id = self.mapping.get(google_contact["resourceName"], None) + if not monica_id: + google_contacts_not_synced.append(google_contact) + continue + + # Get monica contact + try: + monica_contact = self.monica.get_contact(monica_id) + assert monica_contact + except Exception: + errors += 1 + msg = ( + f"'{self.google.get_contact_names(google_contact)[3]}'" + f" ('{google_contact['resourceName']}'): " + f"Wrong id or missing Monica contact for id '{monica_id}'." + ) + self.log.error(msg) + print("\nError: " + msg) + return google_contacts_not_synced, errors + + def __check_monica_contacts(self, monica_contacts: List[dict]) -> Tuple[List[dict], int]: + """Checks every Google contact if it is currently in sync""" + errors = 0 + monica_contacts_not_synced = [] + monica_contacts_count = len(monica_contacts) + # Check every Monica contact + for num, monica_contact in enumerate(monica_contacts): + print(f"Processing Monica contact {num+1} of {monica_contacts_count}") + + # Get Google id + google_id = self.reverse_mapping.get(str(monica_contact["id"]), None) + if not google_id: + monica_contacts_not_synced.append(monica_contact) + continue + + # Get Google contact + try: + google_contact = self.google.get_contact(google_id) + assert google_contact + except Exception: + errors += 1 + msg = ( + f"'{monica_contact['complete_name']}' ('{monica_contact['id']}'): " + f"Wrong id or missing Google contact for id '{google_id}'." + ) + self.log.error(msg) + print("\nError: " + msg) + return monica_contacts_not_synced, errors + + def __check_results( + self, + orphaned_entries: List[str], + monica_contacts_not_synced: List[dict], + google_contacts_not_synced: List[dict], + ) -> None: + if orphaned_entries: + self.log.info("The following database entries are orphaned:") + for google_id in orphaned_entries: + entry = self.database.find_by_id(google_id) + if not entry: + raise DatabaseError("Database externally modified, entry not found!") + self.log.info( + f"'{google_id}' <-> '{entry.monica_id}' " + f"('{entry.google_full_name}' <-> '{entry.monica_full_name}')" + ) + self.log.info( + "This doesn't cause sync errors, but you can fix it doing initial sync '-i'" + ) + if not monica_contacts_not_synced and not google_contacts_not_synced: + self.log.info("All contacts are currently in sync") + elif monica_contacts_not_synced: + self.log.info("The following Monica contacts are currently not in sync:") + for monica_contact in monica_contacts_not_synced: + self.log.info(f"'{monica_contact['complete_name']}' ('{monica_contact['id']}')") + self.log.info("You can do a sync back '-sb' to fix that") + if google_contacts_not_synced: + self.log.info("The following Google contacts are currently not in sync:") + for google_contact in google_contacts_not_synced: + google_id = google_contact["resourceName"] + g_contact_display_name = self.google.get_contact_names(google_contact)[3] + self.log.info(f"'{g_contact_display_name}' ('{google_id}')") + self.log.info("You can do a full sync '-f' to fix that") + + def check_database(self) -> None: + """Checks if there are orphaned database entries which need to be resolved. + The following checks and assumptions will be made: + 1. Google contact id NOT IN database + -> Info: contact is currently not in sync + 2. Google contact id IN database BUT Monica contact not found + -> Error: deleted Monica contact or wrong id + 3. Monica contact id NOT IN database + -> Info: contact is currently not in sync + 4. Monica contact id IN database BUT Google contact not found + -> Error: deleted Google contact or wrong id + 5. Google contact id IN database BUT Monica AND Google contact not found + -> Warning: orphaned database entry""" + # Initialization + start_time = datetime.now() + errors = 0 + msg = "Starting database check..." + self.log.info(msg) + print("\n" + msg) + + # Get contacts + google_contacts = self.google.get_contacts(refetch_data=True, requestSyncToken=False) + monica_contacts = self.monica.get_contacts() + + # Check every Google contact + google_contacts_not_synced, error_count = self.__check_google_contacts(google_contacts) + errors += error_count + + # Check every Monica contact + monica_contacts_not_synced, error_count = self.__check_monica_contacts(monica_contacts) + errors += error_count + + # Check for orphaned database entries + google_ids = [c["resourceName"] for c in google_contacts] + monica_ids = [str(c["id"]) for c in monica_contacts] + orphaned_entries = [ + google_id + for google_id, monica_id in self.mapping.items() + if google_id not in google_ids and monica_id not in monica_ids + ] + + # Log results + self.__check_results(orphaned_entries, monica_contacts_not_synced, google_contacts_not_synced) + + # Finished + if errors: + msg = "Database check failed. Consider doing initial sync '-i' again!" + else: + msg = "Database check finished, no critical errors found!" + msg2 = ( + "If you encounter non-synced contacts on both sides that match each other " + "you should do an initial sync '-i' again to match them." + ) + self.log.info(msg) + self.log.info(msg2) + print("\n" + msg) + print(msg2) + + # Print and log statistics + self.__print_check_statistics( + start_time, + errors, + len(orphaned_entries), + len(monica_contacts_not_synced), + len(google_contacts_not_synced), + len(monica_contacts), + len(google_contacts), + ) + + def __print_check_statistics( + self, + start_time: datetime, + errors: int, + orphaned: int, + monica_contacts_not_synced: int, + google_contacts_not_synced: int, + monica_contacts: int, + google_contacts: int, + ) -> None: + """Prints and logs a pretty check statistic of the last database check.""" + tme = str(datetime.now() - start_time).split(".")[0] + "h" + err = str(errors) + (8 - len(str(errors))) * " " + oph = str(orphaned) + (8 - len(str(orphaned))) * " " + mns = str(monica_contacts_not_synced) + (8 - len(str(monica_contacts_not_synced))) * " " + gns = str(google_contacts_not_synced) + (8 - len(str(google_contacts_not_synced))) * " " + cmc = str(monica_contacts) + (8 - len(str(monica_contacts))) * " " + cgc = str(google_contacts) + (8 - len(str(google_contacts))) * " " + msg = ( + "\n" + f"Check statistics: \n" + f"+-----------------------------------------+\n" + f"| Check time: {tme } |\n" + f"| Errors: {err } |\n" + f"| Orphaned database entries: {oph } |\n" + f"| Monica contacts not in sync: {mns } |\n" + f"| Google contacts not in sync: {gns } |\n" + f"| Checked Monica contacts: {cmc } |\n" + f"| Checked Google contacts: {cgc } |\n" + f"+-----------------------------------------+" + ) + print(msg) + self.log.info(msg) + + def __merge_and_update_nbd(self, monica_contact: dict, google_contact: dict) -> None: + """Updates names, birthday and deceased date by merging an existing Monica contact with + a given Google contact.""" + # Get names + names_and_birthday = self.__get_monica_details(google_contact) + + # Get deceased info + deceased_date = monica_contact["information"]["dates"]["deceased_date"]["date"] + is_d_date_age_based = monica_contact["information"]["dates"]["deceased_date"]["is_age_based"] + deceased_year, deceased_month, deceased_day = None, None, None + if deceased_date: + date = self.__convert_monica_timestamp(deceased_date) + deceased_year = date.year + deceased_month = date.month + deceased_day = date.day + + # Assemble form object + google_form = MonicaContactUploadForm( + **names_and_birthday, + is_deceased=monica_contact["is_dead"], + is_deceased_date_known=bool(deceased_date), + deceased_year=deceased_year, + deceased_month=deceased_month, + deceased_day=deceased_day, + deceased_age_based=is_d_date_age_based, + ) + + # Check if contacts are already equal + monica_form = self.__get_monica_form(monica_contact) + if google_form.data == monica_form.data: + return + + # Upload contact + self.monica.update_contact(monica_id=monica_contact["id"], data=google_form.data) + + def __get_monica_form(self, monica_contact: dict) -> MonicaContactUploadForm: + """Creates a Monica contact upload form from a given Monica contact for comparison.""" + # Get names + first_name = monica_contact["first_name"] or "" + last_name = monica_contact["last_name"] or "" + full_name = monica_contact["complete_name"] or "" + nickname = monica_contact["nickname"] or "" + middle_name = self.__get_monica_middle_name(first_name, last_name, nickname, full_name) + + # Get birthday details + birthday_timestamp = monica_contact["information"]["dates"]["birthdate"]["date"] + birthdate_year, birthdate_month, birthdate_day = None, None, None + if birthday_timestamp: + is_year_unknown = monica_contact["information"]["dates"]["birthdate"]["is_year_unknown"] + date = self.__convert_monica_timestamp(birthday_timestamp) + birthdate_year = date.year if not is_year_unknown else None + birthdate_month = date.month + birthdate_day = date.day + + # Get deceased info + deceased_date = monica_contact["information"]["dates"]["deceased_date"]["date"] + is_d_date_age_based = monica_contact["information"]["dates"]["deceased_date"]["is_age_based"] + deceased_year, deceased_month, deceased_day = None, None, None + if deceased_date: + date = self.__convert_monica_timestamp(deceased_date) + deceased_year = date.year + deceased_month = date.month + deceased_day = date.day + + # Assemble form object + return MonicaContactUploadForm( + first_name=first_name, + monica=self.monica, + last_name=last_name, + nick_name=nickname, + middle_name=middle_name, + gender_type=monica_contact["gender_type"], + birthdate_day=birthdate_day, + birthdate_month=birthdate_month, + birthdate_year=birthdate_year, + is_birthdate_known=bool(birthday_timestamp), + is_deceased=monica_contact["is_dead"], + is_deceased_date_known=bool(deceased_date), + deceased_year=deceased_year, + deceased_month=deceased_month, + deceased_day=deceased_day, + deceased_age_based=is_d_date_age_based, + create_reminders=self.monica.create_reminders, + ) + + def create_monica_contact(self, google_contact: dict) -> dict: + """Creates a new Monica contact from a given Google contact and returns it.""" + form_data = self.__get_monica_details(google_contact) + + # Assemble form object + form = MonicaContactUploadForm(**form_data) + # Upload contact + monica_contact = self.monica.create_contact( + data=form.data, reference_id=google_contact["resourceName"] + ) + return monica_contact + + def __get_monica_details(self, google_contact: dict) -> Dict[str, Any]: + # Get names + first_name, last_name = self.__get_monica_names_from_google_contact(google_contact) + middle_name = self.google.get_contact_names(google_contact)[1] + display_name = self.google.get_contact_names(google_contact)[3] + nickname = self.google.get_contact_names(google_contact)[6] + # First name is required for Monica + if not first_name: + first_name = display_name + last_name = "" + + # Get birthday + birthday = google_contact.get("birthdays", None) + birthdate_year, birthdate_month, birthdate_day = None, None, None + if birthday: + birthdate_year = birthday[0].get("date", {}).get("year", None) + birthdate_month = birthday[0].get("date", {}).get("month", None) + birthdate_day = birthday[0].get("date", {}).get("day", None) + is_birthdate_known = all([birthdate_month, birthdate_day]) + + form_data = { + "first_name": first_name, + "monica": self.monica, + "last_name": last_name, + "middle_name": middle_name, + "nick_name": nickname, + "birthdate_day": birthdate_day if is_birthdate_known else None, + "birthdate_month": birthdate_month if is_birthdate_known else None, + "birthdate_year": birthdate_year if is_birthdate_known else None, + "is_birthdate_known": all([birthdate_month, birthdate_day]), + "create_reminders": self.monica.create_reminders, + } + return form_data + + def __convert_google_timestamp(self, timestamp: str) -> Union[datetime, None]: + """Converts Google timestamp to a datetime object.""" + try: + return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + return None + + def __convert_monica_timestamp(self, timestamp: str) -> datetime: + """Converts Monica timestamp to a datetime object.""" + return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") + + def __interactive_monica_id_search(self, google_contact: dict) -> str: + """Advanced search by first and last name for a given Google contact. + Tries to find a matching Monica contact and asks for user choice if + at least one candidate has been found. Creates a new Monica contact + if necessary or chosen by User. Returns Monica contact id.""" + # Initialization + candidates = [] + g_contact_given_name = self.google.get_contact_names(google_contact)[0] + g_contact_family_name = self.google.get_contact_names(google_contact)[2] + g_contact_display_name = self.google.get_contact_names(google_contact)[3] + monica_contact = None + + # Process every Monica contact + for monica_contact in self.monica.get_contacts(): + contact_in_database = str(monica_contact["id"]) in self.mapping.values() + is_name_match = ( + g_contact_given_name == monica_contact["first_name"] + or g_contact_family_name == monica_contact["last_name"] + ) + if not contact_in_database and is_name_match: + # If the id isn't in the database and first or last name matches + # add potential candidate to list + candidates.append(monica_contact) + + # If there is at least one candidate let the user choose + choice = None + if candidates: + print("\nPossible syncing conflict, please choose your alternative by number:") + print(f"\tWhich Monica contact should be connected to '{g_contact_display_name}'?") + for num, monica_contact in enumerate(candidates): + print(f"\t{num+1}: {monica_contact['complete_name']}") + print(f"\t{num+2}: Create a new Monica contact") + choice = self.__get_user_input(allowed_nums=list(range(1, len(candidates) + 2))) + # Created a sublist with the selected candidate + # or an empty list if user votes for a new contact + candidates = candidates[choice - 1 : choice] + + # No candidates found, let the user choose to create a new contact + elif not self.skip_creation_prompt: + print(f"\nNo Monica contact has been found for '{g_contact_display_name}'") + print("\tCreate a new Monica contact?") + print("\t0: No (abort initial sync)") + print("\t1: Yes") + print("\t2: Yes to all") + choice = choice = self.__get_user_input(allowed_nums=[0, 1, 2]) + if choice == 0: + raise UserChoice("Sync aborted by user choice") + if choice == 2: + # Skip further contact creation prompts + self.skip_creation_prompt = True + + # If there are no candidates (user vote or nothing found) create a new Monica contact + if not candidates: + # Create new Monica contact + monica_contact = self.create_monica_contact(google_contact) + + # Print success + msg = f"'{g_contact_display_name}' ('{monica_contact['id']}'): New Monica contact created" + self.log.info(msg) + print("Conflict resolved: " + msg) + + # There must be exactly one candidate from user vote + else: + monica_contact = candidates[0] + + # Update database and mapping + database_entry = DatabaseEntry( + google_contact["resourceName"], + monica_contact["id"], + g_contact_display_name, + monica_contact["complete_name"], + ) + self.database.insert_data(database_entry) + self.__update_mapping(google_contact["resourceName"], str(monica_contact["id"])) + + # Print success + msg = ( + f"'{google_contact['resourceName']}' <-> '{monica_contact['id']}': " + "New sync connection added" + ) + self.log.info(msg) + print("Conflict resolved: " + msg) + + return str(monica_contact["id"]) + + def __get_user_input(self, allowed_nums: List[int]) -> int: + """Prompts for a number entered by the user""" + # If running from GitHub Actions always choose 1 + if os.getenv("CI", 0): + print("Running from CI -> 1") + return 1 + while True: + try: + choice = int(input("Enter your choice (number only): ")) + if choice in allowed_nums: + return choice + else: + raise BadUserInput() + except Exception: + print("Bad input, please try again!\n") + + def __simple_monica_id_search(self, google_contact: dict) -> Union[str, None]: + """Simple search by displayname for a given Google contact. + Tries to find a matching Monica contact and returns its id or None if not found""" + # Initialization + g_contact_given_name = self.google.get_contact_names(google_contact)[0] + g_contact_middle_name = self.google.get_contact_names(google_contact)[1] + g_contact_family_name = self.google.get_contact_names(google_contact)[2] + g_contact_display_name = self.google.get_contact_names(google_contact)[3] + candidates = [] + + # Process every Monica contact + for monica_contact in self.monica.get_contacts(): + # Get monica data + m_contact_id = str(monica_contact["id"]) + m_contact_first_name = monica_contact["first_name"] or "" + m_contact_last_name = monica_contact["last_name"] or "" + m_contact_full_name = monica_contact["complete_name"] or "" + m_contact_nickname = monica_contact["nickname"] or "" + m_contact_middle_name = self.__get_monica_middle_name( + m_contact_first_name, m_contact_last_name, m_contact_nickname, m_contact_full_name + ) + # Check if the Monica contact is already assigned to a Google contact + is_monica_contact_assigned = m_contact_id in self.mapping.values() + # Check if display names match + is_display_name_match = g_contact_display_name == m_contact_full_name + # Pre-check that the Google contact has a given and a family name + has_names = g_contact_given_name and g_contact_family_name + # Check if names match when ignoring honorifix prefixes + is_without_prefix_match = has_names and ( + " ".join([g_contact_given_name, g_contact_family_name]) == m_contact_full_name + ) + # Check if first, middle and last name matches + is_first_last_middle_name_match = ( + m_contact_first_name == g_contact_given_name + and m_contact_middle_name == g_contact_middle_name + and m_contact_last_name == g_contact_family_name + ) + # Assemble all conditions + matches = [is_display_name_match, is_without_prefix_match, is_first_last_middle_name_match] + if not is_monica_contact_assigned and any(matches): + # Add possible candidate + candidates.append(monica_contact) + + # If there is only one candidate + if len(candidates) == 1: + monica_contact = candidates[0] + + # Update database and mapping + database_entry = DatabaseEntry( + google_contact["resourceName"], + monica_contact["id"], + g_contact_display_name, + monica_contact["complete_name"], + ) + self.database.insert_data(database_entry) + self.__update_mapping(google_contact["resourceName"], str(monica_contact["id"])) + return str(monica_contact["id"]) + + # Simple search failed + return None + + def __get_monica_names_from_google_contact(self, google_contact: dict) -> Tuple[str, str]: + """Creates first and last name from a Google contact with respect to honoric + suffix/prefix.""" + given_name, _, family_name, _, prefix, suffix, _ = self.google.get_contact_names(google_contact) + if prefix: + given_name = f"{prefix} {given_name}".strip() + if suffix: + family_name = f"{family_name} {suffix}".strip() + return given_name, family_name diff --git a/helpers/__init__.py b/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logs/.empty b/logs/.empty new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index a6e3d08..35350f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ google-api-python-client>=1.7.9 google-auth-httplib2>=0.0.3 google-auth-oauthlib>=0.4.0 -oauth2client>=4.1.3 \ No newline at end of file +oauth2client>=4.1.3 +python-dotenv>=0.19.2 diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..9dba282 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,13 @@ +sonar.projectKey=antonplagemann_GoogleMonicaSync +sonar.organization=antonplagemann +sonar.python.version=3.7, 3.8, 3.9, 3.10 + +# This is the name and version displayed in the SonarCloud UI. +#sonar.projectName=GoogleMonicaSync +#sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +#sonar.sources=. + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 diff --git a/test/99_sample_google_contacts.csv b/test/99_sample_google_contacts.csv new file mode 100644 index 0000000..7e27eba --- /dev/null +++ b/test/99_sample_google_contacts.csv @@ -0,0 +1,100 @@ +Given Name;Family Name;Name;Nickname;Organization 1 - Name;Occupation;Address 1 - Street;Address 1 - City;Address 1 - Region;Address 1 - Postal Code;Phone 1 - Value;Phone 2 - Value;E-mail 1 - Value;Birthday;Notes +James;Butt;James Butt;Nicky;Benton, John B Jr;Baker;6649 N Blue Gum St;New Orleans;LA;70116;504-621-8927;504-845-1427;jbutt@gmail.com;2015-11-07;Random note #7001 +Josephine;Darakjy;Josephine Darakjy;Nicky;Chanay, Jeffrey A Esq;Farmer;4 B Blue Ridge Blvd;Brighton;MI;48116;810-292-9388;810-374-9840;josephine_darakjy@darakjy.org;1946-11-08;Random note #8626 +Art;Venere;Art Venere;Nicky;Chemel, James L Cpa;Hairdresser;8 W Cerritos Ave #54;Bridgeport;NJ;8014;856-636-8749;856-264-4130;art@venere.org;1947-03-05;Random note #5054 +Lenna;Paprocki;Lenna Paprocki;Nicky;Feltz Printing Service;Mason;639 Main St;Anchorage;AK;99501;907-385-4412;907-921-2010;lpaprocki@hotmail.com;1999-11-06;Random note #6686 +Donette;Foller;Donette Foller;Nicky;Printing Dimensions;Policeman;34 Center St;Hamilton;OH;45011;513-570-1893;513-549-4561;donette.foller@cox.net;1998-05-17;Random note #1079 +Simona;Morasca;Simona Morasca;Nicky;Chapman, Ross E Esq;Soldier;3 Mcauley Dr;Ashland;OH;44805;419-503-2484;419-800-6759;simona@morasca.com;2003-03-02;Random note #1798 +Mitsue;Tollner;Mitsue Tollner;Nicky;Morlong Associates;Butcher;7 Eads St;Chicago;IL;60632;773-573-6914;773-924-8565;mitsue_tollner@yahoo.com;1944-02-24;Random note #6810 +Leota;Dilliard;Leota Dilliard;Nicky;Commercial Press;Fireman;7 W Jackson Blvd;San Jose;CA;95111;408-752-3500;408-813-1105;leota@hotmail.com;1980-10-27;Random note #785 +Sage;Wieser;Sage Wieser;Nicky;Truhlar And Truhlar Attys;Journalist;5 Boston Ave #88;Sioux Falls;SD;57105;605-414-2147;605-794-4895;sage_wieser@cox.net;1954-05-07;Random note #9024 +Kris;Marrier;Kris Marrier;Nicky;King, Christopher A Esq;Mechanic;228 Runamuck Pl #2808;Baltimore;MD;21224;410-655-8723;410-804-4694;kris@gmail.com;1977-10-14; +Minna;Amigon;Minna Amigon;Nicky;Dorl, James J Esq;Postman;2371 Jerrold Ave;Kulpsville;PA;19443;215-874-1229;215-422-8694;minna_amigon@yahoo.com;2003-11-22;Random note #855 +Abel;Maclead;Abel Maclead;Nicky;Rangoni Of Florence;Taxi driver;37275 St Rt 17m M;Middle Island;NY;11953;631-335-3414;631-677-3675;amaclead@gmail.com;1964-10-30;Random note #2577 +Kiley;Caldarera;Kiley Caldarera;Nicky;Feiner Bros;Carpenter;25 E 75th St #69;Los Angeles;CA;90034;310-498-5651;310-254-3084;kiley.caldarera@aol.com;1932-04-08;Random note #8081 +Graciela;Ruta;Graciela Ruta;Nicky;Buckley Miller & Wright;Fisherman;98 Connecticut Ave Nw;Chagrin Falls;OH;44023;440-780-8425;440-579-7763;gruta@cox.net;1995-10-03;Random note #3376 +Cammy;Albares;Cammy Albares;Nicky;Rousseaux, Michael Esq;Judge;56 E Morehead St;Laredo;TX;78045;956-537-6195;956-841-7216;calbares@gmail.com;1987-06-28;Random note #9515 +Mattie;Poquette;Mattie Poquette;Nicky;Century Communications;Painter;73 State Road 434 E;Phoenix;AZ;85013;602-277-4385;602-953-6360;mattie@aol.com;1968-07-12;Random note #6340 +Meaghan;Garufi;Meaghan Garufi;Nicky;Bolton, Wilbur Esq;Secretary;69734 E Carrillo St;Mc Minnville;TN;37110;931-313-9635;931-235-7959;meaghan@hotmail.com;1953-04-08;Random note #3649 +Gladys;Rim;Gladys Rim;Nicky;T M Byxbee Company Pc;Teacher;322 New Horizon Blvd;Milwaukee;WI;53207;414-661-9598;414-377-2880;gladys.rim@rim.org;1996-06-15;Random note #9439 +Yuki;Whobrey;Yuki Whobrey;Nicky;Farmers Insurance Group;Cook;1 State Route 27;Taylor;MI;48180;313-288-7937;313-341-4470;yuki_whobrey@aol.com;1979-07-05;Random note #336 +Fletcher;Flosi;Fletcher Flosi;Nicky;Post Box Services Plus;Gardener;394 Manchester Blvd;Rockford;IL;61109;815-828-2147;815-426-5657;fletcher.flosi@yahoo.com;1994-08-29;Random note #2401 +Bette;Nicka;Bette Nicka;Nicky;Sport En Art;Lawyer;6 S 33rd St;Aston;PA;19014;610-545-3615;610-492-4643;bette_nicka@cox.net;1996-12-29;Random note #5193 +Veronika;Inouye;Veronika Inouye;Nicky;C 4 Network Inc;Plumber;6 Greenleaf Ave;San Jose;CA;95111;408-540-1785;408-813-4592;vinouye@aol.com;1978-12-24;Random note #7450 +Willard;Kolmetz;Willard Kolmetz;Nicky;Ingalls, Donald R Esq;Singer;618 W Yakima Ave;Irving;TX;75062;972-303-9197;972-896-4882;willard@hotmail.com;1973-02-23;Random note #2967 +Maryann;Royster;Maryann Royster;Nicky;Franklin, Peter L Esq;Waiter;74 S Westgate St;Albany;NY;12204;518-966-7987;518-448-8982;mroyster@royster.com;1992-04-02;Random note #4686 +Alisha;Slusarski;Alisha Slusarski;Nicky;Wtlz Power 107 Fm;Baker;3273 State St;Middlesex;NJ;8846;732-658-3154;732-635-3453;alisha@slusarski.com;1953-06-14;Random note #6402 +Allene;Iturbide;Allene Iturbide;Nicky;Ledecky, David Esq;Farmer;1 Central Ave;Stevens Point;WI;54481;715-662-6764;715-530-9863;allene_iturbide@cox.net;1949-07-12;Random note #5211 +Chanel;Caudy;Chanel Caudy;Nicky;Professional Image Inc;Hairdresser;86 Nw 66th St #8673;Shawnee;KS;66218;913-388-2079;913-899-1103;chanel.caudy@caudy.org;1936-11-22;Random note #2309 +Ezekiel;Chui;Ezekiel Chui;Nicky;Sider, Donald C Esq;Mason;2 Cedar Ave #84;Easton;MD;21601;410-669-1642;410-235-8738;ezekiel@chui.com;1938-04-18;Random note #4470 +Willow;Kusko;Willow Kusko;Nicky;U Pull It;Policeman;90991 Thorburn Ave;New York;NY;10011;212-582-4976;212-934-5167;wkusko@yahoo.com;1938-01-18;Random note #7765 +Bernardo;Figeroa;Bernardo Figeroa;Nicky;Clark, Richard Cpa;Soldier;386 9th Ave N;Conroe;TX;77301;936-336-3951;936-597-3614;bfigeroa@aol.com;1986-05-21;Random note #6010 +Ammie;Corrio;Ammie Corrio;Nicky;Moskowitz, Barry S;Butcher;74874 Atlantic Ave;Columbus;OH;43215;614-801-9788;614-648-3265;ammie@corrio.com;1946-04-12; +Francine;Vocelka;Francine Vocelka;Nicky;Cascade Realty Advisors Inc;Fireman;366 South Dr;Las Cruces;NM;88011;505-977-3911;505-335-5293;francine_vocelka@vocelka.com;1948-11-14;Random note #2612 +Ernie;Stenseth;Ernie Stenseth;Nicky;Knwz Newsradio;Journalist;45 E Liberty St;Ridgefield Park;NJ;7660;201-709-6245;201-387-9093;ernie_stenseth@aol.com;2014-10-09;Random note #9601 +Albina;Glick;Albina Glick;Nicky;Giampetro, Anthony D;Mechanic;4 Ralph Ct;Dunellen;NJ;8812;732-924-7882;732-782-6701;albina@glick.com;2018-07-07;Random note #6093 +Alishia;Sergi;Alishia Sergi;Nicky;Milford Enterprises Inc;Postman;2742 Distribution Way;New York;NY;10025;212-860-1579;212-753-2740;asergi@gmail.com;1948-05-24;Random note #3825 +Solange;Shinko;Solange Shinko;Nicky;Mosocco, Ronald A;Taxi driver;426 Wolf St;Metairie;LA;70002;504-979-9175;504-265-8174;;1973-10-21;Random note #2199 +Jose;Stockham;Jose Stockham;Nicky;Tri State Refueler Co;Carpenter;128 Bransten Rd;New York;NY;10011;212-675-8570;212-569-4233;jose@yahoo.com;2015-11-04;Random note #1081 +Rozella;Ostrosky;Rozella Ostrosky;Nicky;Parkway Company;Fisherman;17 Morena Blvd;Camarillo;CA;93012;805-832-6163;;rozella.ostrosky@ostrosky.com;1994-05-08;Random note #564 +Valentine;Gillian;Valentine Gillian;Nicky;Fbs Business Finance;Judge;775 W 17th St;San Antonio;TX;78204;210-812-9597;210-300-6244;valentine_gillian@gmail.com;2005-10-20;Random note #1837 +Kati;Rulapaugh;Kati Rulapaugh;Nicky;Eder Assocs Consltng Engrs Pc;Painter;6980 Dorsett Rd;Abilene;KS;67410;785-463-7829;785-219-7724;kati.rulapaugh@hotmail.com;2013-02-09;Random note #8666 +Youlanda;Schemmer;Youlanda Schemmer;Nicky;Tri M Tool Inc;Secretary;2881 Lewis Rd;Prineville;OR;97754;541-548-8197;541-993-2611;youlanda@aol.com;1986-11-30;Random note #8605 +Dyan;Oldroyd;Dyan Oldroyd;Nicky;International Eyelets Inc;Teacher;7219 Woodfield Rd;Overland Park;KS;66204;913-413-4604;913-645-8918;doldroyd@aol.com;1969-09-24;Random note #7307 +Roxane;Campain;Roxane Campain;Nicky;Rapid Trading Intl;Cook;1048 Main St;Fairbanks;AK;99708;907-231-4722;907-335-6568;roxane@hotmail.com;1959-10-24;Random note #127 +Lavera;Perin;Lavera Perin;Nicky;Abc Enterprises Inc;Gardener;678 3rd Ave;Miami;FL;33196;305-606-7291;305-995-2078;lperin@perin.org;1966-10-16;Random note #318 +Erick;Ferencz;Erick Ferencz;Nicky;Cindy Turner Associates;Lawyer;20 S Babcock St;Fairbanks;AK;99712;907-741-1044;907-227-6777;erick.ferencz@aol.com;2000-07-30;Random note #3579 +Fatima;Saylors;Fatima Saylors;Nicky;Stanton, James D Esq;Plumber;2 Lighthouse Ave;Hopkins;MN;55343;952-768-2416;952-479-2375;fsaylors@saylors.org;1960-09-15;Random note #2391 +Jina;Briddick;Jina Briddick;Nicky;;Singer;38938 Park Blvd;Boston;MA;2128;617-399-5124;617-997-5771;jina_briddick@briddick.com;1931-05-29;Random note #2716 +Kanisha;Waycott;Kanisha Waycott;Nicky;Schroer, Gene E Esq;Waiter;5 Tomahawk Dr;Los Angeles;CA;90006;323-453-2780;323-315-7314;kanisha_waycott@yahoo.com;1981-12-13;Random note #1230 +Emerson;Bowley;Emerson Bowley;Nicky;Knights Inn;Baker;762 S Main St;Madison;WI;53711;608-336-7444;608-658-7940;emerson.bowley@bowley.org;1945-06-13;Random note #4069 +Blair;Malet;Blair Malet;Nicky;Bollinger Mach Shp & Shipyard;Farmer;209 Decker Dr;Philadelphia;PA;19132;215-907-9111;215-794-4519;bmalet@yahoo.com;1995-11-15;Random note #490 +Brock;Bolognia;Brock Bolognia;Nicky;Orinda News;Hairdresser;4486 W O St #1;New York;NY;10003;212-402-9216;212-617-5063;bbolognia@yahoo.com;2020-12-26;Random note #3020 +Lorrie;Nestle;Lorrie Nestle;Nicky;Ballard Spahr Andrews;Mason;39 S 7th St;Tullahoma;TN;37388;931-875-6644;931-303-6041;lnestle@hotmail.com;1954-01-08;Random note #7558 +Sabra;Uyetake;Sabra Uyetake;Nicky;Lowy Limousine Service;Policeman;98839 Hawthorne Blvd #6101;Columbia;SC;29201;;803-681-3678;sabra@uyetake.org;2006-10-31;Random note #121 +Marjory;Mastella;Marjory Mastella;Nicky;Vicon Corporation;Soldier;71 San Mateo Ave;Wayne;PA;19087;610-814-5533;610-379-7125;mmastella@mastella.com;1959-08-25;Random note #3756 +Karl;Klonowski;Karl Klonowski;Nicky;Rossi, Michael M;Butcher;76 Brooks St #9;Flemington;NJ;8822;908-877-6135;908-470-4661;karl_klonowski@yahoo.com;1938-02-26;Random note #791 +Tonette;Wenner;Tonette Wenner;Nicky;Northwest Publishing;Fireman;4545 Courthouse Rd;Westbury;NY;11590;;;twenner@aol.com;1954-01-18;Random note #3832 +Amber;Monarrez;Amber Monarrez;Nicky;Branford Wire & Mfg Co;Journalist;14288 Foster Ave #4121;Jenkintown;PA;19046;215-934-8655;215-329-6386;amber_monarrez@monarrez.org;1970-12-29; +Shenika;Seewald;Shenika Seewald;Nicky;;Mechanic;4 Otis St;Van Nuys;CA;91405;818-423-4007;818-749-8650;shenika@gmail.com;2002-03-07;Random note #9051 +Delmy;Ahle;Delmy Ahle;Nicky;Wye Technologies Inc;Postman;65895 S 16th St;Providence;RI;2909;401-458-2547;401-559-8961;delmy.ahle@hotmail.com;1950-02-19;Random note #4100 +Deeanna;Juhas;Deeanna Juhas;Nicky;Healy, George W Iv;Taxi driver;14302 Pennsylvania Ave;Huntingdon Valley;PA;19006;215-211-9589;215-417-9563;deeanna_juhas@gmail.com;1973-09-26;Random note #119 +Blondell;Pugh;Blondell Pugh;Nicky;Alpenlite Inc;Carpenter;201 Hawk Ct;Providence;RI;2904;401-960-8259;401-300-8122;bpugh@aol.com;1978-08-24;Random note #4952 +Jamal;Vanausdal;Jamal Vanausdal;Nicky;Hubbard, Bruce Esq;Fisherman;53075 Sw 152nd Ter #615;Monroe Township;NJ;8831;732-234-1546;732-904-2931;jamal@vanausdal.org;2019-10-17;Random note #1645 +Cecily;Hollack;;;Arthur A Oliver & Son Inc;Judge;59 N Groesbeck Hwy;Austin;;78731;512-486-3817;512-861-3814;cecily@hollack.org;1994-03-05;Random note #8170 +Carmelina;Lindall;Carmelina Lindall;Nicky;George Jessop Carter Jewelers;Painter;2664 Lewis Rd;Littleton;CO;;303-724-7371;303-874-5160;carmelina_lindall@lindall.com;1991-01-27;Random note #8950 +Maurine;;Maurine Yglesias;Nicky;Schultz, Thomas C Md;Secretary;59 Shady Ln #53;Milwaukee;;53214;414-748-1374;414-573-7719;maurine_yglesias@yglesias.com;1994-08-03;Random note #221 +Tawna;Buvens;Tawna Buvens;Nicky;H H H Enterprises Inc;;3305 Nabell Ave #679;;NY;10009;212-674-9610;212-462-9157;;2014-04-27; +Penney;Weight;Penney Weight;Nicky;Hawaiian King Hotel;Cook;18 Fountain St;Anchorage;AK;99515;907-797-9628;907-873-2882;penney_weight@aol.com;1972/12/28;Random note #5584 +Elly;Morocco;Elly Morocco;Nicky;Killion Industries;Gardener;7 W 32nd St;Erie;PA;16502;814-393-5571;814-420-3553;elly_morocco@gmail.com;1946-11-12;Random note #5289 +Ilene;Eroman;Ilene Eroman;;Robinson, William J Esq;Lawyer;2853 S Central Expy;;MD;21061;410-914-9018;410-937-4543;ilene.eroman@hotmail.com;1973-07-31;Random note #4154 +Vallie;Mondella;Vallie Mondella;Nicky;Private Properties;;74 W College St;Boise;ID;83707;;;vmondella@mondella.com;1945-05-05;Random note #4632 +Kallie;Blackwood;Kallie Blackwood;Nicky;Rowley Schlimgen Inc;Singer;701 S Harrison Rd;San Francisco;CA;94104;415-315-2761;415-604-7609;kallie.blackwood@gmail.com;1994-10-03;Random note #8940 +Johnetta;Abdallah;Johnetta Abdallah;Nicky;Forging Specialties;Waiter;1088 Pinehurst St;Chapel Hill;NC;27514;919-225-9345;919-715-3791;johnetta_abdallah@aol.com;1957-07-04;Random note #7130 +Bobbye;Rhym;Bobbye Rhym;Nicky;;Baker;30 W 80th St #1995;San Carlos;CA;94070;650-528-5783;650-811-9032;brhym@rhym.com;1983-10-21;Random note #6940 +Micaela;Rhymes;Micaela Rhymes;Nicky;H Lee Leonard Attorney At Law;Farmer;20932 Hedley St;Concord;CA;94520;925-647-3298;925-522-7798;micaela_rhymes@gmail.com;1969-08-14;Random note #3477 +Tamar;Hoogland;Tamar Hoogland;Nicky;A K Construction Co;Hairdresser;;London;OH;43140;740-343-8575;740-526-5410;tamar@hotmail.com;1982-08-27;Random note #3060 +Moon;Parlato;Moon Parlato;;Ambelang, Jessica M Md;Mason;74989 Brandon St;Wellsville;NY;14895;585-866-8313;585-498-4278;moon@yahoo.com;1940-08-30;Random note #4464 +Laurel;Reitler;Laurel Reitler;Nicky;Q A Service;Policeman;6 Kains Ave;Baltimore;MD;21215;410-520-4832;410-957-6903;laurel_reitler@reitler.com;1966-11-23;Random note #224 +Delisa;Crupi;Delisa Crupi;Nicky;Wood & Whitacre Contractors;Soldier;47565 W Grand Ave;Newark;NJ;7105;973-354-2040;973-847-9611;delisa.crupi@crupi.com;16.12.2015;Random note #7881 +Viva;Toelkes;Viva Toelkes;Nicky;Mark Iv Press Ltd;Butcher;4284 Dorigo Ln;Chicago;IL;60647;773-446-5569;773-352-3437;viva.toelkes@gmail.com;1986-10-12;Random note #7257 +Elza;Lipke;Elza Lipke;;Museum Of Science & Industry;Fireman;6794 Lake Dr E;Newark;NJ;7104;973-927-3447;973-796-3667;elza@yahoo.com;1958-09-26;Random note #4145 +Devorah;Chickering;Devorah Chickering;Nicky;Garrison Ind;Journalist;31 Douglas Blvd #950;Clovis;NM;88101;505-975-8559;505-950-1763;devorah@hotmail.com;2014-12-05;Random note #1179 +Timothy;Mulqueen;Timothy Mulqueen;Nicky;Saronix Nymph Products;;44 W 4th St;Staten Island;NY;10309;718-332-6527;718-654-7063;timothy_mulqueen@mulqueen.org;1945-12-10;Random note #3496 +Arlette;;Arlette Honeywell;Nicky;Smc Inc;Postman;11279 Loytan St;Jacksonville;FL;32254;904-775-4480;904-514-9918;;1991-06-10;Random note #5574 +Dominque;Dickerson;;Nicky;E A I Electronic Assocs Inc;Taxi driver;69 Marquette Ave;Hayward;CA;94545;510-993-3758;510-901-7640;dominque.dickerson@dickerson.org;1948/04/02;Random note #3693 +Lettie;Isenhower;Lettie Isenhower;Nicky;Conte, Christopher A Esq;Carpenter;70 W Main St;Beachwood;OH;44122;216-657-7668;216-733-8494;lettie_isenhower@yahoo.com;1947-12-31;Random note #9985 +Myra;Munns;Myra Munns;;;Fisherman;461 Prospect Pl #316;Euless;TX;76040;817-914-7518;817-451-3518;mmunns@cox.net;1994-06-27;Random note #2207 +Stephaine;Barfield;Stephaine Barfield;Nicky;Beutelschies & Company;Judge;47154 Whipple Ave Nw;Gardena;CA;90247;310-774-7643;310-968-1219;stephaine@barfield.com;2016-05-28;Random note #1423 +Lai;Gato;Lai Gato;Nicky;Fligg, Kenneth I Jr;Painter;37 Alabama Ave;Evanston;IL;60201;847-728-7286;847-957-4614;lai.gato@gato.org;1936-12-28;Random note #6568 +Stephen;Emigh;Stephen Emigh;Nicky;Sharp, J Daniel Esq;Secretary;3777 E Richmond St #900;Akron;OH;44302;330-537-5358;330-700-2312;stephen_emigh@hotmail.com;2007-08-20;Random note #3386 +Tyra;Shields;Tyra Shields;Nicky;Assink, Anne H Esq;Teacher;3 Fort Worth Ave;Philadelphia;PA;19106;215-255-1641;215-228-8264;tshields@gmail.com;1988-07-04;Random note #4132 +Tammara;Wardrip;Tammara Wardrip;Nicky;Jewel My Shop Inc;Cook;4800 Black Horse Pike;Burlingame;CA;94010;650-803-1936;650-216-5075;twardrip@cox.net;1938-01-27;Random note #9093 +Cory;Gibes;Cory Gibes;Nicky;Chinese Translation Resources;Gardener;83649 W Belmont Ave;San Gabriel;CA;91776;626-572-1096;626-696-2777;cory.gibes@gmail.com;1990-02-24;Random note #9742 +Danica;Bruschke;Danica Bruschke;Nicky;Stevens, Charles T;Lawyer;840 15th Ave;Waco;TX;76708;254-782-8569;254-205-1422;danica_bruschke@gmail.com;2009-01-13;Random note #4985 +Wilda;Giguere;Wilda Giguere;Nicky;Mclaughlin, Luther W Cpa;Plumber;1747 Calle Amanecer #2;Anchorage;AK;99501;907-870-5536;907-914-9482;wilda@cox.net;1979-03-07;Random note #6581 +Elvera;Benimadho;Elvera Benimadho;Nicky;Tree Musketeers;Singer;99385 Charity St #840;San Jose;CA;95110;408-703-8505;408-440-8447;elvera.benimadho@cox.net;1943-08-18;Random note #7000 +Carma;;Carma Vanheusen;Nicky;Springfield Div Oh Edison Co;Waiter;68556 Central Hwy;San Leandro;CA;94577;510-503-7169;510-452-4835;carma@cox.net;1946-03-02;Random note #5433 +Malinda;Hochard;Malinda Hochard;Nicky;Logan Memorial Hospital;Baker;55 Riverside Ave;Indianapolis;IN;46202;317-722-5066;317-472-2412;;1963-07-06;Random note #2983 +Natalie;Fern;Natalie Fern;Nicky;Kelly, Charles G Esq;Farmer;7140 University Ave;Rock Springs;WY;82901;307-704-8713;307-279-3793;natalie.fern@hotmail.com;1988-02-02;Random note #8352 +Lisha;Centini;Lisha Centini;Nicky;Industrial Paper Shredders Inc;Hairdresser;64 5th Ave #1153;Mc Lean;VA;22102;703-235-3937;703-475-7568;lisha@centini.org;1948-12-21;Random note #3975 diff --git a/test/ChaosMonkey.py b/test/ChaosMonkey.py new file mode 100644 index 0000000..f95860f --- /dev/null +++ b/test/ChaosMonkey.py @@ -0,0 +1,533 @@ +from __future__ import annotations + +import argparse +import inspect +import logging +import os +import pickle +import random +import sys +from copy import deepcopy +from os.path import join +from time import sleep +from typing import Dict, List + +# Include parent folder in module search +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # type: ignore +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, parentdir) + +from dotenv import dotenv_values, find_dotenv # type: ignore + +from helpers.ConfigHelper import Config +from helpers.DatabaseHelper import Database, DatabaseEntry +from helpers.Exceptions import DatabaseError +from helpers.GoogleHelper import Google +from helpers.MonicaHelper import Monica +from helpers.SyncHelper import Sync + +LOG_FOLDER = "logs" +LOG_FILENAME = "monkey.log" +STATE_FILENAME = "monkeyState.pickle" +DEFAULT_CONFIG_FILEPATH = join("..", "helpers", ".env.default") + +# Google contact writes may have a propagation delay of several minutes for sync requests. +SLEEP_TIME = 180 + +# Chaos monkey +# Creates, deletes and updates some random contacts at Google and Monica + + +class State: + """Maintains the monkey state""" + + def __init__(self, google_contacts: List[dict], seed: int) -> None: + self.contacts = google_contacts + self.seed = seed + self.original_contacts: Dict[str, dict] = {} + self.deleted_contacts: List[dict] = [] + self.created_contacts: List[dict] = [] + self.deleted_database_entries: List[DatabaseEntry] = [] + self.created_database_entries: List[DatabaseEntry] = [] + self.created_syncback_contact_ids: List[str] = [] + + def save(self, log: logging.Logger) -> None: + """Saves the state to the filesystem""" + with open(STATE_FILENAME, "wb") as state_file: + pickle.dump(self, state_file) + self.log(log, "State saved: \n") + + def log(self, log: logging.Logger, title: str) -> None: + """Logs the current state""" + log.info( + title + f"\tcontacts = {len(self.contacts)}\n" + f"\tseed = {self.seed}\n" + f"\tupdated_contacts = {len(self.original_contacts)}\n" + f"\tdeleted_contacts = {len(self.deleted_contacts)}\n" + f"\tcreated_contacts = {len(self.created_contacts)}\n" + f"\tdeleted_database_entries = {len(self.deleted_database_entries)}\n" + f"\tcreated_database_entries = {len(self.created_database_entries)}" + ) + + @staticmethod + def load(monkey: Monkey) -> State: + """Loads the state from filesystem or creates a new one""" + if os.path.exists(STATE_FILENAME): + with open(STATE_FILENAME, "rb") as state_file: + state: State = pickle.load(state_file) + state.log(monkey.log, "State loaded: \n") + return state + else: + google_contacts = sorted( + monkey.google.get_contacts(), key=lambda contact: contact["resourceName"], reverse=True + ) + seed = monkey.args.seed + if not seed: + seed = random.randint(100000, 999999) # nosec + state = State(google_contacts, seed) + state.log(monkey.log, "State created: \n") + return state + + +class Monkey: + """Main monkey class for producing chaos""" + + def __init__(self) -> None: + self.fields = [ + "names", + "birthdays", + "addresses", + "emailAddresses", + "phoneNumbers", + "biographies", + "organizations", + "occupations", + "memberships", + ] + + def main(self) -> None: + try: + # Create logger + self.__create_logger() + self.log.info("Script started") + + # Create argument parser + self.__create_argument_parser() + + # Load config + self.__load_config() + + # Create sync object + self.__create_sync_helper() + + # Load state + self.state = State.load(self) + + # Set random seed + random.seed(self.state.seed) + self.log.info(f"Using random seed '{self.state.seed}'") + + # Start + if self.args.initial: + # Start initial sync chaos (-i) + self.log.info("Starting initial sync chaos") + self.initial_chaos(self.args.num) + elif self.args.delta: + # Start delta sync chaos (-d) + self.log.info("Starting delta sync chaos") + self.delta_chaos(self.args.num) + # Give the People API some time to process changes before continuing + self.log.info(f"Giving the People API {SLEEP_TIME} seconds to process changes...") + sleep(SLEEP_TIME) + elif self.args.full: + # Start full sync chaos (-f) + self.log.info("Starting full sync chaos") + self.full_chaos(self.args.num) + elif self.args.syncback: + # Start sync back from Monica to Google chaos (-sb) + self.log.info("Starting sync back chaos") + self.syncback_chaos(self.args.num) + elif self.args.check: + # Start database error check chaos (-c) + self.log.info("Starting database check chaos") + self.database_chaos(self.args.num) + elif self.args.restore: + # No chaos anymore 😔 (-r) + self.log.info("Starting restore Google contacts") + self.restore_contacts() + else: + # No arguments + self.log.info("Unknown arguments, exiting...") + self.parser.print_help() + + # Its over now + self.state.save(self.log) + self.log.info("Script ended\n") + + except Exception as e: + self.log.exception(e) + self.state.save(self.log) + self.log.info("Script aborted") + raise SystemExit(1) from e + + def __get_random_contacts(self, num: int) -> List[dict]: + """Returns the specified number of random Google contacts""" + random_indices = random.sample(range(len(self.state.contacts)), num) # nosec + return [deepcopy(self.state.contacts[index]) for index in random_indices] + + def __get_random_fields(self, num: int) -> List[str]: + """Returns the specified number of random Google contact fields""" + random.shuffle(self.fields) + assert num <= len(self.fields), f"Not enough fields! ({num})" + return [self.fields[i] for i in range(num)] + + def __clean_metadata(self, contacts: List[dict]) -> None: + """Delete all metadata entries from a given Google contact""" + for contact in contacts: + contact.pop("resourceName", None) + contact.pop("etag", None) + contact.pop("metadata", None) + for key in tuple(contact): + if not isinstance(contact[key], list): + continue + for entry in contact[key]: + entry.pop("metadata", None) + + def __remove_contacts_from_list(self, contacts: List[dict]) -> None: + """Removes a contact from the list (.remove does not work because of deepcopies)""" + delete_resource_names = [contact["resourceName"] for contact in contacts] + self.state.contacts = [ + contact + for contact in self.state.contacts + if contact["resourceName"] not in delete_resource_names + ] + + def __update_contacts(self, num: int) -> List[dict]: + """Updates random target Google contacts based on random source contacts. + Returns the source contacts (not the updated ones)""" + # Chose random contacts and generate contact pairs + contacts = self.__get_random_contacts(num * 2) + contacts_to_update = contacts[:num] + contacts_to_copy_from = contacts[num:] + + # Update two contacts + update_list = [] + original_contacts = deepcopy(contacts_to_update) + self.__remove_contacts_from_list(contacts_to_update) + for contact_to_update, contact_to_copy_from in zip(contacts_to_update, contacts_to_copy_from): + # Update fields + fields = self.__get_random_fields(random.randint(1, 9)) # nosec + self.log.info( + f"Updating '{','.join(fields)}' on " + f"'{contact_to_update['names'][0]['displayName']}' " + f"('{contact_to_update['resourceName']}')" + ) + for field in fields: + contact_to_update[field] = deepcopy(contact_to_copy_from.get(field, [])) + for entry in contact_to_update[field]: + entry.pop("metadata", None) + + # Delete other fields + for key in tuple(contact_to_update): + if isinstance(contact_to_update[key], list) and key not in self.fields: + del contact_to_update[key] + + # Update contact + update_list.append(contact_to_update) + + # Update contacts + updated_contacts = self.google.update_contacts(update_list) + self.state.contacts += updated_contacts + + # Save original contacts to state + for contact in original_contacts: + if not self.state.original_contacts.get(contact["resourceName"], None): + self.state.original_contacts[contact["resourceName"]] = contact + + # Return source contacts + return contacts_to_copy_from + + def initial_chaos(self, num: int) -> None: + """Creates the given count of Monica contacts. + Produces some easy and some more complex contact matching cases for initial sync""" + # Generate easy matches with no changes + self.__initial_create_easy_matches(num) + + # Generate complex matches by swapping first names + self.__initial_create_complex_matches(num) + + def __initial_create_complex_matches(self, num: int) -> None: + """Creates the given contact count at Monica without changes.""" + contacts = self.__get_random_contacts(num) + contacts_rotated = contacts[1:] + contacts[0:1] + for i in range(len(contacts)): + contacts[i]["names"][0]["givenName"] = deepcopy(contacts_rotated[i]["names"][0]["givenName"]) + self.sync.create_monica_contact(contacts[i]) + + def __initial_create_easy_matches(self, num: int) -> None: + """Creates the given contact count at Monica without changes.""" + for contact in self.__get_random_contacts(num): + self.sync.create_monica_contact(contact) + + def delta_chaos(self, num: int) -> None: + """Updates and deletes the given count of Google contacts""" + # Update random contacts and delete their used source contacts + contacts = self.__update_contacts(num) + delete_mapping = { + contact["resourceName"]: contact["names"][0]["displayName"] for contact in contacts + } + self.google.delete_contacts(delete_mapping) + self.__remove_contacts_from_list(contacts) + self.state.deleted_contacts += contacts + + def full_chaos(self, num: int) -> None: + """Updates and creates the given count of Google contacts""" + # Update random contacts and recreate their used source contacts + contacts = self.__update_contacts(num) + # Clean metadata and id + self.__clean_metadata(contacts) + # Create contact + created_contacts = self.google.create_contacts(contacts) + self.state.contacts += created_contacts + self.state.created_contacts += created_contacts + + def syncback_chaos(self, num: int) -> None: + """Creates the given count of Monica-only contacts for syncback""" + # Ensure that there is no lonely Monica contact yet (could be sometimes) + for contact in self.monica.get_contacts(): + if str(contact["id"]) not in self.database.get_id_mapping().values(): + self.monica.delete_contact(contact["id"], contact["complete_name"]) + # Create random Monica contacts + for contact in self.__get_random_contacts(num): + created_contact = self.sync.create_monica_contact(contact) + self.state.created_syncback_contact_ids.append(str(created_contact["id"])) + + def database_chaos(self, num: int) -> None: + """Deletes the given count of database entries + and creates the given count of imaginary ones""" + # Delete entries + for contact in self.__get_random_contacts(num): + existing_entry = self.database.find_by_id(google_id=contact["resourceName"]) + assert existing_entry, f"No entry for {contact['resourceName']} found!" + self.database.delete(existing_entry.google_id, existing_entry.monica_id) + self.state.deleted_database_entries.append(existing_entry) + self.log.info( + f"Removed '{contact['resourceName']}' " + f"('{contact['names'][0]['displayName']} from database')" + ) + + # Create random entries + for num in range(1, num + 1): + new_entry = DatabaseEntry(f"google/randomEntry{num}", f"monica/randomEntry{num}") + self.database.insert_data(new_entry) + self.state.created_database_entries.append(new_entry) + self.log.info(f"Inserted 'google/randomEntry{num}' into database") + + def restore_contacts(self) -> None: + """Restore all manipulated Google contacts and database entries""" + # Restore updated contacts + self.__revert_updated_contacts() + + # Create deleted database entries + for entry in self.state.deleted_database_entries: + self.database.insert_data(entry) + self.log.info(f"Database row {entry.google_id} restored") + self.state.deleted_database_entries = [] + + # Delete created database entries + for entry in self.state.created_database_entries: + self.database.delete(entry.google_id, entry.monica_id) + self.log.info(f"Database row {entry.google_id} deleted") + self.state.created_database_entries = [] + + # Search created contacts during full sync + delete_mapping_1 = { + contact["resourceName"]: contact["names"][0]["displayName"] + for contact in self.state.created_contacts + } + # Search created contacts during sync back + delete_mapping_2 = {} + for monica_id in self.state.created_syncback_contact_ids: + deleted_entry = self.database.find_by_id(monica_id=monica_id) + if not deleted_entry: + raise DatabaseError(f"Could not find entry for syncback contact {monica_id}") + delete_mapping_2[deleted_entry.google_id] = deleted_entry.google_full_name + # Delete created contacts + self.google.delete_contacts({**delete_mapping_1, **delete_mapping_2}) + self.__remove_contacts_from_list(self.state.created_contacts) + self.state.created_contacts = [] + + # Create deleted contacts + self.__clean_metadata(self.state.deleted_contacts) + # Create contact + created_contacts = self.google.create_contacts(self.state.deleted_contacts) + self.state.contacts += created_contacts + self.state.deleted_contacts = [] + + def __revert_updated_contacts(self): + """Reverts all changes to Google contacts""" + changed_contacts = [ + contact + for contact in self.state.contacts + if contact["resourceName"] in self.state.original_contacts + ] + self.__remove_contacts_from_list(list(self.state.original_contacts.values())) + update_mapping = [ + (original_contact, changed_contact) + for original_contact in self.state.original_contacts.values() + for changed_contact in changed_contacts + if original_contact["resourceName"] == changed_contact["resourceName"] + ] + updated_contacts = [] + for original_contact, changed_contact in update_mapping: + # Revert changes + for key in tuple(original_contact): + if not isinstance(original_contact[key], list): + continue + changed_contact[key] = original_contact[key] + # Delete other fields + for key in tuple(changed_contact): + if isinstance(changed_contact[key], list) and key not in self.fields: + del changed_contact[key] + # Update contact + updated_contacts.append(changed_contact) + self.state.contacts += updated_contacts + self.google.update_contacts(updated_contacts) + self.state.original_contacts = {} + + def __create_logger(self) -> None: + """Creates the logger object""" + # Set logging configuration + if not os.path.exists(LOG_FOLDER): + os.makedirs(LOG_FOLDER) + log = logging.getLogger("monkey") + dotenv_log = logging.getLogger("dotenv.main") + log.setLevel(logging.INFO) + logging_format = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + log_filepath = join(LOG_FOLDER, LOG_FILENAME) + handler = logging.FileHandler(filename=log_filepath, mode="a", encoding="utf8") + handler.setLevel(logging.INFO) + handler.setFormatter(logging_format) + log.addHandler(logging.StreamHandler(sys.stdout)) + log.addHandler(handler) + dotenv_log.addHandler(handler) + self.log = log + + def __create_argument_parser(self) -> None: + """Creates the argument parser object""" + # Setup argument parser + parser = argparse.ArgumentParser(description="Syncs Google contacts to a Monica instance.") + parser.add_argument( + "-i", + "--initial", + action="store_true", + required=False, + help="produce two easy and two more complex contact matching cases for initial sync", + ) + parser.add_argument( + "-d", + "--delta", + action="store_true", + required=False, + help="update two and delete two Google contacts", + ) + parser.add_argument( + "-f", + "--full", + action="store_true", + required=False, + help="update two and create two Google contacts", + ) + parser.add_argument( + "-sb", + "--syncback", + action="store_true", + required=False, + help="create two Monica-only contacts for syncback", + ) + parser.add_argument( + "-c", + "--check", + action="store_true", + required=False, + help="delete two database entries and create two imaginary ones", + ) + parser.add_argument( + "-r", + "--restore", + action="store_true", + required=False, + help="recreate deleted Google contacts", + ) + parser.add_argument( + "-s", "--seed", type=int, required=False, help="custom seed for the random generator" + ) + parser.add_argument( + "-n", "--num", type=int, required=False, default=4, help="number of things to manipulate" + ) + + # Parse arguments + self.parser = parser + self.args = parser.parse_args() + + def __load_config(self) -> None: + """Loads the config from file or environment variables""" + # Load default config + self.log.info("Loading config (last value wins)") + default_config = find_dotenv(DEFAULT_CONFIG_FILEPATH, raise_error_if_not_found=True) + self.log.info(f"Loading default config from {default_config}") + default_config_values = dotenv_values(default_config) + user_config = find_dotenv() + if user_config: + # Load config from file + self.log.info(f"Loading file config from {user_config}") + file_config_values = dotenv_values(user_config) + else: + file_config_values = {} + # Load config from environment vars + self.log.info("Loading os environment config") + environment_config_values = dict(os.environ) + self.log.info("Config loading complete") + raw_config = {**default_config_values, **file_config_values, **environment_config_values} + + # Parse config + self.conf = Config(self.log, raw_config) + self.log.info("Config successfully parsed") + + def __create_sync_helper(self) -> None: + """Creates the main sync class object""" + # Create class objects + self.database = Database(self.log, self.conf.DATABASE_FILE) + self.google = Google( + self.log, + self.database, + self.conf.GOOGLE_CREDENTIALS_FILE, + self.conf.GOOGLE_TOKEN_FILE, + self.conf.GOOGLE_LABELS_INCLUDE, + self.conf.GOOGLE_LABELS_EXCLUDE, + self.args.initial, + ) + self.monica = Monica( + self.log, + self.database, + self.conf.TOKEN, + self.conf.BASE_URL, + self.conf.CREATE_REMINDERS, + self.conf.MONICA_LABELS_INCLUDE, + self.conf.MONICA_LABELS_EXCLUDE, + ) + self.sync = Sync( + self.log, + self.database, + self.monica, + self.google, + self.args.syncback, + self.args.check, + self.conf.DELETE_ON_SYNC, + self.conf.STREET_REVERSAL, + self.conf.FIELDS, + ) + + +if __name__ == "__main__": + Monkey().main() diff --git a/test/SetupToken.py b/test/SetupToken.py new file mode 100644 index 0000000..00bc09b --- /dev/null +++ b/test/SetupToken.py @@ -0,0 +1,115 @@ +"""Gets an API token from a new Monica instance""" + +import logging +import os +import re +import sys +from os.path import join +from time import sleep +from urllib.parse import unquote + +import requests +from bs4 import BeautifulSoup # type: ignore +from dotenv.main import load_dotenv, set_key # type: ignore +from requests import ConnectionError, ConnectTimeout, ReadTimeout + +LOG_FOLDER = "logs" +LOG_FILENAME = "setup.log" +try: + load_dotenv() + PROTOCOL, HOST, PORT = re.findall(r"(https?)://(.+?):(\d+)/api/?", os.getenv("BASE_URL", ""))[0] +except IndexError: + PROTOCOL, HOST, PORT = "http", "localhost", 8080 +ENV_FILE = ".env" +URL = f"{PROTOCOL}://{HOST}:{PORT}" + +# Set logging configuration +if not os.path.exists(LOG_FOLDER): + os.makedirs(LOG_FOLDER) +log = logging.getLogger("setup") +log.setLevel(logging.INFO) +logging_format = logging.Formatter("%(asctime)s %(levelname)s %(message)s") +log_filepath = join(LOG_FOLDER, LOG_FILENAME) +handler = logging.FileHandler(filename=log_filepath, mode="a", encoding="utf8") +handler.setLevel(logging.INFO) +handler.setFormatter(logging_format) +log.addHandler(handler) +log.addHandler(logging.StreamHandler(sys.stdout)) +log.info("Script started") +log.info(f"Host: {PROTOCOL}://{HOST}:{PORT}") + +try: + # Wait for Monica to be ready + log.info("Waiting for Monica to get ready") + waiting_time = 0 + max_time = 300 # Wait max. 5 minutes + while True: + try: + response = requests.get(f"{URL}/register", timeout=0.2) + if response.status_code == 200: + log.info(f"Ready after {waiting_time} seconds") + sleep(1) + break + except (ConnectTimeout, ConnectionError, ReadTimeout): + waiting_time += 1 + sleep(0.8) + if waiting_time > max_time: + raise TimeoutError(f"Waiting time ({max_time} seconds) exceeded!") + print(f"Waiting for Monica: {max_time - waiting_time} seconds remaining") + + # Get register token + log.info("Fetching register page") + response = requests.get(f"{URL}/register") + response.raise_for_status() + cookies = response.cookies + soup = BeautifulSoup(response.text, "html.parser") + inputs = soup.find_all("input") + token = [ + input_tag["value"] for input_tag in soup.find_all("input") if input_tag.get("name") == "_token" + ][0] + + # Register new user + data = { + "_token": token, + "email": "some.user@email.com", + "first_name": "Some", + "last_name": "User", + "password": "0JS65^Pp%kFyQh1q5vPx7Mzcj", + "password_confirmation": "0JS65^Pp%kFyQh1q5vPx7Mzcj", + "policy": "policy", + "lang": "en", + } + log.info("Registering new user") + response = requests.post(f"{URL}/register", cookies=response.cookies, data=data) + response.raise_for_status() + + # Create api token + headers = { + "X-XSRF-TOKEN": unquote(response.cookies.get("XSRF-TOKEN")), + } + data = {"name": "PythonTestToken", "scopes": [], "errors": []} + log.info("Requesting access token") + response = requests.post( + f"{URL}/oauth/personal-access-tokens", + headers=headers, + cookies=response.cookies, + json=data, + ) + response.raise_for_status() + + # Extract token from response + access_token: str = response.json().get("accessToken", "error") + + # Save token to .env file + open(ENV_FILE, "a+").close() + log.info(f"Saving access token '{access_token[:10]}...' to '{ENV_FILE}'") + set_key(ENV_FILE, "TOKEN", access_token) + + log.info("Script finished") + +except Exception as e: + log.exception(e) + log.info("Script aborted") + print(f"\nScript aborted: {type(e).__name__}: {str(e)}") + print(f"See log file ({join(LOG_FOLDER, LOG_FILENAME)}) for all details") + raise SystemExit(1) from e diff --git a/test/UpdateToken.py b/test/UpdateToken.py new file mode 100644 index 0000000..3298df0 --- /dev/null +++ b/test/UpdateToken.py @@ -0,0 +1,44 @@ +import os +from base64 import b64encode + +import requests +from nacl import encoding, public # type: ignore + +REPO = os.getenv("GITHUB_REPOSITORY") +SECRET_NAME = "GOOGLE_TOKEN" # nosec +REPO_TOKEN = os.getenv("REPO_TOKEN") + + +def encrypt(public_key: str, secret_value: str) -> str: + """Encrypt a Unicode string using the public key.""" + public_key = public.PublicKey(public_key.encode("utf-8"), encoding.Base64Encoder()) + sealed_box = public.SealedBox(public_key) + encrypted = sealed_box.encrypt(secret_value.encode("utf-8")) + return b64encode(encrypted).decode("utf-8") + + +# Read token +with open("data/token.pickle", "r") as base64_token: + creds_base64 = base64_token.read() + +# Get repo public key +headers = {"accept": "application/vnd.github.v3+json", "Authorization": f"token {REPO_TOKEN}"} +response = requests.get( + f"https://api.github.com/repos/{REPO}/actions/secrets/public-key", headers=headers +) +response.raise_for_status() +data = response.json() +public_key_id: str = data["key_id"] +public_key: str = data["key"] + +# Encrypt secret +encrypted_secret = encrypt(public_key, creds_base64) +body = {"encrypted_value": encrypted_secret, "key_id": public_key_id} + +# Set secret +response = requests.put( + f"https://api.github.com/repos/{REPO}/actions/secrets/{SECRET_NAME}", + headers=headers, + json=body, +) +response.raise_for_status() diff --git a/test/docker-compose-monica.yml b/test/docker-compose-monica.yml new file mode 100644 index 0000000..7dad101 --- /dev/null +++ b/test/docker-compose-monica.yml @@ -0,0 +1,42 @@ +version: "3.4" + +services: + monica: + image: monica:apache + depends_on: + - db + ports: + - 8080:80 + environment: + - DB_HOST=db + - DB_USERNAME=monica + - DB_PASSWORD=secret + - RATE_LIMIT_PER_MINUTE_API=1000 + volumes: + - data:/var/www/html/storage + restart: always + networks: + - monica_network + + db: + image: mysql:5.7 + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=true + - MYSQL_DATABASE=monica + - MYSQL_USER=monica + - MYSQL_PASSWORD=secret + volumes: + - mysql:/var/lib/mysql + restart: always + networks: + - monica_network + +networks: + monica_network: + name: monica_network + +volumes: + data: + name: data + mysql: + name: mysql diff --git a/test/docker-compose-sync-check.yml b/test/docker-compose-sync-check.yml new file mode 100644 index 0000000..cf86987 --- /dev/null +++ b/test/docker-compose-sync-check.yml @@ -0,0 +1,4 @@ +services: + python: + # Do a database check + command: python -u GMSync.py -c diff --git a/test/docker-compose-sync-delta.yml b/test/docker-compose-sync-delta.yml new file mode 100644 index 0000000..43ef03f --- /dev/null +++ b/test/docker-compose-sync-delta.yml @@ -0,0 +1,4 @@ +services: + python: + # Do a delta sync + command: python -u GMSync.py -d diff --git a/test/docker-compose-sync-full.yml b/test/docker-compose-sync-full.yml new file mode 100644 index 0000000..f7035e1 --- /dev/null +++ b/test/docker-compose-sync-full.yml @@ -0,0 +1,4 @@ +services: + python: + # Do a full sync + command: python -u GMSync.py -f diff --git a/test/docker-compose-sync-initial.yml b/test/docker-compose-sync-initial.yml new file mode 100644 index 0000000..6404756 --- /dev/null +++ b/test/docker-compose-sync-initial.yml @@ -0,0 +1,4 @@ +services: + python: + # Do an initial sync + command: python -u GMSync.py -i diff --git a/test/docker-compose-sync-syncback.yml b/test/docker-compose-sync-syncback.yml new file mode 100644 index 0000000..af32a23 --- /dev/null +++ b/test/docker-compose-sync-syncback.yml @@ -0,0 +1,4 @@ +services: + python: + # Do a sync back + command: python -u GMSync.py -sb diff --git a/test/docker-compose-sync.yml b/test/docker-compose-sync.yml new file mode 100644 index 0000000..5651d27 --- /dev/null +++ b/test/docker-compose-sync.yml @@ -0,0 +1,28 @@ +version: "3.8" + +services: + python: + image: antonplagemann/google-monica-sync:next + + environment: + # The Monica API token + - TOKEN=${TOKEN} + # Your Monica base url (ends with /api) + - BASE_URL=http://monica:80/api + # Script is running within GitHub Actions + - CI=1 + + # Put credentials, sync database and token files in ./data + volumes: + - ../data:/usr/app/data + # Remove the next line if you do not want to access the logs + - ../logs:/usr/app/logs + + # Adjust command if needed (-u needed for getting console output) + command: python -u GMSync.py --help + networks: + - monica_network + +networks: + monica_network: + name: monica_network diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..4f9da9c --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,2 @@ +PyNaCl>=1.4.0 +beautifulsoup4>=4.10.0