-
Notifications
You must be signed in to change notification settings - Fork 368
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
## Describe your changes Refactoring track_deletes logic so we don't need an extra snapshot tables. The logic is now simple: - `upsert` adds/updates records (or set `deletedAt`), including updating the sync_job_id that serves as a generationId - when the sync is finish, if `track_deletes == true` then every records without a `deletedAt` that doesn't have the same `sync_job_id` as the current one is marked as deleted ## Issue ticket number and link https://linear.app/nango/issue/NAN-599/track-deletes-refactoring ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [x] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is:
- Loading branch information
Showing
29 changed files
with
698 additions
and
1,114 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
37 changes: 37 additions & 0 deletions
37
packages/shared/lib/db/migrations/20240322141605_undelete_records_reset_createdat.cjs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| const RECORDS_TABLE = '_nango_sync_data_records'; | ||
| exports.up = async function (knex) { | ||
| await knex.transaction((trx) => { | ||
| return trx | ||
| .raw( | ||
| ` | ||
| CREATE OR REPLACE FUNCTION ${RECORDS_TABLE}_reset_created_at() | ||
| RETURNS TRIGGER AS $$ | ||
| BEGIN | ||
| IF OLD.external_deleted_at IS NOT NULL AND NEW.external_deleted_at IS NULL THEN | ||
| NEW.created_at = NOW(); | ||
| NEW.updated_at = NOW(); | ||
| NEW.external_is_deleted = FALSE; | ||
| END IF; | ||
| RETURN NEW; | ||
| END; | ||
| $$ LANGUAGE plpgsql; | ||
| ` | ||
| ) | ||
| .then(function () { | ||
| return trx.raw(` | ||
| CREATE TRIGGER ${RECORDS_TABLE}_reset_created_at_trigger | ||
| BEFORE UPDATE ON ${RECORDS_TABLE} | ||
| FOR EACH ROW | ||
| EXECUTE FUNCTION ${RECORDS_TABLE}_reset_created_at(); | ||
| `); | ||
| }); | ||
| }); | ||
| }; | ||
|
|
||
| exports.down = async function (knex) { | ||
| await knex.transaction((trx) => { | ||
| return trx.raw(`DROP TRIGGER IF EXISTS ${RECORDS_TABLE}_reset_created_at_trigger ON ${RECORDS_TABLE};`).then(function () { | ||
| return trx.raw(`DROP FUNCTION IF EXISTS ${RECORDS_TABLE}_reset_created_at();`); | ||
| }); | ||
| }); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,40 +1,26 @@ | ||
| import { schema } from '../database.js'; | ||
| import configService from '../../services/config.service.js'; | ||
| import environmentService from '../../services/environment.service.js'; | ||
| import type { Config as ProviderConfig } from '../../models/Provider.js'; | ||
| import type { Environment } from '../../models/Environment.js'; | ||
|
|
||
| export const createConfigSeeds = async (environmentName = ''): Promise<Environment[]> => { | ||
| let result: Environment[]; | ||
| if (environmentName) { | ||
| result = [(await environmentService.createEnvironment(0, environmentName))!]; | ||
| } else { | ||
| result = await schema().select('*').from('_nango_environments'); | ||
| } | ||
|
|
||
| for (const row of result) { | ||
| const { id: environment_id } = row; | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'notion', | ||
| environment_id | ||
| } as ProviderConfig); | ||
| } | ||
|
|
||
| return result; | ||
| export const createConfigSeeds = async (env: Environment): Promise<void> => { | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id: env.id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id: env.id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'google', | ||
| environment_id: env.id | ||
| } as ProviderConfig); | ||
| await configService.createProviderConfig({ | ||
| unique_key: Math.random().toString(36).substring(7), | ||
| provider: 'notion', | ||
| environment_id: env.id | ||
| } as ProviderConfig); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| import type { Environment } from '../../models/Environment.js'; | ||
| import environmentService from '../../services/environment.service.js'; | ||
|
|
||
| export async function createEnvironmentSeed(envName: string = 'test'): Promise<Environment> { | ||
| const env = await environmentService.createEnvironment(0, envName); | ||
| if (!env) { | ||
| throw new Error('Failed to create environment'); | ||
| } | ||
| return env; | ||
| } |
Oops, something went wrong.