feat: field mapping and custom schema (revertinc#207)

* prisma migration to add field mapping schemas * seed hs note model (unverified) * seed hs note model (verified) * seed hs note model * transform note acc to schema mapping * (approach 2) transform note acc to schema field mapping * (approach 2) seed all note fields and fix single note structure * minor fix * common func to transform field mapping to model * disunify using field mapping * allow connections to override field mapping * support custom objects in schema; show custom fields as additional * add winston and morgan logger * Field mapping for contact; support custom field mapping to custom crm field * add field mapping config * common unify and disunify methods * seed contact field mapping * use lodash get to transform nested mappings * unify and disunify for contact * disunify based on account config * unify and disunify for company * unify and disunify for deal * unify and disunify for event * unify and disunify for lead * unify and disunify for task * unify and disunify for user * minor refactor * handle 2 level nesting and fix mapping * make target field optional; multi level nesting * fix * fix * fix obj merge * fix pipedrive get format * preprocess * preprocess unify and disunify * disunify postprocess * flatten before disunify * pipedrive fix contact create * minor fixes * fix pipedrive lead disunify * chore: add account id to morgan logs * chore: cleanup * chore: cleanup * chore: cleanup * feat: ui for field mapping (revertinc#242) * feat: processing ui * feat: immplememt sse and redis pubsub * feat: read sse data to render failed ui * minor changes * feat: api to fetch mappable details and crm object properties * feat: api to create connection field mapping * feat: add custom field mapped to connection schema * feat: reallly basic ui for std field mapping * feat: reallly basic ui for custom field mapping * feat: add styling * fix: fix add btn order * feat: check for t_id in sse msg; send private token for calling further apis * feat: save mappings on click * feat: fix ui * feat: show done screen * feat: mark form dirty if empty fields * feat: show done stage if config doesnt exist * chore: fix pr comments * chore: use temp tenant token for auth; ui fixes * chore: fix dropdown caret; redis timeout * chore: pubsub fix * chore: cleanup * chore: cleanup * chore: resolve conflicts * feat: add remove mapping btn and divider * fix: fix logger * chore: cleanup * chore: move endpoints to fern * update yarn.lock * fix issues * add hubspot create custom properties api --------- Co-authored-by: jatin <sandilya.jatin@gmail.com>
hrutik7 · Nov 15, 2023 · 38c3731 · 38c3731
1 parent c8d275e
commit 38c3731
Show file tree

Hide file tree

Showing 9 changed files with 172 additions and 39 deletions.
diff --git a/fern/definition/crm/properties.yml b/fern/definition/crm/properties.yml
@@ -1,5 +1,69 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
 
+imports:
+    errors: ../common/errors.yml
+    types: ../common/types.yml
+
+types:
+    FieldDetailsType:
+        properties:
+            fieldName: string
+            fieldType: string
+            fieldDescription: string
+    FieldDetailsTypeRequest:
+        extends: FieldDetailsType
+        properties:
+            groupName: string
+            type: string
+    SetObjectPropertiesRequest: FieldDetailsTypeRequest
+    GetObjectPropertiesResponse: unknown
+    SetObjectPropertiesResponse: unknown
+
+service:
+    base-path: /crm/properties
+    auth: false
+    headers:
+        x-revert-api-token:
+            type: string
+            docs: Your official API key for accessing revert apis.
+        x-revert-t-id:
+            type: string
+            docs: The unique customer id used when the customer linked their account.
+        x-api-version:
+            type: optional<string>
+            docs: Optional Revert API version you're using. If missing we default to the latest version of the API.
+    audiences:
+        - external
+    endpoints:
+        getObjectProperties:
+            docs: Get object properties for connection
+            method: GET
+            path: /{objectName}
+            path-parameters:
+                objectName: types.StandardObject
+            request:
+                name: GetObjectPropertiesRequest
+            response: GetObjectPropertiesResponse
+            errors:
+                - errors.UnAuthorizedError
+                - errors.InternalServerError
+                - errors.NotFoundError
+        setCustomProperties:
+            docs: Set custom properties on an object for a given connection
+            method: POST
+            path: /{objectName}
+            path-parameters:
+                objectName: types.StandardObject
+            request:
+                name: SetObjectCustomPropertiesRequest
+                body: SetObjectPropertiesRequest
+            response: SetObjectPropertiesResponse
+            errors:
+                - errors.UnAuthorizedError
+                - errors.InternalServerError
+                - errors.NotFoundError
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+
 imports:
     errors: ../common/errors.yml
     types: ../common/types.yml

diff --git a/packages/backend/constants/common.ts b/packages/backend/constants/common.ts
@@ -43,12 +43,11 @@ export const DEFAULT_SCOPE = {
 };
 
 export const mapIntegrationIdToIntegrationName = {
-    [TP_ID.hubspot]: 'Hubspot',
-    [TP_ID.pipedrive]: 'Pipedrive',
-    [TP_ID.sfdc]: 'Salesforce',
-    [TP_ID.zohocrm]: 'Zoho',
-    [TP_ID.slack]: 'Slack',
-};
+    [TP_ID.hubspot]: "Hubspot",
+    [TP_ID.pipedrive]: "Pipedrive",
+    [TP_ID.sfdc]: "Salesforce",
+    [TP_ID.zohocrm]: "Zoho",
+}
 
 export const rootSchemaMappingId = 'revertRootSchemaMapping';
 
@@ -63,7 +62,7 @@ export enum StandardObjects {
     user = 'user',
 }
 
-export const objectNameMapping: Record<string, Record<CRM_TP_ID, string | undefined>> = {
+export const objectNameMapping: Record<string, Record<TP_ID, string | undefined>> = {
     [StandardObjects.company]: {
         [TP_ID.hubspot]: 'companies',
         [TP_ID.pipedrive]: 'organization',

diff --git a/packages/backend/helpers/crm/transform/unify.ts b/packages/backend/helpers/crm/transform/unify.ts
@@ -1,5 +1,5 @@
-import { accountFieldMappingConfig } from '@prisma/client';
-import { CRM_TP_ID, StandardObjects } from '../../../constants/common';
+import { TP_ID, accountFieldMappingConfig } from '@prisma/client';
+import { StandardObjects } from '../../../constants/common';
 import { transformFieldMappingToModel } from '.';
 import { preprocessUnifyObject } from './preprocess';
 
@@ -11,7 +11,7 @@ export async function unifyObject<T extends Record<string, any>, K>({
     accountFieldMappingConfig,
 }: {
     obj: T;
-    tpId: CRM_TP_ID;
+    tpId: TP_ID;
     objType: StandardObjects;
     tenantSchemaMappingId?: string;
     accountFieldMappingConfig?: accountFieldMappingConfig;

diff --git a/packages/backend/helpers/logger.ts b/packages/backend/helpers/logger.ts
@@ -8,6 +8,30 @@ const enumerateErrorFormat = winston.format((info) => {
     return info;
 });
 
+const logger = winston.createLogger({
+    level: process.env.NODE_ENV === 'development' ? 'debug' : 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        enumerateErrorFormat(),
+        process.env.NODE_ENV === 'development' ? winston.format.colorize() : winston.format.uncolorize(),
+        winston.format.splat(),
+        winston.format.printf(({ timestamp, level, message }) => `[${timestamp}] ${level}: ${message}`)
+    ),
+    transports: [
+        new winston.transports.Console({
+            stderrLevels: ['error'],
+        }),
+    ],
+});
+import winston from 'winston';
+
+const enumerateErrorFormat = winston.format((info) => {
+    if (info instanceof Error) {
+        Object.assign(info, { message: info.stack });
+    }
+    return info;
+});
+
 const logger = winston.createLogger({
     level: process.env.NODE_ENV === 'development' ? 'debug' : 'info',
     format: winston.format.combine(
@@ -37,14 +61,6 @@ const logInfo = (message: string, ...args: any[]) => {
     logger.info(`${message} %o`, args);
 };
 
-const logWarn = (message: string, ...args: any[]) => {
-    if (!args || !args.length) {
-        logger.warn(message);
-        return;
-    }
-    logger.warn(`${message} %o`, args);
-};
-
 const logDebug = (message: string, arg: any) => {
     if (!arg) {
         logger.debug(message);

diff --git a/packages/backend/package.json b/packages/backend/package.json
@@ -60,6 +60,8 @@
         "@types/express": "^4.17.13",
         "axios": "^0.27.2",
         "better-sse": "^0.9.0",
+        "bull": "^4.8.5",
+        "constructs": "^10.0.0",
         "cors": "^2.8.5",
         "discord.js": "13.x",
         "dotenv": "^16.3.1",

diff --git a/packages/backend/prisma/seed.ts b/packages/backend/prisma/seed.ts
@@ -1,6 +1,9 @@
 import { randomUUID } from 'crypto';
 import { ENV, PrismaClient, TP_ID, fieldMappings } from '@prisma/client';
 import { StandardObjects, rootSchemaMappingId } from '../constants/common';
+import { randomUUID } from 'crypto';
+import { ENV, PrismaClient, TP_ID, fieldMappings } from '@prisma/client';
+import { StandardObjects, rootSchemaMappingId } from '../constants/common';
 const prisma = new PrismaClient();
 
 async function main() {
@@ -749,8 +752,7 @@ async function main() {
             },
         ],
     };
-    const allSchemas = Object.keys(allFields).map((obj) => {
-    const allSchemas = Object.keys(allFields).map((obj) => {
+    const allSchemas = Object.keys(allFields).map(obj => {
         return {
             id: randomUUID(),
             fields: allFields[obj as keyof typeof allFields].map((n) => n.target_field_name),
@@ -765,29 +767,28 @@ async function main() {
                     data: allSchemas,
                 },
             },
-            object_schema_ids: allSchemas.map((s) => s.id),
+            object_schema_ids: allSchemas.map(s => s.id)
         },
     });
 
     const fieldMappingForAll: fieldMappings[] = [];
-    Object.values(StandardObjects).forEach((obj) => {
-        Object.values(TP_ID).forEach(async (tpId) => {
-            if (tpId === 'slack') return;
-            const objSchema = allSchemas.find((s) => s.object === obj);
-            const fieldMappings = objSchema?.fields.map((field) => ({
-                id: randomUUID(),
-                source_tp_id: tpId,
-                schema_id: objSchema.id,
-                source_field_name: allFields[obj as 'note' | 'contact'].find((a) => a.target_field_name === field)
-                    ?.source_field_name[tpId]!,
-                target_field_name: field,
-                is_standard_field: true,
-            }));
-            if (fieldMappings) {
-                fieldMappingForAll.push(...fieldMappings);
-            }
-        });
-    });
+        Object.values(StandardObjects).forEach(obj => {
+            Object.values(TP_ID).forEach(async (tpId) => {
+                const objSchema = allSchemas.find(s => s.object === obj);
+                const fieldMappings = objSchema?.fields.map((field) => ({
+                    id: randomUUID(),
+                    source_tp_id: tpId,
+                    schema_id: objSchema.id,
+                    source_field_name: allFields[obj as "note" | "contact"].find(a => a.target_field_name === field)?.source_field_name[tpId]!,
+                    target_field_name: field,
+                    is_standard_field: true,
+                }));
+                if (fieldMappings) {
+                    fieldMappingForAll.push(...fieldMappings);
+                }
+            })
+
+        }) 
     await prisma.fieldMappings.createMany({
         data: fieldMappingForAll,
     });

diff --git a/packages/backend/routes/index.ts b/packages/backend/routes/index.ts
@@ -37,6 +37,8 @@ import chatRouter from './v1/chat';
 import { usersService } from '../services/chat/users';
 import { channelsService } from '../services/chat/channels';
 import { messageService } from '../services/chat/message';
+import { fieldMappingService } from './v1/crm/fieldMapping';
+import { propertiesService } from './properties';
 
 const router = express.Router();
 

diff --git a/packages/client/src/common/oauth/index.tsx b/packages/client/src/common/oauth/index.tsx
@@ -190,6 +190,7 @@ export const OAuthCallback = (props) => {
                         setIsLoading(false);
                         console.error(err);
                         setStatus('Errored out');
+                        window.close();
                     });
             }
         }

diff --git a/yarn.lock b/yarn.lock
@@ -3618,6 +3618,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@dabh/diagnostics@npm:^2.0.2":
+  version: 2.0.3
+  resolution: "@dabh/diagnostics@npm:2.0.3"
+  dependencies:
+    colorspace: 1.1.x
+    enabled: 2.0.x
+    kuler: ^2.0.0
+  checksum: 4879600c55c8315a0fb85fbb19057bad1adc08f0a080a8cb4e2b63f723c379bfc4283b68123a2b078d367b327dd8df12fcb27464efe791addc0a48b9df6d79a1
+  languageName: node
+  linkType: hard
+
 "@discordjs/builders@npm:^0.16.0":
   version: 0.16.0
   resolution: "@discordjs/builders@npm:0.16.0"
@@ -8926,8 +8937,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+<<<<<<< HEAD
 <<<<<<< HEAD
 =======
+=======
+"@types/morgan@npm:^1.9.5":
+  version: 1.9.6
+  resolution: "@types/morgan@npm:1.9.6"
+  dependencies:
+    "@types/node": "*"
+  checksum: 6525248325a74342f929c958be69c0840c8f3a288e003a8904319cae92e531f17a8aa2700701e66775adcca7f9506dd630fec2f95dc04a3e73add04fde42aab8
+  languageName: node
+  linkType: hard
+
+>>>>>>> 68bce1a (feat: field mapping and custom schema (#207))
 "@types/mkdirp@npm:^0.5.2":
   version: 0.5.2
   resolution: "@types/mkdirp@npm:0.5.2"
@@ -22425,6 +22448,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"morgan@npm:^1.10.0":
+  version: 1.10.0
+  resolution: "morgan@npm:1.10.0"
+  dependencies:
+    bson: ^4.7.0
+    kareem: 2.5.1
+    mongodb: 4.14.0
+    mpath: 0.9.0
+    mquery: 4.0.3
+    ms: 2.1.3
+    sift: 16.0.1
+  checksum: b6aa9c51d367112d534c54db09c43cd661da52d9b2e9488caba082c771844f84962b7da74564bbfc2e6e6a0b29ed35d75619973f3e1f1d917fbd0a8a3be6a9cd
+  languageName: node
+  linkType: hard
+
 "morgan@npm:^1.10.0":
   version: 1.10.0
   resolution: "morgan@npm:1.10.0"
@@ -25845,8 +25883,18 @@ __metadata:
   version: 3.0.0
   resolution: "redis-parser@npm:3.0.0"
   dependencies:
+<<<<<<< HEAD
     redis-errors: ^1.0.0
   checksum: 89290ae530332f2ae37577647fa18208d10308a1a6ba750b9d9a093e7398f5e5253f19855b64c98757f7129cccce958e4af2573fdc33bad41405f87f1943459a
+=======
+    "@redis/bloom": 1.2.0
+    "@redis/client": 1.5.9
+    "@redis/graph": 1.1.0
+    "@redis/json": 1.0.4
+    "@redis/search": 1.1.3
+    "@redis/time-series": 1.0.5
+  checksum: 1626d0d739e5f2047824b77b472967af5938fba488a039bd0a5ad90814eb81cb29b7939e16adf0bc39b133165ec1bf147e4f944d0c54a2f9c32c8f004b015940
+>>>>>>> 68bce1a (feat: field mapping and custom schema (#207))
   languageName: node
   linkType: hard