NEOS-504 Reworks Transformer(s) Forms to no longer user DTO Type (#938)

nucleuscloud · Dec 20, 2023 · 5c792ab · 5c792ab
1 parent cd626f6
commit 5c792ab
Show file tree

Hide file tree

Showing 52 changed files with 842 additions and 719 deletions.
diff --git a/frontend/apps/web/app/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx b/frontend/apps/web/app/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx
@@ -30,7 +30,11 @@ import { useToast } from '@/components/ui/use-toast';
 import { useGetConnectionSchema } from '@/libs/hooks/useGetConnectionSchema';
 import { useGetJob } from '@/libs/hooks/useGetJob';
 import { getErrorMessage } from '@/util/util';
-import { TransformerFormValues } from '@/yup-validations/jobs';
+import {
+  JobMappingTransformerForm,
+  convertJobMappingTransformerFormToJobMappingTransformer,
+  convertJobMappingTransformerToForm,
+} from '@/yup-validations/jobs';
 import { yupResolver } from '@hookform/resolvers/yup';
 import {
   DatabaseColumn,
@@ -42,7 +46,6 @@ import {
   JobMappingTransformer,
   JobSource,
   JobSourceOptions,
-  TransformerConfig,
   UpdateJobSourceConnectionRequest,
   UpdateJobSourceConnectionResponse,
 } from '@neosync/sdk';
@@ -84,19 +87,22 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
 
   const allJobMappings =
     schema?.schemas.map((r) => {
+      const t: JobMappingTransformerForm = {
+        source: '',
+        config: { case: '', value: {} },
+      };
       return {
         ...r,
-        transformer: new JobMappingTransformer({}) as TransformerFormValues,
+        transformer: t,
       };
     }) || [];
-
   const form = useForm<SingleTableSchemaFormValues>({
     resolver: yupResolver(SINGLE_TABLE_SCHEMA_FORM_SCHEMA),
     defaultValues: {
       mappings: [],
       numRows: 0,
-      schema: 'foo',
-      table: 'bar',
+      schema: '',
+      table: '',
     },
     values: getJobSource(data?.job, schema?.schemas),
   });
@@ -210,6 +216,8 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
                       )
                       .map((r) => {
                         return {
+                          schema: formValues.schema,
+                          table: value,
                           column: r.column,
                           dataType: r.dataType,
                           transformer: r.transformer,
@@ -249,7 +257,13 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
               <FormLabel>Number of Rows</FormLabel>
               <FormDescription>The number of rows to generate.</FormDescription>
               <FormControl>
-                <Input value={field.value} onChange={field.onChange} />
+                <Input
+                  type="number"
+                  {...field}
+                  onChange={(e) => {
+                    field.onChange(e.target.valueAsNumber);
+                  }}
+                />
               </FormControl>
               <FormMessage />
             </FormItem>
@@ -310,19 +324,25 @@ function getJobSource(
       schemaMap[c.schema] = {
         [c.table]: {
           [c.column]: {
-            transformer: c?.transformer ?? new JobMappingTransformer({}),
+            transformer: convertJobMappingTransformerToForm(
+              c?.transformer ?? new JobMappingTransformer({})
+            ),
           },
         },
       };
     } else if (!schemaMap[c.schema][c.table]) {
       schemaMap[c.schema][c.table] = {
         [c.column]: {
-          transformer: c?.transformer ?? new JobMappingTransformer({}),
+          transformer: convertJobMappingTransformerToForm(
+            c?.transformer ?? new JobMappingTransformer({})
+          ),
         },
       };
     } else {
       schemaMap[c.schema][c.table][c.column] = {
-        transformer: c.transformer ?? new JobMappingTransformer({}),
+        transformer: convertJobMappingTransformerToForm(
+          c.transformer ?? new JobMappingTransformer({})
+        ),
       };
     }
   });
@@ -337,15 +357,17 @@ function getJobSource(
         c.column
       );
       const transformer =
-        colMapping?.transformer ?? new JobMappingTransformer({});
+        colMapping?.transformer ??
+        convertJobMappingTransformerToForm(new JobMappingTransformer({}));
 
       return {
+        schema: schema,
+        table: table,
         column: c.column,
         dataType: c.dataType,
-        transformer: transformer as TransformerFormValues,
+        transformer: transformer,
       };
     });
-
   return {
     mappings: mappings,
     numRows: numRows,
@@ -370,16 +392,14 @@ async function updateJobConnection(
         new UpdateJobSourceConnectionRequest({
           id: job.id,
           mappings: values.mappings.map((m) => {
-            const jmt = new JobMappingTransformer({
-              source: m.transformer.source,
-              config: m.transformer.config as TransformerConfig,
-            });
-
             return new JobMapping({
               schema: values.schema,
               table: values.table,
               column: m.column,
-              transformer: jmt,
+              transformer:
+                convertJobMappingTransformerFormToJobMappingTransformer(
+                  m.transformer
+                ),
             });
           }),
           source: new JobSource({

diff --git a/frontend/apps/web/app/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx b/frontend/apps/web/app/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx
@@ -30,9 +30,11 @@ import { useGetConnections } from '@/libs/hooks/useGetConnections';
 import { useGetJob } from '@/libs/hooks/useGetJob';
 import { getErrorMessage } from '@/util/util';
 import {
+  JobMappingTransformerForm,
   SCHEMA_FORM_SCHEMA,
   SOURCE_FORM_SCHEMA,
-  TransformerFormValues,
+  convertJobMappingTransformerFormToJobMappingTransformer,
+  convertJobMappingTransformerToForm,
 } from '@/yup-validations/jobs';
 import { yupResolver } from '@hookform/resolvers/yup';
 import {
@@ -69,7 +71,7 @@ export interface SchemaMap {
   [schema: string]: {
     [table: string]: {
       [column: string]: {
-        transformer: JobMappingTransformer;
+        transformer: JobMappingTransformerForm;
       };
     };
   };
@@ -263,16 +265,14 @@ async function updateJobConnection(
         new UpdateJobSourceConnectionRequest({
           id: job.id,
           mappings: values.mappings.map((m) => {
-            const jmt = new JobMappingTransformer({
-              source: m.transformer.source,
-              config: m.transformer.config as TransformerConfig,
-            });
-
             return new JobMapping({
               schema: m.schema,
               table: m.table,
               column: m.column,
-              transformer: jmt,
+              transformer:
+                convertJobMappingTransformerFormToJobMappingTransformer(
+                  m.transformer
+                ),
             });
           }),
           source: new JobSource({
@@ -363,8 +363,26 @@ function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
       schemaMap[c.schema] = {
         [c.table]: {
           [c.column]: {
-            transformer:
+            transformer: convertJobMappingTransformerToForm(
               c?.transformer ??
+                new JobMappingTransformer({
+                  source: 'passthrough',
+                  config: new TransformerConfig({
+                    config: {
+                      case: 'passthroughConfig',
+                      value: new Passthrough({}),
+                    },
+                  }),
+                })
+            ),
+          },
+        },
+      };
+    } else if (!schemaMap[c.schema][c.table]) {
+      schemaMap[c.schema][c.table] = {
+        [c.column]: {
+          transformer: convertJobMappingTransformerToForm(
+            c.transformer ??
               new JobMappingTransformer({
                 source: 'passthrough',
                 config: new TransformerConfig({
@@ -373,15 +391,14 @@ function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
                     value: new Passthrough({}),
                   },
                 }),
-              }),
-          },
+              })
+          ),
         },
       };
-    } else if (!schemaMap[c.schema][c.table]) {
-      schemaMap[c.schema][c.table] = {
-        [c.column]: {
-          transformer:
-            c.transformer ??
+    } else {
+      schemaMap[c.schema][c.table][c.column] = {
+        transformer: convertJobMappingTransformerToForm(
+          c.transformer ??
             new JobMappingTransformer({
               source: 'passthrough',
               config: new TransformerConfig({
@@ -390,22 +407,8 @@ function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
                   value: new Passthrough({}),
                 },
               }),
-            }),
-        },
-      };
-    } else {
-      schemaMap[c.schema][c.table][c.column] = {
-        transformer:
-          c.transformer ??
-          new JobMappingTransformer({
-            source: 'passthrough',
-            config: new TransformerConfig({
-              config: {
-                case: 'passthroughConfig',
-                value: new Passthrough({}),
-              },
-            }),
-          }),
+            })
+        ),
       };
     }
   });
@@ -420,15 +423,17 @@ function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
       dataType: c.dataType,
       transformer:
         colMapping?.transformer ??
-        new JobMappingTransformer({
-          source: 'passthrough',
-          config: new TransformerConfig({
-            config: {
-              case: 'passthroughConfig',
-              value: new Passthrough({}),
-            },
-          }),
-        }),
+        convertJobMappingTransformerToForm(
+          new JobMappingTransformer({
+            source: 'passthrough',
+            config: new TransformerConfig({
+              config: {
+                case: 'passthroughConfig',
+                value: new Passthrough({}),
+              },
+            }),
+          })
+        ),
     };
   });
   const destinationIds = job?.destinations.map((d) => d.connectionId);
@@ -443,7 +448,7 @@ function getJobSource(job?: Job, schema?: DatabaseColumn[]): SourceFormValues {
     sourceId: getConnectionIdFromSource(job.source) || '',
     mappings: values.mappings.map((mapping) => ({
       ...mapping,
-      transformer: mapping.transformer as TransformerFormValues,
+      transformer: mapping.transformer as JobMappingTransformerForm,
     })),
   };
 
@@ -476,7 +481,7 @@ export function getColumnMapping(
   schema: string,
   table: string,
   column: string
-): { transformer: JobMappingTransformer } | undefined {
+): { transformer: JobMappingTransformerForm } | undefined {
   if (!schemaMap[schema]) {
     return;
   }
@@ -504,7 +509,9 @@ async function getUpdatedValues(
   const mappings = schemaRes.schemas.map((r) => {
     return {
       ...r,
-      transformer: new JobMappingTransformer({}),
+      transformer: convertJobMappingTransformerToForm(
+        new JobMappingTransformer({})
+      ),
     };
   });
 
@@ -519,10 +526,7 @@ async function getUpdatedValues(
     ...values,
     mappings: values.mappings.map((mapping) => ({
       ...mapping,
-      transformer: mapping.transformer as {
-        source: string;
-        config: { config: { case: string; value: {} } };
-      },
+      transformer: mapping.transformer,
     })),
   };