[Connector Builder]Add ability to convert from YAML manifest to UI

airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx

@@ -1,4 +1,5 @@
 import { useMonaco } from "@monaco-editor/react";
+import { useFormikContext } from "formik";
 import { load, YAMLException } from "js-yaml";
 import debounce from "lodash/debounce";
 import isMatch from "lodash/isMatch";
@@ -8,20 +9,21 @@ import { useEffect, useMemo, useRef, useState } from "react";
 import { CodeEditor } from "components/ui/CodeEditor";
 
 import { ConnectorManifest } from "core/request/ConnectorManifest";
-import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
+// import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
 import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService";
 
 import { UiYamlToggleButton } from "../Builder/UiYamlToggleButton";
 import { DownloadYamlButton } from "../DownloadYamlButton";
-import { convertToManifest } from "../types";
+import { convertToBuilderFormValues, convertToManifest } from "../types";
 import styles from "./YamlEditor.module.scss";
 
 interface YamlEditorProps {
   toggleYamlEditor: () => void;
 }
 
 export const YamlEditor: React.FC<YamlEditorProps> = ({ toggleYamlEditor }) => {
-  const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
+  const { setValues } = useFormikContext();
+  // const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
   const yamlEditorRef = useRef<editor.IStandaloneCodeEditor>();
   const {
     yamlManifest,
@@ -82,16 +84,23 @@ export const YamlEditor: React.FC<YamlEditorProps> = ({ toggleYamlEditor }) => {
 
   const handleToggleYamlEditor = () => {
     if (yamlIsDirty) {
-      openConfirmationModal({
-        text: "connectorBuilder.toggleModal.text",
-        title: "connectorBuilder.toggleModal.title",
-        submitButtonText: "connectorBuilder.toggleModal.submitButton",
-        onSubmit: () => {
-          setYamlIsValid(true);
-          toggleYamlEditor();
-          closeConfirmationModal();
-        },
-      });
+      // openConfirmationModal({
+      //   text: "connectorBuilder.toggleModal.text",
+      //   title: "connectorBuilder.toggleModal.title",
+      //   submitButtonText: "connectorBuilder.toggleModal.submitButton",
+      //   onSubmit: () => {
+      //     setYamlIsValid(true);
+      //     toggleYamlEditor();
+      //     closeConfirmationModal();
+      //   },
+      // });
+      // setBuilderFormValues(convertToBuilderFormValues(jsonManifest), false);
+      try {
+        setValues(convertToBuilderFormValues(jsonManifest, builderFormValues));
+        toggleYamlEditor();
+      } catch (e) {
+        alert(e.message);
+      }
     } else {
       setYamlIsValid(true);
       toggleYamlEditor();

airbyte-webapp/src/components/connectorBuilder/types.ts

@@ -1,4 +1,6 @@
 import { JSONSchema7 } from "json-schema";
+import isEqual from "lodash/isEqual";
+import { v4 as uuid } from "uuid";
 import * as yup from "yup";
 
 import { AirbyteJSONSchema } from "core/jsonSchema/types";
@@ -20,6 +22,9 @@ import {
   SubstreamSlicer,
   SubstreamSlicerType,
   CartesianProductStreamSlicer,
+  DatetimeStreamSlicer,
+  ListStreamSlicer,
+  InlineSchemaLoader,
 } from "core/request/ConnectorManifest";
 
 export interface BuilderFormInput {
@@ -444,9 +449,7 @@ export const builderFormValidationSchema = yup.object().shape({
   ),
 });
 
-function builderFormAuthenticatorToAuthenticator(
-  globalSettings: BuilderFormValues["global"]
-): HttpRequesterAuthenticator {
+function builderToManifestAuthenticator(globalSettings: BuilderFormValues["global"]): HttpRequesterAuthenticator {
   if (globalSettings.authenticator.type === "OAuthAuthenticator") {
     return {
       ...globalSettings.authenticator,
@@ -462,7 +465,23 @@ function builderFormAuthenticatorToAuthenticator(
   return globalSettings.authenticator as HttpRequesterAuthenticator;
 }
 
-function builderFormStreamSlicerToStreamSlicer(
+function manifestToBuilderAuthenticator(
+  manifestAuthenticator: HttpRequesterAuthenticator,
+  streamName?: string
+): BuilderFormAuthenticator {
+  if (manifestAuthenticator.type === "OAuthAuthenticator") {
+    return {
+      ...manifestAuthenticator,
+      refresh_request_body: Object.entries(manifestAuthenticator.refresh_request_body ?? {}),
+    };
+  }
+  if (manifestAuthenticator.type === "CustomAuthenticator") {
+    throw new ManifestCompatibilityError(streamName, "uses a CustomAuthenticator");
+  }
+  return manifestAuthenticator;
+}
+
+function builderToManifestStreamSlicer(
   values: BuilderFormValues,
   slicer: BuilderStream["streamSlicer"],
   visitedStreams: string[]
@@ -477,7 +496,7 @@ function builderFormStreamSlicerToStreamSlicer(
     return {
       type: "CartesianProductStreamSlicer",
       stream_slicers: slicer.stream_slicers.map((subSlicer) => {
-        return builderFormStreamSlicerToStreamSlicer(values, subSlicer, visitedStreams);
+        return builderToManifestStreamSlicer(values, subSlicer, visitedStreams);
       }),
     } as unknown as CartesianProductStreamSlicer;
   }
@@ -509,6 +528,47 @@ function builderFormStreamSlicerToStreamSlicer(
   };
 }
 
+function manifestToBuilderStreamSlicer(
+  manifestStreamSlicer: SimpleRetrieverStreamSlicer,
+  streamName?: string
+): NonNullable<BuilderStream["streamSlicer"]> {
+  if (manifestStreamSlicer.type === undefined) {
+    throw new ManifestCompatibilityError(streamName, "stream_slicer has no type");
+  }
+
+  if (manifestStreamSlicer.type === "CustomStreamSlicer") {
+    throw new ManifestCompatibilityError(streamName, "stream_slicer contains a CustomStreamSlicer");
+  }
+
+  if (manifestStreamSlicer.type === "SingleSlice") {
+    throw new ManifestCompatibilityError(streamName, "stream_slicer contains a SingleSlice");
+  }
+
+  if (manifestStreamSlicer.type === "DatetimeStreamSlicer" || manifestStreamSlicer.type === "ListStreamSlicer") {
+    return manifestStreamSlicer as DatetimeStreamSlicer | ListStreamSlicer;
+  }
+
+  if (manifestStreamSlicer.type === "CartesianProductStreamSlicer") {
+    return {
+      type: "CartesianProductStreamSlicer",
+      stream_slicers: manifestStreamSlicer.stream_slicers.map((subSlicer) => {
+        return manifestToBuilderStreamSlicer(subSlicer, streamName) as
+          | Exclude<SimpleRetrieverStreamSlicer, SubstreamSlicer | CartesianProductStreamSlicer>
+          | BuilderSubstreamSlicer;
+      }),
+    };
+  }
+
+  // TODO: add support for substream slicer..
+  // This will be fairly complex as it would need to compare the manifestStreamSlicer's parent stream configs' streams to every other stream in the manifest to find one with an exact match,
+  // and once it does it will need to map that back to the id that was assigned to that stream, which may not have been assigned yet since this conversion is happening stream by stream...
+  if (manifestStreamSlicer.type === "SubstreamSlicer") {
+    throw new ManifestCompatibilityError(streamName, "stream_slicer contains a SubstreamSlicer");
+  }
+
+  throw new ManifestCompatibilityError(streamName, "stream_slicer type is unsupported");
+}
+
 function parseSchemaString(schema?: string) {
   if (!schema) {
     return undefined;
@@ -546,7 +606,7 @@ function builderStreamToDeclarativeSteam(
           request_headers: Object.fromEntries(stream.requestOptions.requestHeaders),
           request_body_json: Object.fromEntries(stream.requestOptions.requestBody),
         } as InterpolatedRequestOptionsProvider,
-        authenticator: builderFormAuthenticatorToAuthenticator(values.global),
+        authenticator: builderToManifestAuthenticator(values.global),
       },
       record_selector: {
         type: "RecordSelector",
@@ -570,7 +630,7 @@ function builderStreamToDeclarativeSteam(
             url_base: values.global?.urlBase,
           }
         : { type: "NoPagination" },
-      stream_slicer: builderFormStreamSlicerToStreamSlicer(values, stream.streamSlicer, [...visitedStreams, stream.id]),
+      stream_slicer: builderToManifestStreamSlicer(values, stream.streamSlicer, [...visitedStreams, stream.id]),
     },
   };
 }
@@ -607,3 +667,149 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest
     spec,
   };
 };
+
+export const convertToBuilderFormValues = (
+  manifest: ConnectorManifest,
+  currentBuilderFormValues: BuilderFormValues
+) => {
+  const builderFormValues = DEFAULT_BUILDER_FORM_VALUES;
+  builderFormValues.global.connectorName = currentBuilderFormValues.global.connectorName;
+
+  console.log("test");
+
+  const streams = manifest.streams;
+  if (streams && streams.length > 0) {
+    if (streams[0].retriever.type !== "SimpleRetriever") {
+      throw new ManifestCompatibilityError(streams[0].name, "doesn't use a SimpleRetriever");
+    }
+    builderFormValues.global.urlBase = streams[0].retriever.requester.url_base;
+    if (streams[0].retriever.requester.authenticator) {
+      builderFormValues.global.authenticator = manifestToBuilderAuthenticator(
+        streams[0].retriever.requester.authenticator,
+        streams[0].name
+      );
+    }
+
+    builderFormValues.streams = streams.map((stream) => {
+      if (stream.retriever.type !== "SimpleRetriever") {
+        throw new ManifestCompatibilityError(stream.name, "doesn't use a SimpleRetriever");
+      }
+
+      const retriever = stream.retriever;
+      const requester = retriever.requester;
+
+      if (!isEqual(retriever.requester.authenticator, builderFormValues.global.authenticator)) {
+        throw new ManifestCompatibilityError(stream.name, "authenticator does not match the first stream's");
+      }
+
+      if (retriever.requester.url_base !== builderFormValues.global.urlBase) {
+        throw new ManifestCompatibilityError(stream.name, "url_base does not match the first stream's");
+      }
+
+      const builderStream = {
+        ...DEFAULT_BUILDER_STREAM_VALUES,
+        id: uuid(),
+        name: stream.name ?? "",
+        urlPath: requester.path,
+        fieldPointer: retriever.record_selector.extractor.field_pointer,
+        requestParameters: Object.entries(requester.request_options_provider?.request_parameters ?? {}),
+        requestHeaders: Object.entries(requester.request_options_provider?.request_headers ?? {}),
+        // try getting this from request_body_data first, and if not set then pull from request_body_json
+        requestBody: Object.entries(
+          requester.request_options_provider?.request_body_data ??
+            requester.request_options_provider?.request_body_json ??
+            {}
+        ),
+      };
+
+      if (![undefined, "GET", "POST"].includes(requester.http_method)) {
+        throw new ManifestCompatibilityError(stream.name, "http_method is not GET or POST");
+      } else {
+        builderStream.httpMethod = (requester.http_method as "GET" | "POST" | undefined) ?? "GET";
+      }
+
+      console.log("stream.primary_key", stream.primary_key);
+      if (retriever.primary_key === undefined) {
+        builderStream.primaryKey = [];
+      } else if (Array.isArray(retriever.primary_key)) {
+        if (retriever.primary_key.length > 0 && Array.isArray(retriever.primary_key[0])) {
+          throw new ManifestCompatibilityError(stream.name, "primary_key contains nested arrays");
+        } else {
+          builderStream.primaryKey = retriever.primary_key as string[];
+        }
+      } else {
+        builderStream.primaryKey = [retriever.primary_key];
+      }
+
+      if (retriever.paginator && retriever.paginator.type === "DefaultPaginator") {
+        if (retriever.paginator.page_token_option === undefined) {
+          throw new ManifestCompatibilityError(stream.name, "paginator does not define a page_token_option");
+        }
+
+        if (retriever.paginator.url_base !== builderFormValues.global.urlBase) {
+          throw new ManifestCompatibilityError(
+            stream.name,
+            "paginator.url_base does not match the first stream's url_base"
+          );
+        }
+
+        builderStream.paginator = {
+          strategy: retriever.paginator.pagination_strategy,
+          pageTokenOption: retriever.paginator.page_token_option,
+          pageSizeOption: retriever.paginator.page_size_option,
+        };
+      }
+
+      if (retriever.stream_slicer) {
+        builderStream.streamSlicer = manifestToBuilderStreamSlicer(retriever.stream_slicer);
+      }
+
+      if (stream.schema_loader) {
+        if (stream.schema_loader.type === "DefaultSchemaLoader") {
+          throw new ManifestCompatibilityError(stream.name, "schema_loader is DefaultSchemaLoader");
+        }
+
+        if (stream.schema_loader.type === "JsonFileSchemaLoader") {
+          throw new ManifestCompatibilityError(stream.name, "schema_loader is JsonFileSchemaLoader");
+        }
+
+        const schemaLoader = stream.schema_loader as InlineSchemaLoader;
+        if (schemaLoader.schema) {
+          builderStream.schema = JSON.stringify(schemaLoader.schema);
+        }
+      }
+
+      return builderStream;
+    });
+  }
+
+  const spec = manifest.spec;
+  if (spec) {
+    const required = spec.connection_specification.required as string[];
+    builderFormValues.inputs = Object.entries(
+      spec.connection_specification.properties as Record<string, AirbyteJSONSchema>
+    ).map(([key, definition]) => {
+      return {
+        key,
+        definition,
+        required: required.includes(key),
+      };
+    });
+    // Mot exactly sure how to deal with inferred inputs, because the manifest may contain inputs that are used in the authenticator but are not named the same as inferred inputs
+    // Maybe we can verify if the authenticator values point to {{config[]}} values, and if they do then we just rename them, if not then we throw an error?
+  }
+
+  return builderFormValues;
+};
+
+export class ManifestCompatibilityError extends Error {
+  __type = "connectorBuilder.manifestCompatibility";
+
+  constructor(public streamName: string | undefined, public message: string) {
+    super(`${streamName ? `Stream ${streamName}: ` : ""} ${message}`);
+  }
+}
+
+export function isManifestCompatibilityError(error: { __type?: string }): error is ManifestCompatibilityError {
+  return error.__type === "connectorBuilder.manifestCompatibility";
+}