diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts
new file mode 100644
index 0000000000000..5917ec348ea46
--- /dev/null
+++ b/airbyte-webapp/src/components/connectorBuilder/types.ts
@@ -0,0 +1,50 @@
+import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorManifest";
+
+export interface BuilderFormValues {
+ global: {
+ connectorName: string;
+ urlBase: string;
+ };
+ streams: BuilderStream[];
+}
+
+export interface BuilderStream {
+ name: string;
+ urlPath: string;
+ fieldPointer: string[];
+ httpMethod: "GET" | "POST";
+}
+
+export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => {
+ const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => {
+ return {
+ name: stream.name,
+ retriever: {
+ name: stream.name,
+ requester: {
+ name: stream.name,
+ url_base: values.global?.urlBase,
+ path: stream.urlPath,
+ // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema
+ config: {},
+ },
+ record_selector: {
+ extractor: {
+ field_pointer: stream.fieldPointer,
+ config: {},
+ },
+ },
+ config: {},
+ },
+ config: {},
+ };
+ });
+
+ return {
+ version: "0.1.0",
+ check: {
+ stream_names: [],
+ },
+ streams: manifestStreams,
+ };
+};
diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.module.scss b/airbyte-webapp/src/components/ui/ListBox/ListBox.module.scss
index f033d34d7cb07..d4ea0a51b4c6a 100644
--- a/airbyte-webapp/src/components/ui/ListBox/ListBox.module.scss
+++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.module.scss
@@ -33,6 +33,7 @@
padding: variables.$spacing-lg variables.$spacing-md;
border-radius: variables.$border-radius-lg;
cursor: pointer;
+ overflow: auto;
}
.active {
diff --git a/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.tsx b/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.tsx
index 9e9027a7c90d6..0682573f4b280 100644
--- a/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.tsx
+++ b/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.tsx
@@ -75,7 +75,7 @@ export const ResizablePanels: React.FC
= ({
return (
= ({
firstPanel.onStopResize?.(args.component.props.flex);
}}
>
-
- {firstPanel.children}
-
+ {firstPanel.children}
{/* NOTE: ReflexElement will not load its contents if wrapped in an empty jsx tag along with ReflexSplitter. They must be evaluated/rendered separately. */}
{!hideSecondPanel && (
@@ -107,7 +105,7 @@ export const ResizablePanels: React.FC = ({
)}
{!hideSecondPanel && (
= ({
secondPanel.onStopResize?.(args.component.props.flex);
}}
>
-
- {secondPanel.children}
-
+ {secondPanel.children}
)}
diff --git a/airbyte-webapp/src/core/form/FormikPatch.ts b/airbyte-webapp/src/core/form/FormikPatch.ts
new file mode 100644
index 0000000000000..37eee6eb0c956
--- /dev/null
+++ b/airbyte-webapp/src/core/form/FormikPatch.ts
@@ -0,0 +1,29 @@
+import { flatten } from "flat";
+import { useFormikContext } from "formik";
+import { useEffect } from "react";
+
+export const FormikPatch: React.FC = () => {
+ const { setFieldTouched, isSubmitting, isValidating, errors } = useFormikContext();
+
+ /* Fixes issue https://github.com/airbytehq/airbyte/issues/1978
+ Problem described here https://github.com/formium/formik/issues/445
+ The problem is next:
+
+ When we touch the field, it would be set as touched field correctly.
+ If validation fails on submit - Formik detects touched object mapping based
+ either on initialValues passed to Formik or on current value set.
+ So in case of creation, if we touch an input, don't change value and
+ press submit - our touched map will be cleared.
+
+ This hack just touches all fields on submit.
+ */
+ useEffect(() => {
+ if (isSubmitting && !isValidating) {
+ for (const path of Object.keys(flatten(errors))) {
+ setFieldTouched(path, true, false);
+ }
+ }
+ }, [errors, isSubmitting, isValidating, setFieldTouched]);
+
+ return null;
+};
diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json
index 2a4cf7201265f..6278ecd7b740d 100644
--- a/airbyte-webapp/src/locales/en.json
+++ b/airbyte-webapp/src/locales/en.json
@@ -58,6 +58,7 @@
"form.delete": "Delete",
"form.change": "Change",
"form.add": "Add",
+ "form.create": "Create",
"form.saveChanges": "Save changes",
"form.openDatepicker": "Open datepicker",
"form.datepickerTimeCaption": "Time (UTC)",
@@ -638,6 +639,25 @@
"connectorBuilder.testConnector": "TEST YOUR CONNECTOR",
"connectorBuilder.couldNotDetectStreams": "Could not detect streams in the YAML editor:",
"connectorBuilder.ensureProperYaml": "In order to test a stream, ensure that the YAML is structured as described in the docs.",
+ "connectorBuilder.addStreamModal.title": "New stream",
+ "connectorBuilder.addStreamModal.streamNameLabel": "Stream name",
+ "connectorBuilder.addStreamModal.streamNameTooltip": "Name of the new stream",
+ "connectorBuilder.addStreamModal.urlPathLabel": "URL path",
+ "connectorBuilder.addStreamModal.urlPathTooltip": "URL path of the endpoint for this stream",
+ "connectorBuilder.resetModal.text": "This will erase all streams and values that are currently set. Are you sure you want to continue?",
+ "connectorBuilder.resetModal.title": "Reset Connector Builder",
+ "connectorBuilder.resetModal.submitButton": "Reset",
+ "connectorBuilder.streamsHeading": "STREAMS ({number})",
+ "connectorBuilder.globalConfiguration": "Global Configuration",
+ "connectorBuilder.noStreamsMessage": "Add a stream to test it here",
+ "connectorBuilder.toggleModal.text": "Toggling back to the UI will erase any changes you have made in the YAML editor.\n\nIn order to export your current yaml, click the Download YAML button.",
+ "connectorBuilder.toggleModal.title": "Warning",
+ "connectorBuilder.toggleModal.submitButton": "Confirm",
+ "connectorBuilder.connectorImgAlt": "Connector Image",
+ "connectorBuilder.uiYamlToggle.ui": "UI",
+ "connectorBuilder.uiYamlToggle.yaml": "YAML",
+ "connectorBuilder.resetAll": "Reset all",
+ "connectorBuilder.emptyName": "(empty)",
"cloudApi.loginCallbackUrlError": "There was an error connecting to the developer portal. Please try again."
}
diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.module.scss b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.module.scss
index 79e8dc13b02b0..231e4a5f1a0ca 100644
--- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.module.scss
+++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.module.scss
@@ -1,16 +1,23 @@
@use "scss/colors";
@use "scss/variables";
+@use "scss/mixins";
.leftPanel {
overflow: hidden;
}
.rightPanel {
+ @include mixins.left-shadow;
+
border-top-left-radius: variables.$border-radius-lg;
border-bottom-left-radius: variables.$border-radius-lg;
background-color: colors.$white;
}
-.container {
+.gradientBg {
background-image: linear-gradient(colors.$grey-50, colors.$grey-100);
}
+
+.solidBg {
+ background-color: colors.$grey-50;
+}
diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
index 54e5de58cd68d..cb648dcd5cae2 100644
--- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
+++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
@@ -1,36 +1,58 @@
+import classnames from "classnames";
+import { Formik } from "formik";
import { useIntl } from "react-intl";
+import { useToggle } from "react-use";
+import { Builder } from "components/connectorBuilder/Builder/Builder";
import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel";
import { YamlEditor } from "components/connectorBuilder/YamlEditor";
import { ResizablePanels } from "components/ui/ResizablePanels";
-import { ConnectorBuilderStateProvider } from "services/connectorBuilder/ConnectorBuilderStateService";
+import {
+ ConnectorBuilderStateProvider,
+ useConnectorBuilderState,
+} from "services/connectorBuilder/ConnectorBuilderStateService";
import styles from "./ConnectorBuilderPage.module.scss";
const ConnectorBuilderPageInner: React.FC = () => {
const { formatMessage } = useIntl();
+ const [showYamlEditor, toggleYamlEditor] = useToggle(false);
+
+ const { builderFormValues } = useConnectorBuilderState();
return (
- ,
- className: styles.leftPanel,
- minWidth: 100,
- }}
- secondPanel={{
- children: ,
- className: styles.rightPanel,
- flex: 0.33,
- minWidth: 60,
- overlay: {
- displayThreshold: 325,
- header: formatMessage({ id: "connectorBuilder.testConnector" }),
- rotation: "counter-clockwise",
- },
- }}
- />
+ undefined}>
+ {({ values }) => (
+
+ {showYamlEditor ? (
+
+ ) : (
+
+ )}
+ >
+ ),
+ className: styles.leftPanel,
+ minWidth: 100,
+ }}
+ secondPanel={{
+ children: ,
+ className: styles.rightPanel,
+ flex: 0.33,
+ minWidth: 60,
+ overlay: {
+ displayThreshold: 325,
+ header: formatMessage({ id: "connectorBuilder.testConnector" }),
+ rotation: "counter-clockwise",
+ },
+ }}
+ />
+ )}
+
);
};
diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx
index 6d7edd57d25c9..2d758484a7717 100644
--- a/airbyte-webapp/src/pages/routes.tsx
+++ b/airbyte-webapp/src/pages/routes.tsx
@@ -55,7 +55,6 @@ const MainViewRoutes: React.FC = () => {
} />
} />
} />
- } />
} />
@@ -109,6 +108,7 @@ export const Routing: React.FC = () => {
);
return (
+ } />
{OldRoutes}
} />
} />
diff --git a/airbyte-webapp/src/scss/_mixins.scss b/airbyte-webapp/src/scss/_mixins.scss
index b6c4748378cd7..37c71fc5755d5 100644
--- a/airbyte-webapp/src/scss/_mixins.scss
+++ b/airbyte-webapp/src/scss/_mixins.scss
@@ -3,3 +3,11 @@
@mixin shadow {
box-shadow: 0 2px 4px rgba(colors.$dark-blue-900, 12%);
}
+
+@mixin left-shadow {
+ box-shadow: -2px 0 10px rgba(colors.$dark-blue-900, 12%);
+}
+
+@mixin right-shadow {
+ box-shadow: 2px 0 10px rgba(colors.$dark-blue-900, 12%);
+}
diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
index e5e9bcbeb659a..3e079cb14a077 100644
--- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
+++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
@@ -1,28 +1,51 @@
+import { dump } from "js-yaml";
import React, { useContext, useEffect, useMemo, useState } from "react";
import { useIntl } from "react-intl";
+import { useLocalStorage } from "react-use";
-import {
- StreamReadRequestBodyConfig,
- StreamsListReadStreamsItem,
- StreamsListRequestBodyManifest,
-} from "core/request/ConnectorBuilderClient";
-import { useAppMonitoringService } from "hooks/services/AppMonitoringService";
+import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types";
+
+import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient";
+import { ConnectorManifest } from "core/request/ConnectorManifest";
import { useListStreams } from "./ConnectorBuilderApiService";
+export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = {
+ global: {
+ connectorName: "",
+ urlBase: "",
+ },
+ streams: [],
+};
+
+const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = {
+ version: "0.1.0",
+ check: {
+ stream_names: [],
+ },
+ streams: [],
+};
+
+export type BuilderView = "global" | number;
+
interface Context {
- jsonManifest: StreamsListRequestBodyManifest;
+ builderFormValues: BuilderFormValues;
+ jsonManifest: ConnectorManifest;
+ yamlManifest: string;
yamlEditorIsMounted: boolean;
yamlIsValid: boolean;
streams: StreamsListReadStreamsItem[];
streamListErrorMessage: string | undefined;
- selectedStream?: StreamsListReadStreamsItem;
+ testStreamIndex: number;
+ selectedView: BuilderView;
configString: string;
configJson: StreamReadRequestBodyConfig;
- setJsonManifest: (jsonValue: StreamsListRequestBodyManifest) => void;
+ setBuilderFormValues: (values: BuilderFormValues) => void;
+ setJsonManifest: (jsonValue: ConnectorManifest) => void;
setYamlEditorIsMounted: (value: boolean) => void;
setYamlIsValid: (value: boolean) => void;
- setSelectedStream: (streamName: string) => void;
+ setTestStreamIndex: (streamIndex: number) => void;
+ setSelectedView: (view: BuilderView) => void;
setConfigString: (configString: string) => void;
}
@@ -31,10 +54,30 @@ export const ConnectorBuilderStateContext = React.createContext(
export const ConnectorBuilderStateProvider: React.FC> = ({ children }) => {
const { formatMessage } = useIntl();
- // json manifest
- const [jsonManifest, setJsonManifest] = useState({});
+ // manifest values
+ const [builderFormValues, setBuilderFormValues] = useLocalStorage(
+ "connectorBuilderFormValues",
+ DEFAULT_BUILDER_FORM_VALUES
+ );
+ const formValues = builderFormValues ?? DEFAULT_BUILDER_FORM_VALUES;
+
+ const [jsonManifest, setJsonManifest] = useLocalStorage(
+ "connectorBuilderJsonManifest",
+ DEFAULT_JSON_MANIFEST_VALUES
+ );
+ const manifest = jsonManifest ?? DEFAULT_JSON_MANIFEST_VALUES;
+
+ useEffect(() => {
+ setJsonManifest(convertToManifest(formValues));
+ }, [formValues, setJsonManifest]);
+
const [yamlIsValid, setYamlIsValid] = useState(true);
- const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(false);
+ const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(true);
+
+ const [yamlManifest, setYamlManifest] = useState("");
+ useEffect(() => {
+ setYamlManifest(dump(jsonManifest));
+ }, [jsonManifest]);
// config
const [configString, setConfigString] = useState("{\n \n}");
@@ -54,7 +97,7 @@ export const ConnectorBuilderStateProvider: React.FC {
return streamListRead?.streams ?? [];
}, [streamListRead]);
- const firstStreamName = streams.length > 0 ? streams[0].name : undefined;
- const [selectedStreamName, setSelectedStream] = useState(firstStreamName);
+ const [testStreamIndex, setTestStreamIndex] = useState(0);
useEffect(() => {
- setSelectedStream((prevSelected) =>
- prevSelected !== undefined && streams.map((stream) => stream.name).includes(prevSelected)
- ? prevSelected
- : firstStreamName
+ setTestStreamIndex((prevIndex) =>
+ prevIndex >= streams.length && streams.length > 0 ? streams.length - 1 : prevIndex
);
- }, [streams, firstStreamName]);
+ }, [streams]);
- const selectedStream = streams.find((stream) => stream.name === selectedStreamName);
+ const [selectedView, setSelectedView] = useState("global");
const ctx = {
- jsonManifest,
+ builderFormValues: formValues,
+ jsonManifest: manifest,
+ yamlManifest,
yamlEditorIsMounted,
yamlIsValid,
streams,
streamListErrorMessage,
- selectedStream,
+ testStreamIndex,
+ selectedView,
configString,
configJson,
+ setBuilderFormValues,
setJsonManifest,
setYamlIsValid,
setYamlEditorIsMounted,
- setSelectedStream,
+ setTestStreamIndex,
+ setSelectedView,
setConfigString,
};
@@ -106,17 +151,9 @@ export const useConnectorBuilderState = (): Context => {
};
export const useSelectedPageAndSlice = () => {
- const { trackError } = useAppMonitoringService();
- const { selectedStream } = useConnectorBuilderState();
-
- // this case should never be reached, as this hook should only be called in components that are only rendered when a stream is selected
- if (selectedStream === undefined) {
- const err = new Error("useSelectedPageAndSlice called when no stream is selected");
- trackError(err, { id: "useSelectedPageAndSlice.noSelectedStream" });
- throw err;
- }
+ const { streams, testStreamIndex } = useConnectorBuilderState();
- const selectedStreamName = selectedStream.name;
+ const selectedStreamName = streams[testStreamIndex].name;
const [streamToSelectedSlice, setStreamToSelectedSlice] = useState({ [selectedStreamName]: 0 });
const setSelectedSlice = (sliceIndex: number) => {
diff --git a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml
new file mode 100644
index 0000000000000..89fe28623293a
--- /dev/null
+++ b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml
@@ -0,0 +1,9 @@
+openapi: 3.0.0
+info:
+ title: Connector Manifest schema
+ version: 1.0.0
+paths: {}
+components:
+ schemas:
+ ConnectorManifest:
+ $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json"
diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss
index 9f22de946a315..626cae0e8051a 100644
--- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss
+++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss
@@ -1,7 +1,15 @@
@use "scss/variables";
+@use "scss/mixins";
.leftPanel {
> *:last-child {
padding-bottom: variables.$spacing-page-bottom;
}
}
+
+.rightPanel {
+ @include mixins.left-shadow;
+
+ border-top-left-radius: variables.$border-radius-lg;
+ border-bottom-left-radius: variables.$border-radius-lg;
+}
diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx
index 41b3958ed406e..4420be887bc69 100644
--- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx
+++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx
@@ -34,6 +34,7 @@ export const ConnectorDocumentationLayout: React.FC {
- usePatchFormik();
- return null;
-};
+import { useBuildForm, useBuildUiWidgetsContext, useConstructValidationSchema } from "./useBuildForm";
/**
* This function sets all initial const values in the form to current values
diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx
index 362e9d8ae8005..f36a2e9d9f5a6 100644
--- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx
+++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx
@@ -1,8 +1,6 @@
-import flatten from "flat";
-import { useFormikContext } from "formik";
import { JSONSchema7, JSONSchema7Definition } from "json-schema";
import merge from "lodash/merge";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useMemo, useState } from "react";
import { useIntl } from "react-intl";
import { AnySchema } from "yup";
@@ -103,27 +101,3 @@ export const useBuildUiWidgetsContext = (
// As validation schema depends on what path of oneOf is currently selected in jsonschema
export const useConstructValidationSchema = (jsonSchema: JSONSchema7, uiWidgetsInfo: WidgetConfigMap): AnySchema =>
useMemo(() => buildYupFormForJsonSchema(jsonSchema, uiWidgetsInfo), [uiWidgetsInfo, jsonSchema]);
-
-export const usePatchFormik = (): void => {
- const { setFieldTouched, isSubmitting, isValidating, errors } = useFormikContext();
-
- /* Fixes issue https://github.com/airbytehq/airbyte/issues/1978
- Problem described here https://github.com/formium/formik/issues/445
- The problem is next:
-
- When we touch the field, it would be set as touched field correctly.
- If validation fails on submit - Formik detects touched object mapping based
- either on initialValues passed to Formik or on current value set.
- So in case of creation, if we touch an input, don't change value and
- press submit - our touched map will be cleared.
-
- This hack just touches all fields on submit.
- */
- useEffect(() => {
- if (isSubmitting && !isValidating) {
- for (const path of Object.keys(flatten(errors))) {
- setFieldTouched(path, true, false);
- }
- }
- }, [errors, isSubmitting, isValidating, setFieldTouched]);
-};