diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index aa39b1fc6f0c9..a100a69a98323 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -21,6 +21,7 @@ "@sentry/tracing": "^6.19.6", "@tanstack/react-table": "^8.7.0", "@types/segment-analytics": "^0.0.34", + "@types/uuid": "^9.0.0", "classnames": "^2.3.1", "dayjs": "^1.11.3", "firebase": "^9.8.2", @@ -61,6 +62,7 @@ "styled-components": "^5.3.5", "typesafe-actions": "^5.1.0", "unist-util-visit": "^4.1.0", + "uuid": "^9.0.0", "yup": "^0.32.11" }, "devDependencies": { @@ -14878,6 +14880,11 @@ "integrity": "sha512-FDJNkyhmKLw7uEvTxx5tSXfPeQpO0iy73Ry+PmYZJvQy0QIWX8a7kJ4kLWRf+EbTPJEPDSgPXHaM7pzr5lmvCg==", "dev": true }, + "node_modules/@types/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q==" + }, "node_modules/@types/webpack": { "version": "4.41.26", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.26.tgz", @@ -38581,6 +38588,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/react-scripts/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/react-scripts/node_modules/webpack-dev-middleware": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.0.tgz", @@ -45241,10 +45257,9 @@ } }, "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", "bin": { "uuid": "dist/bin/uuid" } @@ -58236,6 +58251,11 @@ "integrity": "sha512-FDJNkyhmKLw7uEvTxx5tSXfPeQpO0iy73Ry+PmYZJvQy0QIWX8a7kJ4kLWRf+EbTPJEPDSgPXHaM7pzr5lmvCg==", "dev": true }, + "@types/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q==" + }, "@types/webpack": { "version": "4.41.26", "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.41.26.tgz", @@ -76065,6 +76085,12 @@ "optional": true, "peer": true }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true + }, "webpack-dev-middleware": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.0.tgz", @@ -81084,10 +81110,9 @@ "dev": true }, "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==" }, "uuid-browser": { "version": "3.1.0", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index eb578d79b2bf1..53fee68f7d79e 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -40,6 +40,7 @@ "@sentry/tracing": "^6.19.6", "@tanstack/react-table": "^8.7.0", "@types/segment-analytics": "^0.0.34", + "@types/uuid": "^9.0.0", "classnames": "^2.3.1", "dayjs": "^1.11.3", "firebase": "^9.8.2", @@ -80,6 +81,7 @@ "styled-components": "^5.3.5", "typesafe-actions": "^5.1.0", "unist-util-visit": "^4.1.0", + "uuid": "^9.0.0", "yup": "^0.32.11" }, "devDependencies": { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 1933579f2fc44..cffa2cad32f85 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -3,6 +3,7 @@ import merge from "lodash/merge"; import { useState } from "react"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; +import { v4 as uuid } from "uuid"; import * as yup from "yup"; import { Button } from "components/ui/Button"; @@ -55,6 +56,7 @@ export const AddStreamButton: React.FC = ({ onAddStream, b ...initialValues, name: values.streamName, urlPath: values.urlPath, + id: uuid(), }), ]); setIsOpen(false); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.module.scss new file mode 100644 index 0000000000000..dc1c5654cdc50 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.module.scss @@ -0,0 +1,13 @@ +@use "scss/variables"; +@use "scss/colors"; + +$removeButtonWidth: 20px; + +.itemWrapper { + display: flex; + gap: variables.$spacing-md; +} + +.itemContainer { + flex-grow: 1; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.tsx new file mode 100644 index 0000000000000..60d3d3067e100 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderList.tsx @@ -0,0 +1,55 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { useField } from "formik"; +import React, { ReactElement, useMemo } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Button } from "components/ui/Button"; + +import styles from "./BuilderList.module.scss"; +import { RemoveButton } from "./RemoveButton"; + +interface BuilderListProps { + children: (props: { buildPath: (path: string) => string }) => ReactElement; + basePath: string; + emptyItem: object; +} + +export const BuilderList: React.FC = ({ children, emptyItem, basePath }) => { + const [list, , helpers] = useField(basePath); + + const buildPathFunctions = useMemo( + () => + new Array(list.value.length).fill(undefined).map((_value, index) => { + return (path: string) => `${basePath}[${index}]${path !== "" ? "." : ""}${path}`; + }), + [basePath, list.value.length] + ); + + return ( + <> + {buildPathFunctions.map((buildPath, currentItemIndex) => ( +
+
{children({ buildPath })}
+ { + const updatedItems = list.value.filter((_, index) => index !== currentItemIndex); + helpers.setValue(updatedItems); + }} + /> +
+ ))} +
+ +
+ + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx index 2fc53cb59e283..e6bec6bbbbcdb 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx @@ -11,7 +11,7 @@ interface Option { default?: object; } -interface OneOfOption { +export interface OneOfOption { label: string; // label shown in the dropdown menu typeValue: string; // value to set on the `type` field for this component - should match the oneOf type definition default?: object; // default values for the path diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.module.scss index 62a4b592138c3..daba1c15331c4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.module.scss @@ -21,19 +21,3 @@ $removeButtonWidth: 20px; .kvLabel { color: colors.$grey-400; } - -.removeButton { - border: none; - background-color: transparent; - color: colors.$grey; - cursor: pointer; - padding: 0; - width: $removeButtonWidth; - height: $removeButtonWidth; - font-size: 18px; - transition: variables.$transition; - - &:hover { - color: colors.$red; - } -} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx index 4875344242349..14b2f5fe0817d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx @@ -1,5 +1,3 @@ -import { faXmark } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useField } from "formik"; import { FormattedMessage } from "react-intl"; @@ -10,6 +8,7 @@ import { Input } from "components/ui/Input"; import { Text } from "components/ui/Text"; import styles from "./KeyValueListField.module.scss"; +import { RemoveButton } from "./RemoveButton"; interface KeyValueInputProps { keyValue: [string, string]; @@ -32,9 +31,7 @@ const KeyValueInput: React.FC = ({ keyValue, onChange, onRem onChange([keyValue[0], e.target.value])} /> - + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.module.scss new file mode 100644 index 0000000000000..21074ab294631 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.module.scss @@ -0,0 +1,20 @@ +@use "scss/variables"; +@use "scss/colors"; + +$removeButtonWidth: 20px; + +.removeButton { + border: none; + background-color: transparent; + color: colors.$grey; + cursor: pointer; + padding: 0; + width: $removeButtonWidth; + height: $removeButtonWidth; + font-size: 18px; + transition: variables.$transition; + + &:hover { + color: colors.$red; + } +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.tsx new file mode 100644 index 0000000000000..50326541ef615 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/RemoveButton.tsx @@ -0,0 +1,12 @@ +import { faXmark } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import styles from "./RemoveButton.module.scss"; + +export const RemoveButton = ({ onClick }: { onClick: () => void }) => { + return ( + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamReferenceField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamReferenceField.tsx new file mode 100644 index 0000000000000..a5d650efe64e9 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamReferenceField.tsx @@ -0,0 +1,58 @@ +import { useField } from "formik"; +import { useMemo } from "react"; +import { FormattedMessage } from "react-intl"; + +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; +import { Text } from "components/ui/Text"; + +import { BuilderStream } from "../types"; +import styles from "./BuilderField.module.scss"; + +interface StreamReferenceFieldProps { + // path to the location in the Connector Manifest schema which should be set by this component + path: string; + label: string; + tooltip?: string; + optional?: boolean; + currentStreamIndex: number; +} + +export const StreamReferenceField: React.FC = ({ + path, + label, + tooltip, + optional, + currentStreamIndex, + ...props +}) => { + const [streams] = useField("streams"); + const [field, meta, helpers] = useField(path); + const hasError = !!meta.error && meta.touched; + + const options = useMemo(() => { + return streams.value + .filter((_value, index) => index !== currentStreamIndex) + .map((stream) => ({ + value: stream.id, + label: stream.name, + })); + }, [currentStreamIndex, streams.value]); + + return ( + + selected && helpers.setValue(selected.value)} + value={field.value} + error={hasError} + /> + {hasError && ( + + + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx index 514330b89b569..490133d6f8811 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx @@ -8,9 +8,11 @@ import { RequestOption, SimpleRetrieverStreamSlicer } from "core/request/Connect import { timeDeltaRegex } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; -import { BuilderOneOf } from "./BuilderOneOf"; +import { BuilderList } from "./BuilderList"; +import { BuilderOneOf, OneOfOption } from "./BuilderOneOf"; import { BuilderOptional } from "./BuilderOptional"; import { InjectRequestOptionFields } from "./InjectRequestOptionFields"; +import { StreamReferenceField } from "./StreamReferenceField"; import { ToggleGroupField } from "./ToggleGroupField"; interface StreamSlicerSectionProps { @@ -35,6 +37,181 @@ export const StreamSlicerSection: React.FC = ({ stream }; const toggledOn = field.value !== undefined; + const getRegularSlicingOptions = (buildPath: (path: string) => string): OneOfOption[] => [ + { + label: "List", + typeValue: "ListStreamSlicer", + default: { + slice_values: [], + cursor_field: "", + }, + children: ( + <> + + + + label="Slice request option" + tooltip="Optionally configures how the slice values will be sent in requests to the source API" + fieldPath={buildPath("request_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + ), + }, + { + label: "Datetime", + typeValue: "DatetimeStreamSlicer", + default: { + datetime_format: "", + start_datetime: "", + end_datetime: "", + step: "", + cursor_field: "", + }, + children: ( + <> + + + + + + + + + label="Start time request option" + tooltip="Optionally configures how the start datetime will be sent in requests to the source API" + fieldPath={buildPath("start_time_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + label="End time request option" + tooltip="Optionally configures how the end datetime will be sent in requests to the source API" + fieldPath={buildPath("end_time_option")} + initialValues={{ + inject_into: "request_parameter", + type: "RequestOption", + field_name: "", + }} + > + + + + + + + ), + }, + { + label: "Substream", + typeValue: "SubstreamSlicer", + default: { + parent_key: "", + stream_slice_field: "", + parentStreamReference: "", + }, + children: ( + <> + + + + + ), + }, + ]; + return ( = ({ stream label="Mode" tooltip="Stream slicer method to use on this stream" options={[ + ...getRegularSlicingOptions((path: string) => streamFieldPath(`streamSlicer.${path}`)), { - label: "List", - typeValue: "ListStreamSlicer", - children: ( - <> - - - - label="Slice request option" - tooltip="Optionally configures how the slice values will be sent in requests to the source API" - fieldPath={streamFieldPath("streamSlicer.request_option")} - initialValues={{ - inject_into: "request_parameter", - type: "RequestOption", - field_name: "", - }} - > - - - - ), - }, - { - label: "Datetime", - typeValue: "DatetimeStreamSlicer", + label: "Cartesian product", + typeValue: "CartesianProductStreamSlicer", + default: { + stream_slicers: [], + }, children: ( - <> - - - - - - - - - label="Start time request option" - tooltip="Optionally configures how the start datetime will be sent in requests to the source API" - fieldPath={streamFieldPath("streamSlicer.start_time_option")} - initialValues={{ - inject_into: "request_parameter", - type: "RequestOption", - field_name: "", - }} - > - - - - label="End time request option" - tooltip="Optionally configures how the end datetime will be sent in requests to the source API" - fieldPath={streamFieldPath("streamSlicer.end_time_option")} - initialValues={{ - inject_into: "request_parameter", - type: "RequestOption", - field_name: "", - }} - > - - - - + {({ buildPath }) => ( + - - + )} + ), }, ]} diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index bc7c00db45649..6f60eac5cfcd9 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -17,6 +17,9 @@ import { DefaultPaginatorPaginationStrategy, SimpleRetrieverStreamSlicer, HttpRequesterAuthenticator, + SubstreamSlicer, + SubstreamSlicerType, + CartesianProductStreamSlicer, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -53,7 +56,23 @@ export interface BuilderPaginator { pageSizeOption?: RequestOption; } +export interface BuilderSubstreamSlicer { + type: SubstreamSlicerType; + parent_key: string; + stream_slice_field: string; + parentStreamReference: string; + request_option?: RequestOption; +} + +export interface BuilderCartesianProductSlicer { + type: "CartesianProductStreamSlicer"; + stream_slicers: Array< + Exclude | BuilderSubstreamSlicer + >; +} + export interface BuilderStream { + id: string; name: string; urlPath: string; fieldPointer: string[]; @@ -65,7 +84,10 @@ export interface BuilderStream { requestBody: Array<[string, string]>; }; paginator?: BuilderPaginator; - streamSlicer?: SimpleRetrieverStreamSlicer; + streamSlicer?: + | Exclude + | BuilderSubstreamSlicer + | BuilderCartesianProductSlicer; schema?: string; } @@ -80,7 +102,7 @@ export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { streams: [], }; -export const DEFAULT_BUILDER_STREAM_VALUES: BuilderStream = { +export const DEFAULT_BUILDER_STREAM_VALUES: Omit = { name: "", urlPath: "", fieldPointer: [], @@ -231,6 +253,80 @@ const nonPathRequestOptionSchema = yup // eslint-disable-next-line no-useless-escape export const timeDeltaRegex = /^(([\.\d]+?)y)?(([\.\d]+?)m)?(([\.\d]+?)w)?(([\.\d]+?)d)?$/; +const regularSlicerShape = { + cursor_field: yup.mixed().when("type", { + is: (val: string) => val !== "SubstreamSlicer" && val !== "CartesianProductStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + slice_values: yup.mixed().when("type", { + is: "ListStreamSlicer", + then: yup.array().of(yup.string()), + otherwise: (schema) => schema.strip(), + }), + request_option: nonPathRequestOptionSchema, + start_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + end_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + step: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().matches(timeDeltaRegex, "form.pattern.error").required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + datetime_format: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + start_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + end_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + stream_state_field_start: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + stream_state_field_end: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + lookback_window: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + parent_key: yup.mixed().when("type", { + is: "SubstreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + parentStreamReference: yup.mixed().when("type", { + is: "SubstreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + stream_slice_field: yup.mixed().when("type", { + is: "SubstreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), +}; + export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ connectorName: yup.string().required("form.empty.error"), @@ -332,56 +428,10 @@ export const builderFormValidationSchema = yup.object().shape({ streamSlicer: yup .object() .shape({ - cursor_field: yup.string().required("form.empty.error"), - slice_values: yup.mixed().when("type", { - is: "ListStreamSlicer", - then: yup.array().of(yup.string()), - otherwise: (schema) => schema.strip(), - }), - request_option: nonPathRequestOptionSchema, - start_datetime: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - end_datetime: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - step: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string().matches(timeDeltaRegex, "form.pattern.error").required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - datetime_format: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - start_time_option: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: nonPathRequestOptionSchema, - otherwise: (schema) => schema.strip(), - }), - end_time_option: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: nonPathRequestOptionSchema, - otherwise: (schema) => schema.strip(), - }), - stream_state_field_start: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string(), - otherwise: (schema) => schema.strip(), - }), - stream_state_field_end: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string(), - otherwise: (schema) => schema.strip(), - }), - lookback_window: yup.mixed().when("type", { - is: "DatetimeStreamSlicer", - then: yup.string(), + ...regularSlicerShape, + stream_slicers: yup.mixed().when("type", { + is: "CartesianProductStreamSlicer", + then: yup.array().of(yup.object().shape(regularSlicerShape)), otherwise: (schema) => schema.strip(), }), }) @@ -409,6 +459,53 @@ function builderFormAuthenticatorToAuthenticator( return globalSettings.authenticator as HttpRequesterAuthenticator; } +function builderFormStreamSlicerToStreamSlicer( + values: BuilderFormValues, + slicer: BuilderStream["streamSlicer"], + visitedStreams: string[] +): SimpleRetrieverStreamSlicer | undefined { + if (!slicer) { + return undefined; + } + if (slicer.type !== "SubstreamSlicer" && slicer.type !== "CartesianProductStreamSlicer") { + return slicer; + } + if (slicer.type === "CartesianProductStreamSlicer") { + return { + type: "CartesianProductStreamSlicer", + stream_slicers: slicer.stream_slicers.map((subSlicer) => { + return builderFormStreamSlicerToStreamSlicer(values, subSlicer, visitedStreams); + }), + } as unknown as CartesianProductStreamSlicer; + } + const parentStream = values.streams.find(({ id }) => id === slicer.parentStreamReference); + if (!parentStream) { + return { + type: "SubstreamSlicer", + parent_stream_configs: [], + }; + } + if (visitedStreams.includes(parentStream.id)) { + // circular dependency + return { + type: "SubstreamSlicer", + parent_stream_configs: [], + }; + } + return { + type: "SubstreamSlicer", + parent_stream_configs: [ + { + type: "ParentStreamConfig", + parent_key: slicer.parent_key, + request_option: slicer.request_option, + stream_slice_field: slicer.stream_slice_field, + stream: builderStreamToDeclarativeSteam(values, parentStream, visitedStreams), + }, + ], + }; +} + function parseSchemaString(schema?: string) { if (!schema) { return undefined; @@ -420,60 +517,65 @@ function parseSchemaString(schema?: string) { } } -export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { - const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { - return { - type: "DeclarativeStream", +function builderStreamToDeclarativeSteam( + values: BuilderFormValues, + stream: BuilderStream, + visitedStreams: string[] +): DeclarativeStream { + return { + type: "DeclarativeStream", + name: stream.name, + primary_key: stream.primaryKey, + schema_loader: parseSchemaString(stream.schema), + retriever: { + type: "SimpleRetriever", name: stream.name, primary_key: stream.primaryKey, - schema_loader: parseSchemaString(stream.schema), - retriever: { - type: "SimpleRetriever", + requester: { + type: "HttpRequester", name: stream.name, - primary_key: stream.primaryKey, - requester: { - type: "HttpRequester", - name: stream.name, - url_base: values.global?.urlBase, - path: stream.urlPath, - request_options_provider: { - // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. - // type: "InterpolatedRequestOptionsProvider", - request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), - request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), - request_body_json: Object.fromEntries(stream.requestOptions.requestBody), - } as InterpolatedRequestOptionsProvider, - authenticator: builderFormAuthenticatorToAuthenticator(values.global), - // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema - config: {}, - }, - record_selector: { - type: "RecordSelector", - extractor: { - type: "DpathExtractor", - field_pointer: stream.fieldPointer, - }, + url_base: values.global?.urlBase, + path: stream.urlPath, + request_options_provider: { + // TODO can't declare type here because the server will error out, but the types dictate it is needed. Fix here once server is fixed. + // type: "InterpolatedRequestOptionsProvider", + request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), + request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), + request_body_json: Object.fromEntries(stream.requestOptions.requestBody), + } as InterpolatedRequestOptionsProvider, + authenticator: builderFormAuthenticatorToAuthenticator(values.global), + }, + record_selector: { + type: "RecordSelector", + extractor: { + type: "DpathExtractor", + field_pointer: stream.fieldPointer, }, - paginator: stream.paginator - ? { - type: "DefaultPaginator", - page_token_option: { - ...stream.paginator.pageTokenOption, - // ensures that empty field_name is not set, as connector builder server cannot accept a field_name if inject_into is set to 'path' - field_name: stream.paginator.pageTokenOption?.field_name - ? stream.paginator.pageTokenOption?.field_name - : undefined, - }, - page_size_option: stream.paginator.pageSizeOption, - pagination_strategy: stream.paginator.strategy, - url_base: values.global?.urlBase, - } - : { type: "NoPagination" }, - stream_slicer: stream.streamSlicer, - config: {}, }, - }; - }); + paginator: stream.paginator + ? { + type: "DefaultPaginator", + page_token_option: { + ...stream.paginator.pageTokenOption, + // ensures that empty field_name is not set, as connector builder server cannot accept a field_name if inject_into is set to 'path' + field_name: stream.paginator.pageTokenOption?.field_name + ? stream.paginator.pageTokenOption?.field_name + : undefined, + }, + page_size_option: stream.paginator.pageSizeOption, + pagination_strategy: stream.paginator.strategy, + url_base: values.global?.urlBase, + } + : { type: "NoPagination" }, + stream_slicer: builderFormStreamSlicerToStreamSlicer(values, stream.streamSlicer, [...visitedStreams, stream.id]), + }, + }; +} + +export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { + const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => + builderStreamToDeclarativeSteam(values, stream, []) + ); const allInputs = [...values.inputs, ...getInferredInputs(values)]; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 16fcf517dda02..38d4c8e33638d 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -718,6 +718,7 @@ "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel", "connectorBuilder.optionalFieldsLabel": "Optional fields", "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times", + "connectorBuilder.addNewSlicer": "Add new slicer", "connectorBuilder.streamConfiguration": "Configuration", "connectorBuilder.streamSchema": "Schema", "connectorBuilder.invalidSchema": "Invalid JSON - please fix schema to have it applied",