Ingestion: Added form in Big Query type to edit the queries.

datahub-web-react/src/app/entity/shared/components/styled/ExpandedActor.tsx

@@ -22,8 +22,6 @@ const ActorTag = styled(Tag)`
 `;
 
 export const ExpandedActor = ({ actor, popOver, closable, onClose }: Props) => {
-    console.log(actor);
-
     const entityRegistry = useEntityRegistry();
 
     let name = '';

datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx

@@ -1,5 +1,5 @@
-import { Button, Checkbox, Form, Input, Tooltip } from 'antd';
 import React from 'react';
+import { Button, Checkbox, Form, Input, Select, Tooltip } from 'antd';
 import styled from 'styled-components/macro';
 import { MinusCircleOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons';
 import { FieldType, RecipeField } from './utils';
@@ -52,6 +52,10 @@ interface ListFieldProps {
     removeMargin?: boolean;
 }
 
+interface SelectFieldProps {
+    field: RecipeField;
+}
+
 function ListField({ field, removeMargin }: ListFieldProps) {
     return (
         <Form.List name={field.name}>
@@ -80,6 +84,30 @@ function ListField({ field, removeMargin }: ListFieldProps) {
     );
 }
 
+function SelectField({ field }: SelectFieldProps) {
+    return (
+        <Form.Item
+            name={field.name}
+            label={
+                <Label>
+                    {field.label}
+                    <Tooltip overlay={field.tooltip}>
+                        <StyledQuestion />
+                    </Tooltip>
+                </Label>
+            }
+        >
+            {field.tableLineages && (
+                <Select>
+                    {field.tableLineages.map((lineage) => (
+                        <Select.Option value={lineage.value}>{lineage.label}</Select.Option>
+                    ))}
+                </Select>
+            )}
+        </Form.Item>
+    );
+}
+
 interface Props {
     field: RecipeField;
     removeMargin?: boolean;
@@ -90,6 +118,8 @@ function FormField(props: Props) {
 
     if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;
 
+    if (field.type === FieldType.SELECT) return <SelectField field={field} />;
+
     const isBoolean = field.type === FieldType.BOOLEAN;
     const input = isBoolean ? <Checkbox /> : <Input />;
     const valuePropName = isBoolean ? 'checked' : 'value';

datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.ts

@@ -1,10 +1,18 @@
 import { set, get } from 'lodash';
 import { SNOWFLAKE } from '../../conf/snowflake/snowflake';
+import { BIGQUERY } from '../../conf/bigquery/bigquery';
+import { REDSHIFT } from '../../conf/redshift/redshift';
 
 export enum FieldType {
     TEXT,
     BOOLEAN,
     LIST,
+    SELECT,
+}
+
+interface LineageType {
+    label: string;
+    value: string;
 }
 
 export interface RecipeField {
@@ -15,6 +23,7 @@ export interface RecipeField {
     fieldPath: string;
     rules: any[] | null;
     section?: string;
+    tableLineages?: LineageType[];
     getValueFromRecipeOverride?: (recipe: any) => any;
     setValueOnRecipeOverride?: (recipe: any, value: any) => any;
 }
@@ -102,6 +111,97 @@ export const SNOWFLAKE_ROLE: RecipeField = {
     rules: null,
 };
 
+export const BIGQUERY_PROJECT_ID: RecipeField = {
+    name: 'project_id',
+    label: 'BigQuery Project ID',
+    tooltip:
+        'Project ID to ingest usage from. If not specified, will infer from environment. Deprecated in favour of projects.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.project_id',
+    rules: null,
+};
+
+export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
+    name: 'credential.project_id',
+    label: 'Credentials Project ID',
+    tooltip: 'Project id to set the credentials.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.project_id',
+    rules: null,
+};
+
+export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
+    name: 'credential.private_key_id',
+    label: 'Private Key Id',
+    tooltip: 'Private key id.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.private_key_id',
+    rules: null,
+};
+
+export const BIGQUERY_PRIVATE_KEY: RecipeField = {
+    name: 'credential.private_key',
+    label: 'Private Key',
+    tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.private_key',
+    rules: null,
+};
+
+export const BIGQUERY_CLIENT_EMAIL: RecipeField = {
+    name: 'credential.client_email',
+    label: 'Client Email',
+    tooltip: 'Client email.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.client_email',
+    rules: null,
+};
+
+export const BIGQUERY_CLIENT_ID: RecipeField = {
+    name: 'credential.client_id',
+    label: 'Client ID',
+    tooltip: 'Client ID.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.client_id',
+    rules: null,
+};
+
+export const REDSHIFT_HOST_PORT: RecipeField = {
+    name: 'host_port',
+    label: 'Host Port',
+    tooltip: 'Host URL.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.host_port',
+    rules: null,
+};
+
+export const REDSHIFT_DATABASE: RecipeField = {
+    name: 'database',
+    label: 'Database',
+    tooltip: 'Database (catalog).',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.database',
+    rules: null,
+};
+
+export const REDSHIFT_USERNAME: RecipeField = {
+    name: 'redshift.username',
+    label: 'Username',
+    tooltip: 'Username',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.username',
+    rules: null,
+};
+
+export const REDSHIFT_PASSWORD: RecipeField = {
+    name: 'redshift.password',
+    label: 'Password',
+    tooltip: 'Password',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.password',
+    rules: null,
+};
+
 const includeLineageFieldPathA = 'source.config.include_table_lineage';
 const includeLineageFieldPathB = 'source.config.include_view_lineage';
 export const INCLUDE_LINEAGE: RecipeField = {
@@ -157,6 +257,29 @@ export const STATEFUL_INGESTION_ENABLED: RecipeField = {
     rules: null,
 };
 
+export const INCLUDE_UPSTREAM_LINEAGE_IN_REPORT: RecipeField = {
+    name: 'include_upstream_lineage_in_report',
+    label: 'Include Upstream Lineage In Report.',
+    tooltip: 'Useful for debugging lineage information. Set to True to see the raw lineage created internally.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.include_upstream_lineage_in_report',
+    rules: null,
+};
+
+export const TABLE_LINEAGE_MODE: RecipeField = {
+    name: 'table_lineage_mode',
+    label: 'Table Lineage Mode',
+    tooltip: 'Option to enable/disable lineage generation. Is enabled by default.',
+    type: FieldType.SELECT,
+    fieldPath: 'source.config.table_lineage_mode',
+    rules: null,
+    tableLineages: [
+        { label: 'stl_scan_based', value: 'stl_scan_based' },
+        { label: 'sql_based', value: 'sql_based' },
+        { label: 'mixed', value: 'mixed' },
+    ],
+};
+
 const databaseAllowFieldPath = 'source.config.database_pattern.allow';
 export const DATABASE_ALLOW: RecipeField = {
     name: 'database_pattern.allow',
@@ -282,6 +405,28 @@ export const RECIPE_FIELDS = {
             VIEW_DENY,
         ],
     },
+    [BIGQUERY]: {
+        fields: [
+            BIGQUERY_PROJECT_ID,
+            BIGQUERY_CREDENTIAL_PROJECT_ID,
+            BIGQUERY_PRIVATE_KEY,
+            BIGQUERY_PRIVATE_KEY_ID,
+            BIGQUERY_CLIENT_EMAIL,
+            BIGQUERY_CLIENT_ID,
+        ],
+        advancedFields: [
+            INCLUDE_LINEAGE,
+            PROFILING_ENABLED,
+            STATEFUL_INGESTION_ENABLED,
+            INCLUDE_UPSTREAM_LINEAGE_IN_REPORT,
+        ],
+        filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY],
+    },
+    [REDSHIFT]: {
+        fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD],
+        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
+        filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY],
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts

@@ -21,8 +21,10 @@ source:
             client_id: # Your BQ client id, e.g. "123456678890"
 `;
 
+export const BIGQUERY = 'bigquery';
+
 const bigqueryConfig: SourceConfig = {
-    type: 'bigquery',
+    type: BIGQUERY,
     placeholderRecipe,
     displayName: 'BigQuery',
     docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/bigquery/',

datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts

@@ -23,8 +23,10 @@ source:
             enabled: false
 `;
 
+export const REDSHIFT = 'redshift';
+
 const redshiftConfig: SourceConfig = {
-    type: 'redshift',
+    type: REDSHIFT,
     placeholderRecipe,
     displayName: 'Redshift',
     docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/redshift/',