diff --git a/docs/docs/experimentation/connect-a-warehouse.md b/docs/docs/experimentation/connect-a-warehouse.md index 2bd01eb3d13e..1b28b31293bc 100644 --- a/docs/docs/experimentation/connect-a-warehouse.md +++ b/docs/docs/experimentation/connect-a-warehouse.md @@ -50,6 +50,65 @@ The current status is shown on the warehouse card in **Environment Settings > Wa - **Connected**: events have been received; you are ready to run experiments. - **Errored**: something went wrong; [contact support](/support/). +## Bring your own ClickHouse + +Instead of the managed Flagsmith warehouse, you can store experiment events in your own **ClickHouse** instance. + +1. Configure your instance by running the following against it, using an administrative user (the statements require + `CREATE USER`, `GRANT`, `CREATE DATABASE` and `CREATE TABLE` privileges). Replace `` and `` + with your own values. This is a one-off setup step; the same SQL is available to copy from the Flagsmith dashboard. + + ```sql + -- Create a dedicated user for Flagsmith + CREATE USER IF NOT EXISTS + IDENTIFIED WITH sha256_password BY ''; + + -- Allow it to write and read experiment events + GRANT SELECT, INSERT ON flagsmith_exp.events TO ; + + -- Allow it to check the events table exists + GRANT SHOW TABLES, SHOW COLUMNS ON flagsmith_exp.* TO ; + + -- Create the database and events table + CREATE DATABASE IF NOT EXISTS flagsmith_exp; + + CREATE TABLE IF NOT EXISTS flagsmith_exp.events + ( + environment_key LowCardinality(String), + event LowCardinality(String), + feature_name LowCardinality(String), + timestamp DateTime64(3), + collected_at DateTime64(3), + identifier String, + value String CODEC(ZSTD(3)), + traits String CODEC(ZSTD(3)), + metadata String CODEC(ZSTD(3)), + sdk_language LowCardinality(String), + sdk_version LowCardinality(String), + + INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4, + + CONSTRAINT environment_key_not_empty CHECK environment_key != '', + CONSTRAINT event_not_empty CHECK event != '', + CONSTRAINT timestamp_sane CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3) + ) + ENGINE = MergeTree + PARTITION BY toYYYYMMDD(timestamp) + ORDER BY (environment_key, event, feature_name, timestamp, identifier); + ``` + +2. Go to **Environment Settings > Warehouse**, select **ClickHouse** and enter your connection details: host, port (9440 + by default, the native protocol port), database (`flagsmith_exp` by default), and the username and password of the + user you created in step 1. +3. Click **Test connection**. Once the test succeeds, save the connection. +4. Click **Send your first event** on the connection card to verify events are arriving in your warehouse; the button + disappears once the first event is received. + +Connections use the ClickHouse native protocol, with TLS enabled by default. + +Experiment results and exposure queries are currently computed from the managed Flagsmith warehouse; serving results +directly from your own ClickHouse instance is not supported yet. + ## Coming soon Bring your own warehouse: connections for **Snowflake**, **BigQuery** and **Databricks**, so experiment results are diff --git a/frontend/web/components/CalloutBar.tsx b/frontend/web/components/CalloutBar.tsx index c2b59716d3e5..b78b6ef541c3 100644 --- a/frontend/web/components/CalloutBar.tsx +++ b/frontend/web/components/CalloutBar.tsx @@ -5,7 +5,7 @@ export type CalloutBarTheme = 'light' | 'dark' type CalloutBarProps = { theme?: CalloutBarTheme - icon: ReactNode + icon?: ReactNode prefix: ReactNode label: ReactNode expanded?: boolean @@ -14,7 +14,7 @@ type CalloutBarProps = { const CalloutBar: FC = ({ expanded, - icon, + icon = <>{'<>'}, label, onClick, prefix, diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/ClickHouseConfigForm.tsx b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/ClickHouseConfigForm.tsx index d6f09ce9d3c7..eec12ea85c92 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/ClickHouseConfigForm.tsx +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/ClickHouseConfigForm.tsx @@ -4,6 +4,7 @@ import Input from 'components/base/forms/Input' import Switch from 'components/Switch' import ErrorMessage from 'components/ErrorMessage' import FieldError from 'components/base/forms/FieldError' +import WarehouseSetupSqlHelp from './WarehouseSetupSqlHelp' import WarningMessage from 'components/WarningMessage' import { ClickHouseConfig } from 'common/types/responses' import { useTestWarehouseConnectionConfigMutation } from 'common/services/useWarehouseConnection' @@ -21,6 +22,7 @@ import { getButtonLabel, getTestFailureWarning, getWarehouseErrorMessage, + isMissingEventsTableDetail, } from './warehouseFormUtils' import './ConfigForm.scss' @@ -50,11 +52,14 @@ const ClickHouseConfigForm: FC = ({ const [database, setDatabase] = useState(defaults.database) const [username, setUsername] = useState(defaults.username) const [password, setPassword] = useState('') - const [secure, setSecure] = useState(defaults.secure) + const [secure] = useState(defaults.secure) const [isSaving, setIsSaving] = useState(false) const [error, setError] = useState(false) const [testState, setTestState] = useState('idle') const [testDetail, setTestDetail] = useState(null) + // Sticky: the setup SQL stays offered once a test reports the table missing, + // even while the user edits fields (which resets the live test state). + const [showSetupSql, setShowSetupSql] = useState(false) const testRevision = useRef(0) const [testConnectionConfig, { isLoading: isTesting }] = @@ -104,6 +109,7 @@ const ClickHouseConfigForm: FC = ({ } else { setTestState('errored') setTestDetail(result.status_detail) + setShowSetupSql(isMissingEventsTableDetail(result.status_detail)) } } catch { if (revision !== testRevision.current) return @@ -147,7 +153,7 @@ const ClickHouseConfigForm: FC = ({
- + ) => @@ -179,13 +185,13 @@ const ClickHouseConfigForm: FC = ({ />
- + ) => setField(setDatabase)(e.target.value) } - placeholder='flagsmith' + placeholder='flagsmith_exp' />
@@ -197,7 +203,7 @@ const ClickHouseConfigForm: FC = ({ onChange={(e: React.ChangeEvent) => setField(setUsername)(e.target.value) } - placeholder='default' + placeholder='The user from the setup script' /> @@ -223,13 +229,20 @@ const ClickHouseConfigForm: FC = ({
- +
+ {isEdit && ( + + )} + {error && }
diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/WarehouseConnectionCard.tsx b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/WarehouseConnectionCard.tsx index 4daf8b516e6f..4989590c7ffe 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/WarehouseConnectionCard.tsx +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/WarehouseConnectionCard.tsx @@ -9,6 +9,8 @@ import Tooltip from 'components/Tooltip' import Icon from 'components/icons/Icon' import Button from 'components/base/forms/Button' import WarehouseEventCodeHelp from './WarehouseEventCodeHelp' +import WarehouseSetupSqlHelp from './WarehouseSetupSqlHelp' +import { CLICKHOUSE_DEFAULTS } from './clickhouseConfig' import WarehouseStats from './WarehouseStats' type WarehouseConnectionCardProps = { @@ -57,6 +59,12 @@ const WarehouseConnectionCard: FC = ({ const isFlagsmith = connection.warehouse_type === 'flagsmith' const isPending = connection.status === 'pending_connection' const isConnected = connection.status === 'connected' + // ClickHouse connections are born connected, so the first-event nudge stays + // until events actually arrive in the customer's warehouse. + const showSendFirstEvent = isFlagsmith + ? !isPending && !isConnected + : connection.warehouse_type === 'clickhouse' && + !connection.total_events_received const handleDelete = () => { openConfirm({ @@ -143,8 +151,20 @@ const WarehouseConnectionCard: FC = ({
)} + {connection.warehouse_type === 'clickhouse' && ( +
+ +
+ )}
- {onTestConnection && ( + {onTestConnection && !isConnected && ( )} - {isFlagsmith && !isPending && !isConnected && ( + {showSendFirstEvent && ( +
+ +) + +WarehouseSqlSnippet.displayName = 'WarehouseSqlSnippet' +export default WarehouseSqlSnippet diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts new file mode 100644 index 000000000000..a5562eec5a1b --- /dev/null +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts @@ -0,0 +1,59 @@ +import fs from 'fs' +import path from 'path' +import { CLICKHOUSE_DEFAULTS } from 'components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig' +import { + getClickHouseOnboardingSql, + getClickHouseSetupSql, +} from 'components/pages/environment-settings/tabs/warehouse-tab/clickhouseSetupSql' + +describe('getClickHouseSetupSql', () => { + it('interpolates the configured database into both statements', () => { + const sql = getClickHouseSetupSql('analytics') + + expect(sql).toContain('CREATE DATABASE IF NOT EXISTS analytics;') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS analytics.events') + expect(sql).not.toContain('flagsmith_exp') + }) + + it('matches the form default database', () => { + expect(getClickHouseSetupSql(CLICKHOUSE_DEFAULTS.database)).toContain( + 'CREATE TABLE IF NOT EXISTS flagsmith_exp.events', + ) + }) +}) + +describe('getClickHouseOnboardingSql', () => { + it('creates a placeholder user, grants it access, and includes the DDL', () => { + const sql = getClickHouseOnboardingSql() + + expect(sql).toContain('CREATE USER IF NOT EXISTS ') + expect(sql).toContain( + "IDENTIFIED WITH sha256_password BY '';", + ) + expect(sql).toContain( + 'GRANT SELECT, INSERT ON flagsmith_exp.events TO ;', + ) + expect(sql).toContain( + 'GRANT SHOW TABLES, SHOW COLUMNS ON flagsmith_exp.* TO ;', + ) + expect(sql).toContain(getClickHouseSetupSql('flagsmith_exp')) + }) + + it('matches the copy of the script in the documentation', () => { + const doc = fs.readFileSync( + path.join( + __dirname, + '../../../../../../../../docs/docs/experimentation/connect-a-warehouse.md', + ), + 'utf8', + ) + const fencedSql = doc.match(/```sql\n([\s\S]*?)```/)?.[1] ?? '' + const dedented = fencedSql + .split('\n') + .map((line) => line.replace(/^ {3}/, '')) + .join('\n') + .trim() + + expect(dedented).toEqual(getClickHouseOnboardingSql()) + }) +}) diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/warehouseFormUtils.test.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/warehouseFormUtils.test.ts index 588d487ac737..76da13c4ccc2 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/warehouseFormUtils.test.ts +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/warehouseFormUtils.test.ts @@ -9,4 +9,23 @@ describe('getTestFailureWarning', () => { ])('formats detail %p with a single sentence break', (detail, expected) => { expect(getTestFailureWarning(detail)).toContain(expected) }) + + it('does not claim a failed connection when only the events table is missing', () => { + const detail = + 'Events table not found in the configured database. Run the setup SQL to create it.' + + const warning = getTestFailureWarning(detail) + + expect(warning).not.toContain("couldn't establish a connection") + expect(warning).toContain(detail) + expect(warning).toContain( + "events won't be delivered until the table exists", + ) + }) + + it('keeps the sentence boundary when the missing-table detail lacks punctuation', () => { + expect(getTestFailureWarning('Events table not found')).toContain( + 'Events table not found. You can save anyway', + ) + }) }) diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig.ts index a727a392802b..81bd293a927a 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig.ts +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig.ts @@ -1,11 +1,11 @@ import { ClickHouseConfig } from 'common/types/responses' export const CLICKHOUSE_DEFAULTS: ClickHouseConfig = { - database: 'flagsmith', + database: 'flagsmith_exp', host: '', port: 9440, secure: true, - username: 'default', + username: '', } export type ClickHouseFormState = { diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseSetupSql.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseSetupSql.ts new file mode 100644 index 000000000000..048829449399 --- /dev/null +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/clickhouseSetupSql.ts @@ -0,0 +1,41 @@ +export const getClickHouseSetupSql = ( + database: string, +): string => `CREATE DATABASE IF NOT EXISTS ${database}; + +CREATE TABLE IF NOT EXISTS ${database}.events +( + environment_key LowCardinality(String), + event LowCardinality(String), + feature_name LowCardinality(String), + timestamp DateTime64(3), + collected_at DateTime64(3), + identifier String, + value String CODEC(ZSTD(3)), + traits String CODEC(ZSTD(3)), + metadata String CODEC(ZSTD(3)), + sdk_language LowCardinality(String), + sdk_version LowCardinality(String), + + INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4, + + CONSTRAINT environment_key_not_empty CHECK environment_key != '', + CONSTRAINT event_not_empty CHECK event != '', + CONSTRAINT timestamp_sane CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3) +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (environment_key, event, feature_name, timestamp, identifier);` + +export const getClickHouseOnboardingSql = + (): string => `-- Create a dedicated user for Flagsmith +CREATE USER IF NOT EXISTS + IDENTIFIED WITH sha256_password BY ''; + +-- Allow it to write and read experiment events +GRANT SELECT, INSERT ON flagsmith_exp.events TO ; + +-- Allow it to check the events table exists +GRANT SHOW TABLES, SHOW COLUMNS ON flagsmith_exp.* TO ; + +-- Create the database and events table +${getClickHouseSetupSql('flagsmith_exp')}` diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/warehouseFormUtils.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/warehouseFormUtils.ts index 4dfe4413817a..6fbc6bae1011 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/warehouseFormUtils.ts +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/warehouseFormUtils.ts @@ -8,7 +8,20 @@ export const getWarehouseErrorMessage = (isEdit: boolean): string => isEdit ? 'update' : 'create' } warehouse connection. Please try again.` +export const isMissingEventsTableDetail = (detail: string | null): boolean => + !!detail?.startsWith('Events table not found') + +const punctuate = (detail: string): string => + `${detail}${/[.!?]$/.test(detail) ? '' : '.'}` + export const getTestFailureWarning = (detail: string | null): string => { - const reason = detail ? `: ${detail}${/[.!?]$/.test(detail) ? '' : '.'}` : '.' + // The missing-table detail means the connection itself succeeded, so the + // "couldn't establish a connection" lead-in would be wrong. + if (detail && isMissingEventsTableDetail(detail)) { + return `${punctuate( + detail, + )} You can save anyway and test again later, but events won't be delivered until the table exists.` + } + const reason = detail ? `: ${punctuate(detail)}` : '.' return `We couldn't establish a connection${reason} You can save anyway and test again later, but events won't be delivered until the connection succeeds.` }