From c7cc9c3c15f3496e761edf5ade494e72d6de1b76 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 13:03:04 +0100 Subject: [PATCH 1/7] Add queryable columns shape allow-list --- .changeset/queryable-columns.md | 5 + .../lib/electric/shapes/api/params.ex | 3 + .../sync-service/lib/electric/shapes/shape.ex | 80 +++++++++- .../lib/electric/shapes/shape/subset.ex | 7 +- .../test/electric/plug/router_test.exs | 140 ++++++++++++++++++ .../test/electric/shapes/shape_test.exs | 65 ++++++++ website/docs/sync/api/http.md | 6 + website/docs/sync/guides/auth.md | 12 +- website/docs/sync/guides/shapes.md | 15 +- website/electric-api.yaml | 34 ++++- 10 files changed, 351 insertions(+), 16 deletions(-) create mode 100644 .changeset/queryable-columns.md diff --git a/.changeset/queryable-columns.md b/.changeset/queryable-columns.md new file mode 100644 index 0000000000..784effc5c4 --- /dev/null +++ b/.changeset/queryable-columns.md @@ -0,0 +1,5 @@ +--- +"@core/sync-service": patch +--- + +Add `queryable_columns` as a server-side shape allow-list for columns that may be queried or synced. This lets proxies decouple the `columns` sync projection from the security boundary while preventing `where`, subset filters and ordering, and projected columns from referencing non-queryable columns. diff --git a/packages/sync-service/lib/electric/shapes/api/params.ex b/packages/sync-service/lib/electric/shapes/api/params.ex index 6492011699..4273f78c5a 100644 --- a/packages/sync-service/lib/electric/shapes/api/params.ex +++ b/packages/sync-service/lib/electric/shapes/api/params.ex @@ -152,6 +152,7 @@ defmodule Electric.Shapes.Api.Params do field(:live, :boolean, default: false) field(:where, :string) field(:columns, ColumnList) + field(:queryable_columns, ColumnList) field(:shape_definition, :string) field(:replica, Ecto.Enum, values: [:default, :full], default: :default) field(:params, {:map, :string}, default: %{}) @@ -319,6 +320,7 @@ defmodule Electric.Shapes.Api.Params do table = fetch_change!(changeset, :table) where = fetch_field!(changeset, :where) columns = get_change(changeset, :columns, nil) + queryable_columns = get_change(changeset, :queryable_columns, nil) replica = fetch_field!(changeset, :replica) params = fetch_field!(changeset, :params) compaction_enabled? = fetch_field!(changeset, @tmp_compaction_flag) @@ -328,6 +330,7 @@ defmodule Electric.Shapes.Api.Params do where: where, params: params, columns: columns, + queryable_columns: queryable_columns, replica: replica, inspector: api.inspector, feature_flags: api.feature_flags, diff --git a/packages/sync-service/lib/electric/shapes/shape.ex b/packages/sync-service/lib/electric/shapes/shape.ex index f146a9b038..5e51775117 100644 --- a/packages/sync-service/lib/electric/shapes/shape.ex +++ b/packages/sync-service/lib/electric/shapes/shape.ex @@ -37,6 +37,7 @@ defmodule Electric.Shapes.Shape do :where, :selected_columns, :explicitly_selected_columns, + :queryable_columns, shape_dependencies: [], shape_dependencies_handles: [], tag_structure: [], @@ -69,6 +70,7 @@ defmodule Electric.Shapes.Shape do where: Electric.Replication.Eval.Expr.t() | nil, selected_columns: [String.t(), ...], explicitly_selected_columns: [String.t(), ...], + queryable_columns: [String.t(), ...] | nil, tag_structure: [String.t() | [String.t(), ...]], replica: replica(), storage: storage_config() | nil, @@ -163,6 +165,7 @@ defmodule Electric.Shapes.Shape do relation: [type: {:tuple, [:string, :string]}, required: true], where: [type: :any], columns: [type: {:or, [{:list, :string}, nil]}], + queryable_columns: [type: {:or, [{:list, :string}, nil]}], params: [type: {:map, :string, :string}, default: %{}], autofill_pk_select?: [type: :boolean, default: false], replica: [ @@ -230,14 +233,26 @@ defmodule Electric.Shapes.Shape do {:ok, {oid, table} = relation} <- validate_relation(opts, inspector), {:ok, column_info, pk_cols} <- load_column_info(relation, inspector), {:ok, supported_features} <- load_supported_features(inspector), + {:ok, stored_queryable_columns, effective_queryable_columns} <- + validate_queryable_columns(column_info, pk_cols, opts), {:ok, selected_columns, explicitly_selected_columns} <- - validate_selected_columns(column_info, pk_cols, supported_features, opts), - refs = Inspector.columns_to_expr(column_info), + validate_selected_columns( + column_info, + pk_cols, + supported_features, + Map.put(opts, :queryable_columns, effective_queryable_columns) + ), + refs = + column_info + |> filter_columns(effective_queryable_columns) + |> Inspector.columns_to_expr(), {:ok, where, shape_dependencies} <- validate_where_clause(Map.get(opts, :where), opts, refs) do flags = [ - if(is_nil(Map.get(opts, :columns)), do: :selects_all_columns), + if(is_nil(Map.get(opts, :columns)) and is_nil(Map.get(opts, :queryable_columns)), + do: :selects_all_columns + ), if(any_columns_generated?(column_info, selected_columns), do: :selects_generated_columns ), @@ -258,6 +273,7 @@ defmodule Electric.Shapes.Shape do where: where, selected_columns: selected_columns, explicitly_selected_columns: explicitly_selected_columns, + queryable_columns: stored_queryable_columns, replica: Map.get(opts, :replica, :default), storage: Map.get(opts, :storage) || %{compaction: :disabled}, shape_dependencies: shape_dependencies, @@ -322,7 +338,7 @@ defmodule Electric.Shapes.Shape do end defp build_shape_dependencies(subqueries, opts) do - shared_opts = Map.drop(opts, [:where, :columns, :relation]) + shared_opts = Map.drop(opts, [:where, :columns, :queryable_columns, :relation]) subqueries |> Enum.with_index() @@ -461,7 +477,9 @@ defmodule Electric.Shapes.Shape do autofill_pk_select? = Map.fetch!(opts, :autofill_pk_select?) missing_pk_cols = pk_cols -- columns_to_select + queryable_columns = Map.fetch!(opts, :queryable_columns) invalid_cols = columns_to_select -- Enum.map(column_info, & &1.name) + not_queryable_cols = columns_to_select -- queryable_columns generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in columns_to_select)) err_msg = @@ -473,6 +491,9 @@ defmodule Electric.Shapes.Shape do invalid_cols != [] -> "The following columns are not found on the table: " <> Enum.join(invalid_cols, ", ") + not_queryable_cols != [] -> + "The following columns are not queryable: " <> Enum.join(not_queryable_cols, ", ") + generated_cols != [] and not supports_generated_column_replication -> "The following columns are generated and cannot be included in the shape: " <> (generated_cols |> Enum.map(& &1.name) |> Enum.join(", ")) @@ -499,12 +520,13 @@ defmodule Electric.Shapes.Shape do column_info, _pk_cols, %{supports_generated_column_replication: supports_generated_column_replication}, - _opts + opts ) do - generated_cols = Enum.filter(column_info, & &1.is_generated) + queryable_columns = Map.fetch!(opts, :queryable_columns) + generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in queryable_columns)) if generated_cols == [] or supports_generated_column_replication do - all_columns = column_info |> Enum.map(& &1.name) |> Enum.sort() + all_columns = Enum.sort(queryable_columns) {:ok, all_columns, all_columns} else err_msg = @@ -517,6 +539,47 @@ defmodule Electric.Shapes.Shape do end end + defp validate_queryable_columns(column_info, pk_cols, opts) do + all_column_names = Enum.map(column_info, & &1.name) + requested_queryable_columns = Map.get(opts, :queryable_columns) + effective_queryable_columns = requested_queryable_columns || all_column_names + + missing_pk_cols = pk_cols -- effective_queryable_columns + invalid_cols = effective_queryable_columns -- all_column_names + + err_msg = + cond do + missing_pk_cols != [] -> + "The list of queryable columns must include all primary key columns, missing: " <> + Enum.join(missing_pk_cols, ", ") + + invalid_cols != [] -> + "The following queryable columns are not found on the table: " <> + Enum.join(invalid_cols, ", ") + + effective_queryable_columns == [] -> + "The list of queryable columns must not be empty" + + true -> + nil + end + + if is_nil(err_msg) do + stored_queryable_columns = + if is_nil(requested_queryable_columns), + do: nil, + else: Enum.sort(requested_queryable_columns) + + {:ok, stored_queryable_columns, Enum.sort(effective_queryable_columns)} + else + {:error, {:queryable_columns, [err_msg]}} + end + end + + defp filter_columns(column_info, column_names) do + Enum.filter(column_info, &(&1.name in column_names)) + end + defp table_not_found_error(relation), do: {:error, @@ -985,6 +1048,7 @@ defmodule Electric.Shapes.Shape do where: shape.where, selected_columns: shape.selected_columns, explicitly_selected_columns: shape.explicitly_selected_columns, + queryable_columns: shape.queryable_columns, storage: shape.storage, replica: shape.replica, shape_dependencies: Enum.map(shape.shape_dependencies, &to_json_safe/1), @@ -1025,6 +1089,7 @@ defmodule Electric.Shapes.Shape do selected_columns: selected_columns, explicitly_selected_columns: Map.get(data, "explicitly_selected_columns", selected_columns), + queryable_columns: Map.get(data, "queryable_columns"), storage: storage_config_from_json(storage), replica: String.to_existing_atom(replica), shape_dependencies: shape_dependencies, @@ -1095,6 +1160,7 @@ defmodule Electric.Shapes.Shape do flags: flags, where: where, selected_columns: actual_selected_columns, + queryable_columns: Map.get(data, "queryable_columns"), replica: String.to_atom(Map.get(data, "replica", "default")), storage: storage_config_from_json(Map.get(data, "storage")) }} diff --git a/packages/sync-service/lib/electric/shapes/shape/subset.ex b/packages/sync-service/lib/electric/shapes/shape/subset.ex index 21ec519f67..47c2799304 100644 --- a/packages/sync-service/lib/electric/shapes/shape/subset.ex +++ b/packages/sync-service/lib/electric/shapes/shape/subset.ex @@ -44,7 +44,7 @@ defmodule Electric.Shapes.Shape.Subset do end end - defp load_column_info(%{root_table_id: table_oid, root_table: table}, inspector) do + defp load_column_info(%{root_table_id: table_oid, root_table: table} = shape, inspector) do case Inspector.load_column_info(table_oid, inspector) do :table_not_found -> {:error, @@ -53,7 +53,10 @@ defmodule Electric.Shapes.Shape.Subset do "If the table name contains capitals or special characters you must quote it."}} {:ok, columns} -> - {:ok, columns} + case shape.queryable_columns do + nil -> {:ok, columns} + queryable_columns -> {:ok, Enum.filter(columns, &(&1.name in queryable_columns))} + end end end diff --git a/packages/sync-service/test/electric/plug/router_test.exs b/packages/sync-service/test/electric/plug/router_test.exs index ebef506cd2..fea7c5ed0e 100644 --- a/packages/sync-service/test/electric/plug/router_test.exs +++ b/packages/sync-service/test/electric/plug/router_test.exs @@ -1061,6 +1061,95 @@ defmodule Electric.Plug.RouterTest do assert [%{"key" => ^key, "value" => ^value}, _] = Jason.decode!(conn.resp_body) end + @tag with_sql: [ + "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", + "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" + ] + test "GET allows where clauses on queryable columns that are not synced", %{opts: opts} do + conn = + conn("GET", "/v1/shape", %{ + table: "queryable_users", + offset: "-1", + queryable_columns: "id,name,secret_token", + columns: "id,name", + where: "secret_token LIKE 'super%'" + }) + |> Router.call(opts) + + assert %{status: 200} = conn + + assert [ + %{ + "headers" => %{"operation" => "insert"}, + "value" => %{"id" => "1", "name" => "Alice"} + }, + %{"headers" => %{"control" => "snapshot-end"}} + ] = Jason.decode!(conn.resp_body) + end + + @tag with_sql: [ + "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", + "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" + ] + test "GET rejects where clauses and synced columns outside queryable_columns", %{opts: opts} do + conn = + conn("GET", "/v1/shape", %{ + table: "queryable_users", + offset: "-1", + queryable_columns: "id,name", + columns: "id,name", + where: "secret_token LIKE 'super%'" + }) + |> Router.call(opts) + + assert %{status: 400} = conn + assert %{"errors" => %{"where" => [message]}} = Jason.decode!(conn.resp_body) + assert message =~ "unknown reference secret_token" + + conn = + conn("GET", "/v1/shape", %{ + table: "queryable_users", + offset: "-1", + queryable_columns: "id,name", + columns: "id,name,secret_token" + }) + |> Router.call(opts) + + assert %{status: 400} = conn + + assert %{ + "errors" => %{ + "columns" => ["The following columns are not queryable: secret_token"] + } + } = Jason.decode!(conn.resp_body) + end + + @tag with_sql: [ + "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", + "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" + ] + test "GET defaults synced columns to queryable_columns when columns are omitted", %{ + opts: opts + } do + conn = + conn("GET", "/v1/shape", %{ + table: "queryable_users", + offset: "-1", + queryable_columns: "id,name" + }) + |> Router.call(opts) + + assert %{status: 200} = conn + + assert [ + %{ + "headers" => %{"operation" => "insert"}, + "value" => %{"id" => "1", "name" => "Alice"} + }, + %{"headers" => %{"control" => "snapshot-end"}} + ] = Jason.decode!(conn.resp_body) + end + test "GET works when there are changes not related to the shape in the same txn", %{ opts: opts, db_conn: db_conn @@ -3477,6 +3566,57 @@ defmodule Electric.Plug.RouterTest do ) end + @tag with_sql: [ + "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", + "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')", + "INSERT INTO queryable_users VALUES (2, 'Bob', 'public')" + ] + test "subset snapshots allow subset__where on queryable columns that are not synced", ctx do + conn = + conn("GET", "/v1/shape", %{ + "table" => "queryable_users", + "offset" => "-1", + "queryable_columns" => "id,name,secret_token", + "columns" => "id,name", + "subset__where" => "secret_token = 'supersecret123'" + }) + |> Router.call(ctx.opts) + + assert %{status: 200} = conn + + assert %{ + "metadata" => _, + "data" => [ + %{ + "value" => %{"id" => "1", "name" => "Alice"} + } + ] + } = Jason.decode!(conn.resp_body) + end + + @tag with_sql: [ + "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", + "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" + ] + test "subset snapshots reject subset__where clauses outside queryable_columns", ctx do + conn = + conn("GET", "/v1/shape", %{ + "table" => "queryable_users", + "offset" => "-1", + "queryable_columns" => "id,name", + "columns" => "id,name", + "subset__where" => "secret_token = 'supersecret123'" + }) + |> Router.call(ctx.opts) + + assert %{status: 400} = conn + + assert %{"errors" => %{"subset" => %{"where" => [message]}}} = + Jason.decode!(conn.resp_body) + + assert message =~ "unknown reference secret_token" + end + @tag with_sql: [ "INSERT INTO items VALUES (gen_random_uuid(), 'test value 1')", "INSERT INTO items VALUES (gen_random_uuid(), 'test value 2')" diff --git a/packages/sync-service/test/electric/shapes/shape_test.exs b/packages/sync-service/test/electric/shapes/shape_test.exs index d2bc900e69..24540c5b2a 100644 --- a/packages/sync-service/test/electric/shapes/shape_test.exs +++ b/packages/sync-service/test/electric/shapes/shape_test.exs @@ -618,6 +618,70 @@ defmodule Electric.Shapes.ShapeTest do Shape.new("col_table", inspector: inspector, columns: ["id", "value2"]) end + @tag with_sql: [ + "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" + ] + test "builds a shape with queryable columns and narrower synced columns", %{ + inspector: inspector + } do + assert {:ok, + %Shape{ + selected_columns: ["id", "value1"], + queryable_columns: ["id", "value1", "value2"], + where: %{query: "value2 = 'allowed'"} + }} = + Shape.new("col_table", + inspector: inspector, + queryable_columns: ["id", "value1", "value2"], + columns: ["id", "value1"], + where: "value2 = 'allowed'" + ) + end + + @tag with_sql: [ + "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" + ] + test "defaults synced columns to queryable columns when columns are omitted", %{ + inspector: inspector + } do + assert {:ok, + %Shape{ + selected_columns: ["id", "value1"], + queryable_columns: ["id", "value1"], + flags: flags + }} = + Shape.new("col_table", + inspector: inspector, + queryable_columns: ["id", "value1"] + ) + + refute Map.get(flags, :selects_all_columns) + end + + @tag with_sql: [ + "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" + ] + test "validates selected columns and where clauses against queryable columns", %{ + inspector: inspector + } do + assert {:error, {:columns, ["The following columns are not queryable: value2"]}} = + Shape.new("col_table", + inspector: inspector, + queryable_columns: ["id", "value1"], + columns: ["id", "value2"] + ) + + assert {:error, {:where, message}} = + Shape.new("col_table", + inspector: inspector, + queryable_columns: ["id", "value1"], + columns: ["id", "value1"], + where: "value2 = 'blocked'" + ) + + assert message =~ "unknown reference value2" + end + @tag with_sql: [ "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" ] @@ -1026,6 +1090,7 @@ defmodule Electric.Shapes.ShapeTest do root_pk: ["first", "second", "third"], root_column_count: 4, selected_columns: ["first", "second", "third", "fourth"], + queryable_columns: nil, flags: %{selects_all_columns: true}, where: nil } diff --git a/website/docs/sync/api/http.md b/website/docs/sync/api/http.md index 434e7f67e4..39593a408e 100644 --- a/website/docs/sync/api/http.md +++ b/website/docs/sync/api/http.md @@ -77,6 +77,8 @@ When you make an initial sync request, with `offset=-1`, you're telling the serv When a shape is first requested, Electric queries Postgres for the data and populates the log by turning the query results into insert operations. This allows you to sync shapes without having to pre-define them. Electric then streams out the log data in the response. +The `columns` query parameter controls which columns are synced in the response. The `queryable_columns` query parameter controls which columns may be referenced by the shape `where` clause, subset filters, subset ordering, and the `columns` projection. If `queryable_columns` is set and `columns` is omitted, Electric syncs the queryable columns by default. + Sometimes a log can fit in a single response. Sometimes it's too big and requires multiple requests. In this case, the first request will return a batch of data and an `electric-offset` header. An HTTP client should then continue to make requests setting the `offset` parameter to this header value. This allows the client to paginate through the shape log until it has received all of the current data. ### Control messages @@ -238,6 +240,8 @@ The POST body accepts these parameters: - `offset` - Number of rows to skip (for pagination) - `order_by` - ORDER BY clause (required when using limit/offset) +Subset `where` and `order_by` expressions may only reference columns allowed by `queryable_columns` when it is set. + #### Using GET (legacy) GET requests are still supported for backwards compatibility, using `subset__*` query parameters: @@ -254,6 +258,8 @@ The query parameters include: - `subset__offset` - Number of rows to skip (for pagination) - `subset__order_by` - ORDER BY clause (required when using limit/offset) +Subset `where` and `order_by` expressions may only reference columns allowed by `queryable_columns` when it is set. + #### Response format The response includes the requested data along with PostgreSQL snapshot metadata in a `snapshot-end` control message. This metadata allows clients to determine which subsequent changes have already been incorporated into the snapshot and should be skipped. diff --git a/website/docs/sync/guides/auth.md b/website/docs/sync/guides/auth.md index b655392db3..e9dce5f204 100644 --- a/website/docs/sync/guides/auth.md +++ b/website/docs/sync/guides/auth.md @@ -306,7 +306,8 @@ Electric separates parameters by purpose: - `table` — Root table name (required) - `offset` — Shape log position (required, e.g., `-1` for initial sync) - `handle` — Shape handle for continuation requests -- `columns` — Column selection +- `queryable_columns` — Column allow-list for WHERE, subset filtering, ordering, and synced projections +- `columns` — Synced column projection - `where` — Main shape WHERE clause (for non-subset queries) - `replica`, `log`, `live`, `live_sse` — Protocol options - `secret` / `api_secret` — API authentication @@ -333,7 +334,8 @@ The proxy must set these **shape definition parameters** server-side — they de | Parameter | Where | Security Consideration | |-----------|-------|------------------------| | `table` | URL | **Must be set server-side.** Letting clients specify the table allows access to any table. | -| `columns` | URL | **Should be set server-side.** Clients could request sensitive columns. | +| `queryable_columns` | URL | **Should be set server-side.** This is the column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | +| `columns` | URL | Optional sync projection. If clients can choose it, `queryable_columns` must also be set server-side so clients cannot sync columns outside the allow-list. | | `where` | URL | **Must be set server-side.** This is your authorization filter — the main shape WHERE that restricts all queries. | | `secret` | URL | **Must be set server-side.** Never expose the API secret to clients. | @@ -356,7 +358,7 @@ These parameters are safe because they either can't widen data access or are nee | `order_by` | POST body | Sorting for pagination | :::tip Key Principle -Your proxy is an **authorization layer** that controls the **shape definition** (table, columns, main WHERE). Clients can freely use subset parameters to filter and paginate within that shape — Electric ensures they can only narrow results, never escape the main WHERE clause. +Your proxy is an **authorization layer** that controls the **shape definition** (table, queryable columns, main WHERE). Clients can freely use subset parameters to filter and paginate within that shape — Electric ensures they can only narrow results and can only reference queryable columns. ::: ##### Implementing POST support in your proxy @@ -364,7 +366,7 @@ Your proxy is an **authorization layer** that controls the **shape definition** To support both GET and POST requests: 1. **Accept both methods** on your proxy endpoints -2. **Set shape definition server-side** — table, columns, and main WHERE clause +2. **Set shape definition server-side** — table, queryable columns, and main WHERE clause 3. **For POST**: Forward client subset params (they can only narrow results) 4. **For GET**: Send WHERE as URL query parameters (existing behavior) @@ -393,6 +395,8 @@ export async function handler(request: Request) { // Set shape definition server-side (this is your authorization layer) originUrl.searchParams.set(`table`, `items`) + originUrl.searchParams.set(`queryable_columns`, `id,title,organization_id`) + originUrl.searchParams.set(`columns`, `id,title`) originUrl.searchParams.set(`where`, `"organization_id" = '${user.org_id}'`) originUrl.searchParams.set(`secret`, process.env.ELECTRIC_SECRET) diff --git a/website/docs/sync/guides/shapes.md b/website/docs/sync/guides/shapes.md index 15ab4a810d..202b8c3a2b 100644 --- a/website/docs/sync/guides/shapes.md +++ b/website/docs/sync/guides/shapes.md @@ -47,6 +47,7 @@ Shapes are defined by: - a [table](#table), such as `items` - an optional [where clause](#where-clause) to filter which rows are included in the shape - an optional [columns](#columns) clause to select which columns are included +- an optional [queryable columns](#queryable-columns) clause to restrict which columns may be queried or synced A shape contains all of the rows in the table that match the where clause, if provided. If a columns clause is provided, the synced rows will only contain those selected columns. @@ -237,7 +238,7 @@ If you need an operator that isn't supported yet, please [raise a feature reques ### Columns -This is an optional list of columns to select. When specified, only the columns listed are synced. When not specified all columns are synced. +This is an optional list of columns to sync to the client. It is a projection setting for reducing the amount of data sent over the wire. When specified, only the listed columns are synced. When not specified, all columns are synced, unless [`queryable_columns`](#queryable-columns) is set, in which case the synced columns default to the queryable columns. For example: @@ -246,6 +247,18 @@ For example: The specified columns must always include the primary key column(s), and should be formed as a comma separated list of column names — exactly as they are in the database schema. If the identifier was defined as case sensitive and/or with special characters, then you must quote it. +### Queryable columns + +This is an optional list of columns that may be referenced by shape `where` clauses, subset `where` clauses, subset `order_by` clauses, and the `columns` projection. It is an allow-list for what a request may query or sync; it does not force every listed column to be synced. + +For example, this shape can filter by `org_id` without syncing `org_id` to the client: + +```http +/v1/shape?table=projects&queryable_columns=id,title,org_id&columns=id,title&where=org_id=$1¶ms[1]=org_123 +``` + +The specified queryable columns must include the primary key column(s). If `queryable_columns` is set and `columns` is omitted, Electric syncs the queryable columns by default. + ## Subscribing to shapes Local clients establish shape subscriptions, typically using [client libraries](/docs/sync/api/clients/typescript). These sync data from the [Electric sync engine](/sync/) into the client using the [HTTP API](/docs/sync/api/http). diff --git a/website/electric-api.yaml b/website/electric-api.yaml index 000c7cf215..71598043db 100644 --- a/website/electric-api.yaml +++ b/website/electric-api.yaml @@ -217,7 +217,12 @@ paths: schema: type: string description: |- - Optional list of columns to include in the rows from the `table`. + Optional list of columns to sync in the rows from the `table`. + + This is a projection setting for reducing the data sent to the client. + If `queryable_columns` is set, `columns` may only include columns from + that allow-list. If `queryable_columns` is set and `columns` is omitted, + Electric syncs the queryable columns by default. They should always include the primary key columns, and should be formed as a comma separated list of column names exactly as they are in the database schema. @@ -231,6 +236,24 @@ paths: select_columns_special: value: 'id,"Status-Check"' summary: Only include id and Status-Check columns, quoting the identifiers where necessary. + - name: queryable_columns + in: query + schema: + type: string + description: |- + Optional list of columns that may be referenced by the shape WHERE clause, + subset WHERE clauses, subset ORDER BY clauses, and the `columns` projection. + + This is an allow-list for what the request may query or sync. It does not + force every listed column to be synced. + + Queryable columns should always include the primary key columns, and should + be formed as a comma separated list of column names exactly as they are in + the database schema. + examples: + queryable_columns: + value: 'id,title,org_id' + summary: Allow filtering by org_id while syncing a narrower columns projection. - name: replica in: query schema: @@ -872,7 +895,14 @@ paths: schema: type: string description: |- - Optional list of columns to include in the rows. + Optional list of columns to sync in the rows. + - name: queryable_columns + in: query + schema: + type: string + description: |- + Optional list of columns that may be referenced by WHERE clauses, + subset filters, subset ordering, and the columns projection. - name: replica in: query schema: From 936812bf3879b178ac58942980d33b68ab07784b Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 14:41:38 +0100 Subject: [PATCH 2/7] Simplify queryable columns validation --- .../sync-service/lib/electric/shapes/shape.ex | 49 +++++++++++-------- .../test/electric/shapes/shape_test.exs | 20 ++++++++ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/packages/sync-service/lib/electric/shapes/shape.ex b/packages/sync-service/lib/electric/shapes/shape.ex index 5e51775117..38314b233f 100644 --- a/packages/sync-service/lib/electric/shapes/shape.ex +++ b/packages/sync-service/lib/electric/shapes/shape.ex @@ -233,18 +233,17 @@ defmodule Electric.Shapes.Shape do {:ok, {oid, table} = relation} <- validate_relation(opts, inspector), {:ok, column_info, pk_cols} <- load_column_info(relation, inspector), {:ok, supported_features} <- load_supported_features(inspector), - {:ok, stored_queryable_columns, effective_queryable_columns} <- - validate_queryable_columns(column_info, pk_cols, opts), + {:ok, queryable_columns} <- validate_queryable_columns(column_info, pk_cols, opts), {:ok, selected_columns, explicitly_selected_columns} <- validate_selected_columns( column_info, pk_cols, supported_features, - Map.put(opts, :queryable_columns, effective_queryable_columns) + Map.put(opts, :queryable_columns, queryable_columns) ), refs = column_info - |> filter_columns(effective_queryable_columns) + |> filter_columns(queryable_columns) |> Inspector.columns_to_expr(), {:ok, where, shape_dependencies} <- validate_where_clause(Map.get(opts, :where), opts, refs) do @@ -273,7 +272,7 @@ defmodule Electric.Shapes.Shape do where: where, selected_columns: selected_columns, explicitly_selected_columns: explicitly_selected_columns, - queryable_columns: stored_queryable_columns, + queryable_columns: queryable_columns, replica: Map.get(opts, :replica, :default), storage: Map.get(opts, :storage) || %{compaction: :disabled}, shape_dependencies: shape_dependencies, @@ -479,7 +478,7 @@ defmodule Electric.Shapes.Shape do missing_pk_cols = pk_cols -- columns_to_select queryable_columns = Map.fetch!(opts, :queryable_columns) invalid_cols = columns_to_select -- Enum.map(column_info, & &1.name) - not_queryable_cols = columns_to_select -- queryable_columns + not_queryable_cols = non_queryable_columns(columns_to_select, queryable_columns) generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in columns_to_select)) err_msg = @@ -522,11 +521,11 @@ defmodule Electric.Shapes.Shape do %{supports_generated_column_replication: supports_generated_column_replication}, opts ) do - queryable_columns = Map.fetch!(opts, :queryable_columns) - generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in queryable_columns)) + columns_to_select = Map.fetch!(opts, :queryable_columns) || Enum.map(column_info, & &1.name) + generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in columns_to_select)) if generated_cols == [] or supports_generated_column_replication do - all_columns = Enum.sort(queryable_columns) + all_columns = Enum.sort(columns_to_select) {:ok, all_columns, all_columns} else err_msg = @@ -539,13 +538,24 @@ defmodule Electric.Shapes.Shape do end end - defp validate_queryable_columns(column_info, pk_cols, opts) do + defp non_queryable_columns(_columns_to_select, nil), do: [] + + defp non_queryable_columns(columns_to_select, queryable_columns), + do: columns_to_select -- queryable_columns + + defp validate_queryable_columns(column_info, pk_cols, opts) when is_map(opts) do + case Map.get(opts, :queryable_columns) do + nil -> {:ok, nil} + queryable_columns -> validate_queryable_columns(column_info, pk_cols, queryable_columns) + end + end + + defp validate_queryable_columns(column_info, pk_cols, queryable_columns) + when is_list(queryable_columns) do all_column_names = Enum.map(column_info, & &1.name) - requested_queryable_columns = Map.get(opts, :queryable_columns) - effective_queryable_columns = requested_queryable_columns || all_column_names - missing_pk_cols = pk_cols -- effective_queryable_columns - invalid_cols = effective_queryable_columns -- all_column_names + missing_pk_cols = pk_cols -- queryable_columns + invalid_cols = queryable_columns -- all_column_names err_msg = cond do @@ -557,7 +567,7 @@ defmodule Electric.Shapes.Shape do "The following queryable columns are not found on the table: " <> Enum.join(invalid_cols, ", ") - effective_queryable_columns == [] -> + queryable_columns == [] -> "The list of queryable columns must not be empty" true -> @@ -565,17 +575,14 @@ defmodule Electric.Shapes.Shape do end if is_nil(err_msg) do - stored_queryable_columns = - if is_nil(requested_queryable_columns), - do: nil, - else: Enum.sort(requested_queryable_columns) - - {:ok, stored_queryable_columns, Enum.sort(effective_queryable_columns)} + {:ok, Enum.sort(queryable_columns)} else {:error, {:queryable_columns, [err_msg]}} end end + defp filter_columns(column_info, nil), do: column_info + defp filter_columns(column_info, column_names) do Enum.filter(column_info, &(&1.name in column_names)) end diff --git a/packages/sync-service/test/electric/shapes/shape_test.exs b/packages/sync-service/test/electric/shapes/shape_test.exs index 24540c5b2a..1248170270 100644 --- a/packages/sync-service/test/electric/shapes/shape_test.exs +++ b/packages/sync-service/test/electric/shapes/shape_test.exs @@ -618,6 +618,26 @@ defmodule Electric.Shapes.ShapeTest do Shape.new("col_table", inspector: inspector, columns: ["id", "value2"]) end + @tag with_sql: [ + "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" + ] + test "does not restrict selected columns or where clauses when queryable columns are omitted", + %{ + inspector: inspector + } do + assert {:ok, + %Shape{ + selected_columns: ["id", "value1"], + queryable_columns: nil, + where: %{query: "value2 = 'allowed'"} + }} = + Shape.new("col_table", + inspector: inspector, + columns: ["id", "value1"], + where: "value2 = 'allowed'" + ) + end + @tag with_sql: [ "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" ] From bbd5385f7c99144cd7a5f85b1859e5e632da2ed7 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 14:52:45 +0100 Subject: [PATCH 3/7] Address queryable columns docs feedback --- website/docs/sync/api/http.md | 2 -- website/docs/sync/guides/auth.md | 6 +++--- website/docs/sync/guides/shapes.md | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/website/docs/sync/api/http.md b/website/docs/sync/api/http.md index 39593a408e..46f3bb22a3 100644 --- a/website/docs/sync/api/http.md +++ b/website/docs/sync/api/http.md @@ -77,8 +77,6 @@ When you make an initial sync request, with `offset=-1`, you're telling the serv When a shape is first requested, Electric queries Postgres for the data and populates the log by turning the query results into insert operations. This allows you to sync shapes without having to pre-define them. Electric then streams out the log data in the response. -The `columns` query parameter controls which columns are synced in the response. The `queryable_columns` query parameter controls which columns may be referenced by the shape `where` clause, subset filters, subset ordering, and the `columns` projection. If `queryable_columns` is set and `columns` is omitted, Electric syncs the queryable columns by default. - Sometimes a log can fit in a single response. Sometimes it's too big and requires multiple requests. In this case, the first request will return a batch of data and an `electric-offset` header. An HTTP client should then continue to make requests setting the `offset` parameter to this header value. This allows the client to paginate through the shape log until it has received all of the current data. ### Control messages diff --git a/website/docs/sync/guides/auth.md b/website/docs/sync/guides/auth.md index e9dce5f204..8d4dd655c0 100644 --- a/website/docs/sync/guides/auth.md +++ b/website/docs/sync/guides/auth.md @@ -334,8 +334,8 @@ The proxy must set these **shape definition parameters** server-side — they de | Parameter | Where | Security Consideration | |-----------|-------|------------------------| | `table` | URL | **Must be set server-side.** Letting clients specify the table allows access to any table. | -| `queryable_columns` | URL | **Should be set server-side.** This is the column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | -| `columns` | URL | Optional sync projection. If clients can choose it, `queryable_columns` must also be set server-side so clients cannot sync columns outside the allow-list. | +| `columns` | URL | Optional sync projection. If clients can choose it, use `queryable_columns` to restrict which columns they can sync. | +| `queryable_columns` | URL | **Should be set server-side.** Column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | | `where` | URL | **Must be set server-side.** This is your authorization filter — the main shape WHERE that restricts all queries. | | `secret` | URL | **Must be set server-side.** Never expose the API secret to clients. | @@ -358,7 +358,7 @@ These parameters are safe because they either can't widen data access or are nee | `order_by` | POST body | Sorting for pagination | :::tip Key Principle -Your proxy is an **authorization layer** that controls the **shape definition** (table, queryable columns, main WHERE). Clients can freely use subset parameters to filter and paginate within that shape — Electric ensures they can only narrow results and can only reference queryable columns. +Your proxy is an **authorization layer** that controls the **shape definition** (table, queryable columns, main WHERE). Clients can freely use subset parameters to filter and paginate within that shape — Electric ensures they can never escape the main WHERE clause and can only reference queryable columns. ::: ##### Implementing POST support in your proxy diff --git a/website/docs/sync/guides/shapes.md b/website/docs/sync/guides/shapes.md index 202b8c3a2b..0a157f94b0 100644 --- a/website/docs/sync/guides/shapes.md +++ b/website/docs/sync/guides/shapes.md @@ -238,7 +238,7 @@ If you need an operator that isn't supported yet, please [raise a feature reques ### Columns -This is an optional list of columns to sync to the client. It is a projection setting for reducing the amount of data sent over the wire. When specified, only the listed columns are synced. When not specified, all columns are synced, unless [`queryable_columns`](#queryable-columns) is set, in which case the synced columns default to the queryable columns. +This is an optional list of columns to select. When specified, only the columns listed are synced. When not specified all columns are synced, unless [`queryable_columns`](#queryable-columns) is set, in which case the synced columns default to the queryable columns. For example: From d15d9cd17f63d222397c29cdbac9cbf8005c5691 Mon Sep 17 00:00:00 2001 From: Rob A'Court Date: Mon, 8 Jun 2026 15:02:34 +0100 Subject: [PATCH 4/7] Update copy --- website/docs/sync/guides/auth.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/sync/guides/auth.md b/website/docs/sync/guides/auth.md index 8d4dd655c0..ee711969aa 100644 --- a/website/docs/sync/guides/auth.md +++ b/website/docs/sync/guides/auth.md @@ -335,7 +335,7 @@ The proxy must set these **shape definition parameters** server-side — they de |-----------|-------|------------------------| | `table` | URL | **Must be set server-side.** Letting clients specify the table allows access to any table. | | `columns` | URL | Optional sync projection. If clients can choose it, use `queryable_columns` to restrict which columns they can sync. | -| `queryable_columns` | URL | **Should be set server-side.** Column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | +| `queryable_columns` | URL | **Must be set server-side if the table has sensitive columns.** Column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | | `where` | URL | **Must be set server-side.** This is your authorization filter — the main shape WHERE that restricts all queries. | | `secret` | URL | **Must be set server-side.** Never expose the API secret to clients. | From 53624ab3335d70ad43d9277f55667ce19c910a47 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 15:07:32 +0100 Subject: [PATCH 5/7] Remove unnecessary row --- website/docs/sync/guides/auth.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/docs/sync/guides/auth.md b/website/docs/sync/guides/auth.md index ee711969aa..c58f2d9d7e 100644 --- a/website/docs/sync/guides/auth.md +++ b/website/docs/sync/guides/auth.md @@ -334,7 +334,6 @@ The proxy must set these **shape definition parameters** server-side — they de | Parameter | Where | Security Consideration | |-----------|-------|------------------------| | `table` | URL | **Must be set server-side.** Letting clients specify the table allows access to any table. | -| `columns` | URL | Optional sync projection. If clients can choose it, use `queryable_columns` to restrict which columns they can sync. | | `queryable_columns` | URL | **Must be set server-side if the table has sensitive columns.** Column allow-list for WHERE clauses, subset filters, ordering, and synced projections. | | `where` | URL | **Must be set server-side.** This is your authorization filter — the main shape WHERE that restricts all queries. | | `secret` | URL | **Must be set server-side.** Never expose the API secret to clients. | From 00d50f39d52915c8d46dd5ca541d1bdef4a26000 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 15:35:34 +0100 Subject: [PATCH 6/7] Clarify queryable columns test names --- packages/sync-service/test/electric/plug/router_test.exs | 8 ++++---- packages/sync-service/test/electric/shapes/shape_test.exs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/sync-service/test/electric/plug/router_test.exs b/packages/sync-service/test/electric/plug/router_test.exs index fea7c5ed0e..8c44493420 100644 --- a/packages/sync-service/test/electric/plug/router_test.exs +++ b/packages/sync-service/test/electric/plug/router_test.exs @@ -1065,7 +1065,7 @@ defmodule Electric.Plug.RouterTest do "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" ] - test "GET allows where clauses on queryable columns that are not synced", %{opts: opts} do + test "GET allows where clauses on queryable columns that are not selected", %{opts: opts} do conn = conn("GET", "/v1/shape", %{ table: "queryable_users", @@ -1091,7 +1091,7 @@ defmodule Electric.Plug.RouterTest do "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" ] - test "GET rejects where clauses and synced columns outside queryable_columns", %{opts: opts} do + test "GET rejects where clauses and selected columns outside queryable_columns", %{opts: opts} do conn = conn("GET", "/v1/shape", %{ table: "queryable_users", @@ -1128,7 +1128,7 @@ defmodule Electric.Plug.RouterTest do "CREATE TABLE queryable_users (id BIGINT PRIMARY KEY, name TEXT NOT NULL, secret_token TEXT NOT NULL)", "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')" ] - test "GET defaults synced columns to queryable_columns when columns are omitted", %{ + test "GET defaults selected columns to queryable_columns when columns are omitted", %{ opts: opts } do conn = @@ -3571,7 +3571,7 @@ defmodule Electric.Plug.RouterTest do "INSERT INTO queryable_users VALUES (1, 'Alice', 'supersecret123')", "INSERT INTO queryable_users VALUES (2, 'Bob', 'public')" ] - test "subset snapshots allow subset__where on queryable columns that are not synced", ctx do + test "subset snapshots allow subset__where on queryable columns that are not selected", ctx do conn = conn("GET", "/v1/shape", %{ "table" => "queryable_users", diff --git a/packages/sync-service/test/electric/shapes/shape_test.exs b/packages/sync-service/test/electric/shapes/shape_test.exs index 1248170270..5f467933d4 100644 --- a/packages/sync-service/test/electric/shapes/shape_test.exs +++ b/packages/sync-service/test/electric/shapes/shape_test.exs @@ -641,7 +641,7 @@ defmodule Electric.Shapes.ShapeTest do @tag with_sql: [ "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" ] - test "builds a shape with queryable columns and narrower synced columns", %{ + test "builds a shape with queryable columns and narrower selected columns", %{ inspector: inspector } do assert {:ok, @@ -661,7 +661,7 @@ defmodule Electric.Shapes.ShapeTest do @tag with_sql: [ "CREATE TABLE IF NOT EXISTS col_table (id INT PRIMARY KEY, value1 TEXT, value2 TEXT)" ] - test "defaults synced columns to queryable columns when columns are omitted", %{ + test "defaults selected columns to queryable columns when columns are omitted", %{ inspector: inspector } do assert {:ok, From ad319b94fdaf823bdf09df238db7f63427a84ff9 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 8 Jun 2026 15:47:29 +0100 Subject: [PATCH 7/7] Avoid revealing excluded selected columns --- packages/sync-service/lib/electric/shapes/shape.ex | 12 ++---------- .../sync-service/test/electric/plug/router_test.exs | 2 +- .../sync-service/test/electric/shapes/shape_test.exs | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/packages/sync-service/lib/electric/shapes/shape.ex b/packages/sync-service/lib/electric/shapes/shape.ex index 38314b233f..ff2c3ad40b 100644 --- a/packages/sync-service/lib/electric/shapes/shape.ex +++ b/packages/sync-service/lib/electric/shapes/shape.ex @@ -477,8 +477,8 @@ defmodule Electric.Shapes.Shape do missing_pk_cols = pk_cols -- columns_to_select queryable_columns = Map.fetch!(opts, :queryable_columns) - invalid_cols = columns_to_select -- Enum.map(column_info, & &1.name) - not_queryable_cols = non_queryable_columns(columns_to_select, queryable_columns) + selectable_column_names = queryable_columns || Enum.map(column_info, & &1.name) + invalid_cols = columns_to_select -- selectable_column_names generated_cols = Enum.filter(column_info, &(&1.is_generated and &1.name in columns_to_select)) err_msg = @@ -490,9 +490,6 @@ defmodule Electric.Shapes.Shape do invalid_cols != [] -> "The following columns are not found on the table: " <> Enum.join(invalid_cols, ", ") - not_queryable_cols != [] -> - "The following columns are not queryable: " <> Enum.join(not_queryable_cols, ", ") - generated_cols != [] and not supports_generated_column_replication -> "The following columns are generated and cannot be included in the shape: " <> (generated_cols |> Enum.map(& &1.name) |> Enum.join(", ")) @@ -538,11 +535,6 @@ defmodule Electric.Shapes.Shape do end end - defp non_queryable_columns(_columns_to_select, nil), do: [] - - defp non_queryable_columns(columns_to_select, queryable_columns), - do: columns_to_select -- queryable_columns - defp validate_queryable_columns(column_info, pk_cols, opts) when is_map(opts) do case Map.get(opts, :queryable_columns) do nil -> {:ok, nil} diff --git a/packages/sync-service/test/electric/plug/router_test.exs b/packages/sync-service/test/electric/plug/router_test.exs index 8c44493420..673a52dbaa 100644 --- a/packages/sync-service/test/electric/plug/router_test.exs +++ b/packages/sync-service/test/electric/plug/router_test.exs @@ -1119,7 +1119,7 @@ defmodule Electric.Plug.RouterTest do assert %{ "errors" => %{ - "columns" => ["The following columns are not queryable: secret_token"] + "columns" => ["The following columns are not found on the table: secret_token"] } } = Jason.decode!(conn.resp_body) end diff --git a/packages/sync-service/test/electric/shapes/shape_test.exs b/packages/sync-service/test/electric/shapes/shape_test.exs index 5f467933d4..ef740babf3 100644 --- a/packages/sync-service/test/electric/shapes/shape_test.exs +++ b/packages/sync-service/test/electric/shapes/shape_test.exs @@ -684,7 +684,7 @@ defmodule Electric.Shapes.ShapeTest do test "validates selected columns and where clauses against queryable columns", %{ inspector: inspector } do - assert {:error, {:columns, ["The following columns are not queryable: value2"]}} = + assert {:error, {:columns, ["The following columns are not found on the table: value2"]}} = Shape.new("col_table", inspector: inspector, queryable_columns: ["id", "value1"],