From e819f3f6908b00454f21ee04b92f06a82d11b359 Mon Sep 17 00:00:00 2001 From: CJ Brewer Date: Thu, 17 Oct 2024 12:58:12 -0600 Subject: [PATCH 1/4] feat: update install script and add usage section --- README.md | 31 +++++++++++++++++++++++++++++++ src/install.sql | 27 ++++++++++++++------------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bdfe3ce8..e090daa3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ EQL provides a data format for transmitting and storing encrypted data & indexes ## Table of Contents +- [Installation](#installation) +- [Usage](#usage) - [Encrypted columns](#encrypted-columns) - [Inserting data](#inserting-data) - [Reading data](#reading-data) @@ -21,6 +23,35 @@ EQL provides a data format for transmitting and storing encrypted data & indexes --- +## Installation + +The simplest and fastest way to get up and running with EQL from scratch is to execute the install SQL file directly in your database. + +1. Download the [install.sql] file +2. Run the following command to install the custom types and functions: + +```bash +psql -f install.sql +``` + +## Usage + +Once the custom types and functions are installed, you can start using EQL in your queries. + +1. Create a table with a column of type `cs_encrypted_v1` which will store your encrypted data. +1. Use EQL functions to add indexes for the columns you want to encrypt. + - Indexes are used by Cipherstash Proxy to understand what cryptography schemes are required for your use case. +1. Initialize Cipherstash Proxy for cryptographic operations. + - The Proxy will dynamically encrypt data on the way in and decrypt data on the way out based on the indexes you have defined. +1. Insert data into the defined columns using a specific payload format. + - The payload format is defined in the [data format](#data-format) section. +1. Query the data using the EQL functions defined in the [querying data with EQL](#querying-data-with-eql) section. + - No modifications are required to simply `SELECT` data from your encrypted columns. + - In order to perform `WHERE` and `ORDER BY` queries, you must wrap the queries in the EQL functions defined in the [querying data with EQL](#querying-data-with-eql) section. +1. Integrate with your application via the [helper packages](#helper-packages) to interact with the encrypted data. + +You can find a full getting started guide in the [GETTINGSTARTED.md](GETTINGSTARTED.md) file. + ## Encrypted columns EQL relies on your database schema to define encrypted columns. diff --git a/src/install.sql b/src/install.sql index b7683453..03f62cfa 100644 --- a/src/install.sql +++ b/src/install.sql @@ -19,7 +19,7 @@ DROP DOMAIN IF EXISTS cs_unique_index_v1; CREATE DOMAIN cs_match_index_v1 AS smallint[]; CREATE DOMAIN cs_unique_index_v1 AS text; -CREATE DOMAIN cs_ste_vec_v1 AS text[]; +CREATE DOMAIN cs_ste_vec_index_v1 AS text[]; -- cs_encrypted_v1 is a column type and cannot be dropped if in use DO $$ @@ -120,23 +120,23 @@ BEGIN ATOMIC RETURN cs_unique_v1_v0_0(col); END; --- extracts json containment index from an encrypted column +-- extracts json ste_vec index from an encrypted column CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0_0(col jsonb) - RETURNS cs_ste_vec_v1 + RETURNS cs_ste_vec_index_v1 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE BEGIN ATOMIC - SELECT ARRAY(SELECT jsonb_array_elements(col->'sv'))::cs_ste_vec_v1; + SELECT ARRAY(SELECT jsonb_array_elements(col->'sv'))::cs_ste_vec_index_v1; END; CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0(col jsonb) - RETURNS cs_ste_vec_v1 + RETURNS cs_ste_vec_index_v1 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE BEGIN ATOMIC RETURN cs_ste_vec_v1_v0_0(col); END; CREATE OR REPLACE FUNCTION cs_ste_vec_v1(col jsonb) - RETURNS cs_ste_vec_v1 + RETURNS cs_ste_vec_index_v1 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE BEGIN ATOMIC RETURN cs_ste_vec_v1_v0_0(col); @@ -223,7 +223,7 @@ CREATE FUNCTION _cs_config_check_indexes(val jsonb) RETURNS BOOLEAN LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE BEGIN ATOMIC - SELECT jsonb_object_keys(jsonb_path_query(val, '$.tables.*.*.indexes')) = ANY('{match, ore, unique, json}'); + SELECT jsonb_object_keys(jsonb_path_query(val, '$.tables.*.*.indexes')) = ANY('{match, ore, unique, ste_vec}'); END; @@ -289,7 +289,7 @@ DROP FUNCTION IF EXISTS cs_discard_v1(); DROP FUNCTION IF EXISTS cs_refresh_encrypt_config(); DROP FUNCTION IF EXISTS _cs_config_default(); -DROP FUNCTION IF EXISTS _cs_config_match_1_default(); +DROP FUNCTION IF EXISTS _cs_config_match_default(); DROP FUNCTION IF EXISTS _cs_config_add_table(text, json); DROP FUNCTION IF EXISTS _cs_config_add_column(text, text, json); @@ -367,9 +367,9 @@ $$ LANGUAGE plpgsql; -- --- Default options for match_1 index +-- Default options for match index -- -CREATE FUNCTION _cs_config_match_1_default() +CREATE FUNCTION _cs_config_match_default() RETURNS jsonb LANGUAGE sql STRICT PARALLEL SAFE BEGIN ATOMIC @@ -414,8 +414,8 @@ AS $$ SELECT _cs_config_add_cast(table_name, column_name, cast_as, _config) INTO _config; -- set default options for index if opts empty - IF index_name = 'match_1' AND opts = '{}' THEN - SELECT _cs_config_match_1_default() INTO opts; + IF index_name = 'match' AND opts = '{}' THEN + SELECT _cs_config_match_default() INTO opts; END IF; SELECT _cs_config_add_index(table_name, column_name, index_name, opts, _config) INTO _config; @@ -816,4 +816,5 @@ BEGIN INTO result; RETURN result; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; + From 4eccb58ea0017db77d12c76b6f6fae01fba85a0d Mon Sep 17 00:00:00 2001 From: CJ Brewer Date: Thu, 17 Oct 2024 16:15:13 -0600 Subject: [PATCH 2/4] feat: create reference for activate methods --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index e090daa3..35988d02 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,33 @@ CREATE TABLE users ( In some instances, especially when using langugage specific ORMs, EQL also supports `jsonb` columns rather than the `cs_encrypted_v1` domain type. +### Configuring the column + +In order for CipherStash Proxy to encrypt and decrypt the data, you can initialize the column in the database using the `cs_add_column_v1` function. +This function takes the following parameters: + +- `table_name`: The name of the table containing the encrypted column. +- `column_name`: The name of the encrypted column. + +This function will **not** enable searchable encryption, but will allow you to encrypt and decrypt data. +See [querying data with EQL](#querying-data-with-eql) for more information on how to enable searchable encryption. + +```sql +SELECT cs_add_column_v1('table', 'column'); +``` + +### Activate configuration + +By default, the state of the configuration is `pending` after any modifications. +You can activate the configuration by running the `cs_encrypt_v1` and `cs_activate_v1` function. + +```sql +SELECT cs_encrypt_v1(); +SELECT cs_activate_v1(); +``` + +> **Important:** These functions must be run after any modifications to the configuration. + ### Inserting data When inserting data into the encrypted column, you must wrap the plaintext in the appropriate EQL payload. From bb45ce5e116ba09539979957e1894947ece7b489 Mon Sep 17 00:00:00 2001 From: CJ Brewer Date: Thu, 17 Oct 2024 16:15:57 -0600 Subject: [PATCH 3/4] Update README.md Co-authored-by: Drew Thomas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35988d02..9d19c74b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ EQL provides a data format for transmitting and storing encrypted data & indexes The simplest and fastest way to get up and running with EQL from scratch is to execute the install SQL file directly in your database. -1. Download the [install.sql] file +1. Download the [install.sql](src/install.sql) file 2. Run the following command to install the custom types and functions: ```bash From d8c4e86789ac873afbb3ad65e1c6fd7735d9700d Mon Sep 17 00:00:00 2001 From: CJ Brewer Date: Thu, 17 Oct 2024 16:16:04 -0600 Subject: [PATCH 4/4] Update README.md Co-authored-by: Drew Thomas --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9d19c74b..579de2d1 100644 --- a/README.md +++ b/README.md @@ -40,14 +40,14 @@ Once the custom types and functions are installed, you can start using EQL in yo 1. Create a table with a column of type `cs_encrypted_v1` which will store your encrypted data. 1. Use EQL functions to add indexes for the columns you want to encrypt. - - Indexes are used by Cipherstash Proxy to understand what cryptography schemes are required for your use case. + - Indexes are used by Cipherstash Proxy to understand what cryptography schemes are required for your use case. 1. Initialize Cipherstash Proxy for cryptographic operations. - - The Proxy will dynamically encrypt data on the way in and decrypt data on the way out based on the indexes you have defined. + - The Proxy will dynamically encrypt data on the way in and decrypt data on the way out based on the indexes you have defined. 1. Insert data into the defined columns using a specific payload format. - - The payload format is defined in the [data format](#data-format) section. + - The payload format is defined in the [data format](#data-format) section. 1. Query the data using the EQL functions defined in the [querying data with EQL](#querying-data-with-eql) section. - - No modifications are required to simply `SELECT` data from your encrypted columns. - - In order to perform `WHERE` and `ORDER BY` queries, you must wrap the queries in the EQL functions defined in the [querying data with EQL](#querying-data-with-eql) section. + - No modifications are required to simply `SELECT` data from your encrypted columns. + - In order to perform `WHERE` and `ORDER BY` queries, you must wrap the queries in the EQL functions defined in the [querying data with EQL](#querying-data-with-eql) section. 1. Integrate with your application via the [helper packages](#helper-packages) to interact with the encrypted data. You can find a full getting started guide in the [GETTINGSTARTED.md](GETTINGSTARTED.md) file.