diff --git a/.gitignore b/.gitignore index a2cb8918..893e4e74 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ web_modules/ .env.test.local .env.production.local .env.local +.envrc # parcel-bundler cache (https://parceljs.org/) diff --git a/go/xorm/.envrc.example b/go/xorm/.envrc.example new file mode 100644 index 00000000..5adb8edf --- /dev/null +++ b/go/xorm/.envrc.example @@ -0,0 +1,18 @@ +export CS_WORKSPACE_ID= + +# Used by Stash CLI +export CS_CLIENT_ID= +export CS_CLIENT_KEY= + +# Used by Proxy +export CS_ENCRYPTION__CLIENT_ID= +export CS_ENCRYPTION__CLIENT_KEY= +export CS_AUDIT__ENABLED=false +export CS_DATABASE__PORT=5432 +export CS_DATABASE__USERNAME=postgres +export CS_DATABASE__PASSWORD=postgres +export CS_DATABASE__NAME=gotest +export CS_DATABASE__HOST=localhost +export CS_TEST_ON_CHECKOUT=true +export CS_STATEMENT_HANDLER=mylittleproxy +export CS_UNSAFE_LOGGING=true diff --git a/go/xorm/.gitignore b/go/xorm/.gitignore new file mode 100644 index 00000000..7a6353d6 --- /dev/null +++ b/go/xorm/.gitignore @@ -0,0 +1 @@ +.envrc diff --git a/go/xorm/README.md b/go/xorm/README.md new file mode 100644 index 00000000..561b3310 --- /dev/null +++ b/go/xorm/README.md @@ -0,0 +1,3 @@ +# EQL Go/Xorm example + + diff --git a/go/xorm/cipherstash-encrypt-dsl.sql b/go/xorm/cipherstash-encrypt-dsl.sql new file mode 100644 index 00000000..4acdba50 --- /dev/null +++ b/go/xorm/cipherstash-encrypt-dsl.sql @@ -0,0 +1,830 @@ +DROP CAST IF EXISTS (text AS ore_64_8_v1_term); + +DROP FUNCTION IF EXISTS cs_match_v1; +DROP FUNCTION IF EXISTS cs_match_v1_v0; +DROP FUNCTION IF EXISTS cs_match_v1_v0_0; + +DROP FUNCTION IF EXISTS cs_unique_v1; +DROP FUNCTION IF EXISTS cs_unique_v1_v0; +DROP FUNCTION IF EXISTS cs_unique_v1_v0_0; + +DROP FUNCTION IF EXISTS cs_ore_64_8_v1; +DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0; +DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0_0; + +DROP FUNCTION IF EXISTS _cs_text_to_ore_64_8_v1_term_v1_0; + +DROP FUNCTION IF EXISTS cs_check_encrypted_v1; + +DROP DOMAIN IF EXISTS cs_match_index_v1; +DROP DOMAIN IF EXISTS cs_unique_index_v1; + +CREATE DOMAIN cs_match_index_v1 AS smallint[]; +CREATE DOMAIN cs_unique_index_v1 AS text; +CREATE DOMAIN cs_ste_vec_index_v1 AS text[]; + +-- cs_encrypted_v1 is a column type and cannot be dropped if in use +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_encrypted_v1') THEN + CREATE DOMAIN cs_encrypted_v1 AS JSONB; + END IF; +END +$$; + +DROP FUNCTION IF EXISTS _cs_encrypted_check_kind(jsonb); +CREATE FUNCTION _cs_encrypted_check_kind(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN (val->>'k' = 'ct' AND val ? 'c') AND NOT val ? 'p'; +END; + +CREATE FUNCTION cs_check_encrypted_v1(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN ( + -- version and source are required + val ?& array['v'] AND + + -- table and column + val->'i' ?& array['t', 'c'] AND + + -- plaintext or ciphertext for kind + _cs_encrypted_check_kind(val) + ); +END; + + +-- drop and reset the check constraint +ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check; + +ALTER DOMAIN cs_encrypted_v1 + ADD CONSTRAINT cs_encrypted_v1_check CHECK ( + cs_check_encrypted_v1(VALUE) +); + +CREATE OR REPLACE FUNCTION cs_ciphertext_v1_v0_0(col jsonb) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN col->>'c'; +END; + +CREATE OR REPLACE FUNCTION cs_ciphertext_v1_v0(col jsonb) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ciphertext_v1_v0_0(col); +END; + +CREATE OR REPLACE FUNCTION cs_ciphertext_v1(col jsonb) + RETURNS text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ciphertext_v1_v0_0(col); +END; + +-- extracts match index from an emcrypted column +CREATE OR REPLACE FUNCTION cs_match_v1_v0_0(col jsonb) + RETURNS cs_match_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT ARRAY(SELECT jsonb_array_elements(col->'m'))::cs_match_index_v1; +END; + +CREATE OR REPLACE FUNCTION cs_match_v1_v0(col jsonb) + RETURNS cs_match_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_match_v1_v0_0(col); +END; + +CREATE OR REPLACE FUNCTION cs_match_v1(col jsonb) + RETURNS cs_match_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_match_v1_v0_0(col); +END; + +-- extracts unique index from an encrypted column +CREATE OR REPLACE FUNCTION cs_unique_v1_v0_0(col jsonb) + RETURNS cs_unique_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN col->>'u'; +END; + +CREATE OR REPLACE FUNCTION cs_unique_v1_v0(col jsonb) + RETURNS cs_unique_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_unique_v1_v0_0(col); +END; + +CREATE OR REPLACE FUNCTION cs_unique_v1(col jsonb) + RETURNS cs_unique_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_unique_v1_v0_0(col); +END; + +-- extracts json containment index from an encrypted column +CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0_0(col jsonb) + RETURNS cs_ste_vec_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT ARRAY(SELECT jsonb_array_elements(col->'sv'))::cs_ste_vec_index_v1; +END; + +CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0(col jsonb) + RETURNS cs_ste_vec_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ste_vec_v1_v0_0(col); +END; + +CREATE OR REPLACE FUNCTION cs_ste_vec_v1(col jsonb) + RETURNS cs_ste_vec_index_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ste_vec_v1_v0_0(col); +END; + +-- casts text to ore_64_8_v1_term (bytea) +CREATE FUNCTION _cs_text_to_ore_64_8_v1_term_v1_0(t text) + RETURNS ore_64_8_v1_term + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN t::bytea; +END; + +-- cast to cleanup ore_64_8_v1 extraction +CREATE CAST (text AS ore_64_8_v1_term) + WITH FUNCTION _cs_text_to_ore_64_8_v1_term_v1_0(text) AS IMPLICIT; + +-- extracts ore index from an encrypted column +CREATE FUNCTION cs_ore_64_8_v1_v0_0(val jsonb) + RETURNS ore_64_8_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT (val->>'o')::ore_64_8_v1; +END; + +CREATE FUNCTION cs_ore_64_8_v1_v0(col jsonb) + RETURNS ore_64_8_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ore_64_8_v1_v0_0(col); +END; + +CREATE FUNCTION cs_ore_64_8_v1(col jsonb) + RETURNS ore_64_8_v1 + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN cs_ore_64_8_v1_v0_0(col); +END; +-- +-- Configuration Schema +-- +-- Defines core config state and storage types +-- Creates the cs_configuration_v1 table with constraint and unique indexes +-- +-- + + +-- +-- cs_configuration_data_v1 is a jsonb column that stores the actuak configuration +-- +-- For some reason CREATE DFOMAIN and CREATE TYPE do not support IF NOT EXISTS +-- Types cannot be dropped if used by a table, and we never drop the configuration table +-- DOMAIN constraints are added separately and not tied to DOMAIN creation +-- +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_configuration_data_v1') THEN + CREATE DOMAIN cs_configuration_data_v1 AS JSONB; + END IF; + END +$$; + +-- +-- cs_configuration_state_v1 is an ENUM that defines the valid configuration states +-- +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_configuration_state_v1') THEN + CREATE TYPE cs_configuration_state_v1 AS ENUM ('active', 'inactive', 'encrypting', 'pending'); + END IF; + END +$$; + +-- +-- _cs_check_config_indexes returns true if the table configuration only includes valid index types +-- +-- Used by the cs_configuration_data_v1_check constraint +-- +-- Function types cannot be changed after creation so we always DROP & CREATE for flexibility +-- +DROP FUNCTION IF EXISTS _cs_config_check_indexes(text, text); + +CREATE FUNCTION _cs_config_check_indexes(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_object_keys(jsonb_path_query(val, '$.tables.*.*.indexes')) = ANY('{match, ore, unique, json}'); +END; + + +CREATE FUNCTION _cs_config_check_cast(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}'); +END; + + +-- +-- Drop and reset the check constraint +-- +ALTER DOMAIN cs_configuration_data_v1 DROP CONSTRAINT IF EXISTS cs_configuration_data_v1_check; + +ALTER DOMAIN cs_configuration_data_v1 + ADD CONSTRAINT cs_configuration_data_v1_check CHECK ( + VALUE ?& array['v', 'tables'] AND + VALUE->'tables' <> '{}'::jsonb AND + _cs_config_check_cast(VALUE) AND + _cs_config_check_indexes(VALUE) +); + + +-- +-- CREATE the cs_configuration_v1 TABLE +-- +CREATE TABLE IF NOT EXISTS cs_configuration_v1 +( + id bigint GENERATED ALWAYS AS IDENTITY, + state cs_configuration_state_v1 NOT NULL DEFAULT 'pending', + data cs_configuration_data_v1, + created_at timestamptz not null default current_timestamp, + PRIMARY KEY(id) +); + +-- +-- Define partial indexes to ensure that there is only one active, pending and encrypting config at a time +-- +CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_active ON cs_configuration_v1 (state) WHERE state = 'active'; +CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_pending ON cs_configuration_v1 (state) WHERE state = 'pending'; +CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_encrypting ON cs_configuration_v1 (state) WHERE state = 'encrypting'; +-- +-- Configuration functions +-- +-- + + +-- DROP and CREATE functions +-- Function types cannot be changed after creation so we DROP for flexibility + +DROP FUNCTION IF EXISTS cs_add_column_v1(text, text); +DROP FUNCTION IF EXISTS cs_remove_column_v1(text, text); +DROP FUNCTION IF EXISTS cs_add_index_v1(text, text, text, jsonb); +DROP FUNCTION IF EXISTS cs_remove_index_v1(text, text, text); +DROP FUNCTION IF EXISTS cs_modify_index_v1(text, text, text, jsonb); + +DROP FUNCTION IF EXISTS cs_encrypt_v1(); +DROP FUNCTION IF EXISTS cs_activate_v1(); +DROP FUNCTION IF EXISTS cs_discard_v1(); + +DROP FUNCTION IF EXISTS cs_refresh_encrypt_config(); + +DROP FUNCTION IF EXISTS _cs_config_default(); +DROP FUNCTION IF EXISTS _cs_config_match_1_default(); + +DROP FUNCTION IF EXISTS _cs_config_add_table(text, json); +DROP FUNCTION IF EXISTS _cs_config_add_column(text, text, json); +DROP FUNCTION IF EXISTS _cs_config_add_cast(text, text, text, json); +DROP FUNCTION IF EXISTS _cs_config_add_index(text, text, text, json, json); + + +CREATE FUNCTION _cs_config_default(config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + IF config IS NULL THEN + SELECT jsonb_build_object('v', 1, 'tables', jsonb_build_object()) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION _cs_config_add_table(table_name text, config jsonb) + RETURNS jsonb + -- IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + tbl jsonb; + BEGIN + IF NOT config #> array['tables'] ? table_name THEN + SELECT jsonb_build_object(table_name, jsonb_build_object()) into tbl; + SELECT jsonb_set(config, array['tables'], tbl) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist +CREATE FUNCTION _cs_config_add_column(table_name text, column_name text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + DECLARE + col jsonb; + BEGIN + IF NOT config #> array['tables', table_name] ? column_name THEN + SELECT jsonb_build_object('indexes', jsonb_build_object()) into col; + SELECT jsonb_set(config, array['tables', table_name, column_name], col) INTO config; + END IF; + RETURN config; + END; +$$ LANGUAGE plpgsql; + +-- Set the cast +CREATE FUNCTION _cs_config_add_cast(table_name text, column_name text, cast_as text, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_set(config, array['tables', table_name, column_name, 'cast_as'], to_jsonb(cast_as)) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- Add the column if it doesn't exist +CREATE FUNCTION _cs_config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) + RETURNS jsonb + IMMUTABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT jsonb_insert(config, array['tables', table_name, column_name, 'indexes', index_name], opts) INTO config; + RETURN config; + END; +$$ LANGUAGE plpgsql; + + +-- +-- Default options for match_1 index +-- +CREATE FUNCTION _cs_config_match_1_default() + RETURNS jsonb +LANGUAGE sql STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_build_object( + 'k', 6, + 'm', 2048, + 'include_original', true, + 'tokenizer', json_build_object('kind', 'ngram', 'token_length', 3), + 'token_filters', json_build_array(json_build_object('kind', 'downcase'))); +END; + +-- +-- +-- +CREATE FUNCTION cs_add_index_v1(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') + RETURNS jsonb +AS $$ + DECLARE + o jsonb; + _config jsonb; + BEGIN + + -- set the active config + SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- if index exists + IF _config #> array['tables', table_name, column_name, 'indexes'] ? index_name THEN + RAISE EXCEPTION '% index exists for column: % %', index_name, table_name, column_name; + END IF; + + IF NOT cast_as = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}') THEN + RAISE EXCEPTION '% is not a valid cast type', cast_as; + END IF; + + -- set default config + SELECT _cs_config_default(_config) INTO _config; + + SELECT _cs_config_add_table(table_name, _config) INTO _config; + + SELECT _cs_config_add_column(table_name, column_name, _config) INTO _config; + + SELECT _cs_config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + + -- set default options for index if opts empty + IF index_name = 'match' AND opts = '{}' THEN + SELECT _cs_config_match_1_default() INTO opts; + END IF; + + SELECT _cs_config_add_index(table_name, column_name, index_name, opts, _config) INTO _config; + + -- create a new pending record if we don't have one + INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO UPDATE + SET data = _config; + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_remove_index_v1(table_name text, column_name text, index_name text) + RETURNS jsonb +AS $$ + DECLARE + _config jsonb; + BEGIN + + -- set the active config + SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- if no config + IF _config IS NULL THEN + RAISE EXCEPTION 'No active or pending configuration exists'; + END IF; + + -- if the table doesn't exist + IF NOT _config #> array['tables'] ? table_name THEN + RAISE EXCEPTION 'No configuration exists for table: %', table_name; + END IF; + + -- if the index does not exist + -- IF NOT _config->key ? index_name THEN + IF NOT _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'No % index exists for column: % %', index_name, table_name, column_name; + END IF; + + -- create a new pending record if we don't have one + INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO NOTHING; + + -- remove the index + SELECT _config #- array['tables', table_name, column_name, 'indexes', index_name] INTO _config; + + -- if column is now empty, remove the column + IF _config #> array['tables', table_name, column_name, 'indexes'] = '{}' THEN + SELECT _config #- array['tables', table_name, column_name] INTO _config; + END IF; + + -- if table is now empty, remove the table + IF _config #> array['tables', table_name] = '{}' THEN + SELECT _config #- array['tables', table_name] INTO _config; + END IF; + + -- if config empty delete + -- or update the config + IF _config #> array['tables'] = '{}' THEN + DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + ELSE + UPDATE cs_configuration_v1 SET data = _config WHERE state = 'pending'; + END IF; + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_modify_index_v1(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') + RETURNS jsonb +AS $$ + BEGIN + PERFORM cs_remove_index_v1(table_name, column_name, index_name); + RETURN cs_add_index_v1(table_name, column_name, index_name, cast_as, opts); + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_encrypt_v1() + RETURNS boolean +AS $$ + BEGIN + -- IF NOT cs_ready_for_encryption_v1() THEN + -- RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; + -- END IF; + + IF NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN + RAISE EXCEPTION 'No pending configuration exists to encrypt'; + END IF; + + UPDATE cs_configuration_v1 SET state = 'encrypting' WHERE state = 'pending'; + RETURN true; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION cs_activate_v1() + RETURNS boolean +AS $$ + BEGIN + + IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting') THEN + UPDATE cs_configuration_v1 SET state = 'inactive' WHERE state = 'active'; + UPDATE cs_configuration_v1 SET state = 'active' WHERE state = 'encrypting'; + RETURN true; + ELSE + RAISE EXCEPTION 'No encrypting configuration exists to activate'; + END IF; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION cs_discard_v1() + RETURNS boolean +AS $$ + BEGIN + IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN + DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + RETURN true; + ELSE + RAISE EXCEPTION 'No pending configuration exists to discard'; + END IF; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_add_column_v1(table_name text, column_name text) + RETURNS jsonb +AS $$ + DECLARE + key text; + _config jsonb; + BEGIN + -- set the active config + SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- set default config + SELECT _cs_config_default(_config) INTO _config; + + -- if index exists + IF _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'Config exists for column: % %', table_name, column_name; + END IF; + + SELECT _cs_config_add_table(table_name, _config) INTO _config; + + SELECT _cs_config_add_column(table_name, column_name, _config) INTO _config; + + -- create a new pending record if we don't have one + INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO UPDATE + SET data = _config; + + -- exeunt + RETURN _config; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_remove_column_v1(table_name text, column_name text) + RETURNS jsonb +AS $$ + DECLARE + key text; + _config jsonb; + BEGIN + -- set the active config + SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + + -- if no config + IF _config IS NULL THEN + RAISE EXCEPTION 'No active or pending configuration exists'; + END IF; + + -- if the table doesn't exist + IF NOT _config #> array['tables'] ? table_name THEN + RAISE EXCEPTION 'No configuration exists for table: %', table_name; + END IF; + + -- if the column does not exist + IF NOT _config #> array['tables', table_name] ? column_name THEN + RAISE EXCEPTION 'No configuration exists for column: % %', table_name, column_name; + END IF; + + -- create a new pending record if we don't have one + INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + ON CONFLICT (state) + WHERE state = 'pending' + DO NOTHING; + + -- remove the column + SELECT _config #- array['tables', table_name, column_name] INTO _config; + + -- if table is now empty, remove the table + IF _config #> array['tables', table_name] = '{}' THEN + SELECT _config #- array['tables', table_name] INTO _config; + END IF; + + -- if config empty delete + -- or update the config + IF _config #> array['tables'] = '{}' THEN + DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + ELSE + UPDATE cs_configuration_v1 SET data = _config WHERE state = 'pending'; + END IF; + + -- exeunt + RETURN _config; + + END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION cs_refresh_encrypt_config() + RETURNS void +LANGUAGE sql STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN NULL; +END; + +-- DROP and CREATE functions +-- Function types cannot be changed after creation so we DROP for flexibility +DROP FUNCTION IF EXISTS cs_select_pending_columns_v1; +DROP FUNCTION IF EXISTS cs_select_target_columns_v1; +DROP FUNCTION IF EXISTS cs_count_encrypted_with_active_config_v1; +DROP FUNCTION IF EXISTS cs_create_encrypted_columns_v1(); +DROP FUNCTION IF EXISTS cs_rename_encrypted_columns_v1(); + +DROP FUNCTION IF EXISTS _cs_diff_config_v1; +DROP FUNCTION IF EXISTS _cs_table_from_config_key; +DROP FUNCTION IF EXISTS _cs_column_from_config_key; + + +-- Return the diff of two configurations +-- Returns the set of keys in a that have different values to b +-- The json comparison is on object values held by the key +CREATE OR REPLACE FUNCTION _cs_diff_config_v1(a JSONB, b JSONB) + RETURNS TABLE(table_name TEXT, column_name TEXT) +IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN QUERY + WITH table_keys AS ( + SELECT jsonb_object_keys(a->'tables') AS key + UNION + SELECT jsonb_object_keys(b->'tables') AS key + ), + column_keys AS ( + SELECT tk.key AS table_key, jsonb_object_keys(a->'tables'->tk.key) AS column_key + FROM table_keys tk + UNION + SELECT tk.key AS table_key, jsonb_object_keys(b->'tables'->tk.key) AS column_key + FROM table_keys tk + ) + SELECT + ck.table_key AS table_name, + ck.column_key AS column_name + FROM + column_keys ck + WHERE + (a->'tables'->ck.table_key->ck.column_key IS DISTINCT FROM b->'tables'->ck.table_key->ck.column_key); + END; +$$ LANGUAGE plpgsql; + + +-- Returns the set of columns with pending configuration changes +-- Compares the columns in pending configuration that do not match the active config +CREATE FUNCTION cs_select_pending_columns_v1() + RETURNS TABLE(table_name TEXT, column_name TEXT) +AS $$ + DECLARE + active JSONB; + pending JSONB; + config_id BIGINT; + BEGIN + SELECT data INTO active FROM cs_configuration_v1 WHERE state = 'active'; + + -- set default config + IF active IS NULL THEN + active := '{}'; + END IF; + + SELECT id, data INTO config_id, pending FROM cs_configuration_v1 WHERE state = 'pending'; + + -- set default config + IF config_id IS NULL THEN + RAISE EXCEPTION 'No pending configuration exists to encrypt'; + END IF; + + RETURN QUERY + SELECT d.table_name, d.column_name FROM _cs_diff_config_v1(active, pending) as d; + END; +$$ LANGUAGE plpgsql; + +-- +-- Returns the target columns with pending configuration +-- +-- A `pending` column may be either a plaintext variant or cs_encrypted_v1. +-- A `target` column is always of type cs_encrypted_v1 +-- +-- On initial encryption from plaintext the target column will be `{column_name}_encrypted ` +-- OR NULL if the column does not exist +-- +CREATE FUNCTION cs_select_target_columns_v1() + RETURNS TABLE(table_name TEXT, column_name TEXT, target_column TEXT) + STABLE STRICT PARALLEL SAFE +AS $$ + SELECT + c.table_name, + c.column_name, + s.column_name as target_column + FROM + cs_select_pending_columns_v1() c + LEFT JOIN information_schema.columns s ON + s.table_name = c.table_name AND + (s.column_name = c.table_name OR s.column_name = c.column_name || '_encrypted') AND + s.domain_name = 'cs_encrypted_v1'; +$$ LANGUAGE sql; + + +-- +-- Returns true if all pending columns have a target (encrypted) column +CREATE FUNCTION cs_ready_for_encryption_v1() + RETURNS BOOLEAN + STABLE STRICT PARALLEL SAFE +AS $$ + SELECT EXISTS ( + SELECT * + FROM cs_select_target_columns_v1() AS c + WHERE c.target_column IS NOT NULL); +$$ LANGUAGE sql; + + +-- +-- Creates cs_encrypted_v1 columns for any plaintext columns with pending configuration +-- The new column name is `{column_name}_encrypted` +-- +-- Executes the ALTER TABLE statement +-- `ALTER TABLE {target_table} ADD COLUMN {column_name}_encrypted cs_encrypted_v1;` +-- +CREATE FUNCTION cs_create_encrypted_columns_v1() + RETURNS TABLE(table_name TEXT, column_name TEXT) +AS $$ + BEGIN + FOR table_name, column_name IN + SELECT c.table_name, (c.column_name || '_encrypted') FROM cs_select_target_columns_v1() AS c WHERE c.target_column IS NULL + LOOP + EXECUTE format('ALTER TABLE %I ADD column %I cs_encrypted_v1', table_name, column_name); + RETURN NEXT; + END LOOP; + END; +$$ LANGUAGE plpgsql; + + +-- +-- Renames plaintext and cs_encrypted_v1 columns created for the initial encryption. +-- The source plaintext column is renamed to `{column_name}_plaintext` +-- The target encrypted column is renamed from `{column_name}_encrypted` to `{column_name}` +-- +-- Executes the ALTER TABLE statements +-- `ALTER TABLE {target_table} RENAME COLUMN {column_name} TO {column_name}_plaintext; +-- `ALTER TABLE {target_table} RENAME COLUMN {column_name}_encrypted TO {column_name};` +-- +CREATE FUNCTION cs_rename_encrypted_columns_v1() + RETURNS TABLE(table_name TEXT, column_name TEXT, target_column TEXT) +AS $$ + BEGIN + FOR table_name, column_name, target_column IN + SELECT * FROM cs_select_target_columns_v1() as c WHERE c.target_column = c.column_name || '_encrypted' + LOOP + EXECUTE format('ALTER TABLE %I RENAME %I TO %I;', table_name, column_name, column_name || '_plaintext'); + EXECUTE format('ALTER TABLE %I RENAME %I TO %I;', table_name, target_column, column_name); + RETURN NEXT; + END LOOP; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION cs_count_encrypted_with_active_config_v1(table_name TEXT, column_name TEXT) + RETURNS BIGINT +AS $$ +DECLARE + result BIGINT; +BEGIN + EXECUTE format( + 'SELECT COUNT(%I) FROM %s t WHERE %I->>%L = (SELECT id::TEXT FROM cs_configuration_v1 WHERE state = %L)', + column_name, table_name, column_name, 'v', 'active' + ) + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; + diff --git a/go/xorm/dataset.yml b/go/xorm/dataset.yml new file mode 100644 index 00000000..f0488839 --- /dev/null +++ b/go/xorm/dataset.yml @@ -0,0 +1 @@ +tables: [] diff --git a/go/xorm/docker-compose.yml b/go/xorm/docker-compose.yml new file mode 100644 index 00000000..907ad2b6 --- /dev/null +++ b/go/xorm/docker-compose.yml @@ -0,0 +1,10 @@ +services: + postgres: + image: postgres:16.2-bookworm + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - ${PGPORT:-5432}:5432 + volumes: + - ./init-db:/docker-entrypoint-initdb.d diff --git a/go/xorm/example_queries.go b/go/xorm/example_queries.go new file mode 100644 index 00000000..18e04d8c --- /dev/null +++ b/go/xorm/example_queries.go @@ -0,0 +1,390 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + + "xorm.io/xorm" +) + +// Query on where clause on unecrypted column +func WhereQuery(engine *xorm.Engine) { + // Insert + fmt.Println("") + fmt.Println("") + fmt.Println("Query with where clause on unencrypted field") + fmt.Println("") + fmt.Println("") + + // serializedEmail := serialize("test@test.com", "examples", "encrypted_text") + + // serializedJsonb := serialize(generateJsonbData("birds and spiders", "fountain", "tree"), "examples", "encrypted_jsonb") + + newExample := Example{NonEncryptedField: "sydney", EncryptedInt: 23, EncryptedText: "test@test.com", EncryptedJsonb: generateJsonbData("birds and spiders", "fountain", "tree")} + + _, err := engine.Insert(&newExample) + if err != nil { + log.Fatalf("Could not insert new example: %v", err) + } + fmt.Println("Example inserted:", newExample) + fmt.Println("") + fmt.Println("") + + // Query + var example Example + text := "sydney" + + has, err := engine.Where("non_encrypted_field = ?", text).Get(&example) + if err != nil { + log.Fatalf("Could not retrieve example: %v", err) + } + if has { + fmt.Println("Example retrieved:", example) + fmt.Println("") + fmt.Println("") + + } else { + fmt.Println("Example not found") + } +} + +// Match query on encrypted column long string +func MatchQueryLongString(engine *xorm.Engine) { + fmt.Println("Match query on sentence") + fmt.Println("") + var example Example + + newExample := Example{NonEncryptedField: "sydney", EncryptedText: "this is a long string", EncryptedJsonb: generateJsonbData("bird", "fountain", "tree")} + + _, err := engine.Insert(&newExample) + if err != nil { + log.Fatalf("Could not insert new example: %v", err) + } + fmt.Printf("Example one inserted: %+v\n", newExample) + + serializedStringQuery := serialize("this", "examples", "encrypted_text") + query, err := json.Marshal(serializedStringQuery) + + if err != nil { + log.Fatalf("Error marshaling encrypted_text: %v", err) + } + + has, err := engine.Where("cs_match_v1(encrypted_text) @> cs_match_v1(?)", query).Get(&example) + if err != nil { + log.Fatalf("Could not retrieve example: %v", err) + } + if has { + fmt.Println("Example match query retrieved:", example) + fmt.Println("") + fmt.Println("") + } else { + fmt.Println("Example not found") + } +} + +// Match equery on text +func MatchQueryEmail(engine *xorm.Engine) { + fmt.Println("Match query on email") + fmt.Println("") + var ExampleTwo Example + + newExampleTwo := Example{NonEncryptedField: "sydney", EncryptedText: "somename@gmail.com", EncryptedJsonb: generateJsonbData("bird", "fountain", "tree")} + + _, errTwo := engine.Insert(&newExampleTwo) + if errTwo != nil { + log.Fatalf("Could not insert new example: %v", errTwo) + } + fmt.Printf("Example two inserted!: %+v\n", newExampleTwo) + + serializedEmailQuery := serialize("some", "examples", "encrypted_text") + query, err := json.Marshal(serializedEmailQuery) + + if err != nil { + log.Fatalf("Error marshaling encrypted_text: %v", err) + } + + has, err := engine.Where("cs_match_v1(encrypted_text) @> cs_match_v1(?)", query).Get(&ExampleTwo) + if err != nil { + log.Fatalf("Could not retrieve exampleTwo: %v", err) + } + if has { + fmt.Println("Example match query retrieved:", ExampleTwo) + fmt.Println("") + fmt.Println("") + } else { + fmt.Println("Example two not found") + } +} + +func JsonbQuerySimple(engine *xorm.Engine) { + fmt.Println("Query on jsonb field") + fmt.Println("") + + var example Example + + // Insert 2 examples + newExample := Example{NonEncryptedField: "sydney", EncryptedText: "this entry should be returned", EncryptedJsonb: generateJsonbData("first", "second", "third")} + newExampleTwo := Example{NonEncryptedField: "melbourne", EncryptedText: "a completely different string!", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errTwo := engine.Insert(&newExample) + if errTwo != nil { + log.Fatalf("Could not insert jsonb example: %v", errTwo) + } + fmt.Printf("Example jsonb inserted!: %+v\n", newExample) + + _, errThree := engine.Insert(&newExampleTwo) + if errThree != nil { + log.Fatalf("Could not insert jsonb example two: %v", errThree) + } + fmt.Printf("Example two jsonb inserted!: %+v\n", newExample) + + // create a query + query := map[string]any{ + "top": map[string]any{ + "nested": []any{"first"}, + }, + } + serializedJsonbQuery := serialize(query, "examples", "encrypted_jsonb") + + jsonQueryData, err := json.Marshal(serializedJsonbQuery) + if err != nil { + log.Fatalf("Could not insert jsonb example two: %v", err) + } + + has, err := engine.Where("cs_ste_vec_v1(encrypted_jsonb) @> cs_ste_vec_v1(?)", jsonQueryData).Get(&example) + if err != nil { + log.Fatalf("Could not retrieve jsonb example: %v", err) + } + if has { + fmt.Println("Example jsonb query retrieved:", example) + fmt.Println("") + fmt.Println("") + } else { + fmt.Println("Example two not found") + } + +} + +func JsonbQueryDeepNested(engine *xorm.Engine) { + fmt.Println("Query on deep nested jsonb field") + fmt.Println("") + var example Example + + // Insert 2 examples + + // Json with some nesting + nestedJson := map[string]any{ + "key_one": map[string]any{ + "nested_one": []any{"hello"}, + "nested_two": map[string]any{ + "nested_three": "world", + }, + }, + } + + newExample := Example{NonEncryptedField: "sydney", EncryptedText: "this entry should be returned for deep nested query", EncryptedJsonb: nestedJson} + newExampleTwo := Example{NonEncryptedField: "melbourne", EncryptedText: "the quick brown fox etc", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errTwo := engine.Insert(&newExample) + if errTwo != nil { + log.Fatalf("Could not insert jsonb example: %v", errTwo) + } + fmt.Printf("Example jsonb inserted!: %+v\n", newExample) + + _, errThree := engine.Insert(&newExampleTwo) + if errThree != nil { + log.Fatalf("Could not insert jsonb example two: %v", errThree) + } + fmt.Printf("Example two jsonb inserted!: %+v\n", newExample) + + query := map[string]any{ + "key_one": map[string]any{ + "nested_two": map[string]any{ + "nested_three": "world", + }, + }, + } + + serializedJsonbQuery := serialize(query, "examples", "encrypted_jsonb") + + jsonQueryData, err := json.Marshal(serializedJsonbQuery) + if err != nil { + log.Fatalf("Could not insert jsonb example two: %v", err) + } + + has, err := engine.Where("cs_ste_vec_v1(encrypted_jsonb) @> cs_ste_vec_v1(?)", jsonQueryData).Get(&example) + if err != nil { + log.Fatalf("Could not retrieve jsonb example: %v", err) + } + if has { + fmt.Println("Example jsonb query retrieved:", example) + fmt.Println("") + fmt.Println("") + } else { + fmt.Println("Example not found") + } + +} + +func OreStringRangeQuery(engine *xorm.Engine) { + fmt.Println("Ore String query") + fmt.Println("") + + example1 := Example{NonEncryptedField: "expected result", EncryptedText: "whale", EncryptedJsonb: generateJsonbData("test_one", "test_two", "test_three")} + example2 := Example{NonEncryptedField: "", EncryptedText: "apple", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errExample1 := engine.Insert(&example1) + if errExample1 != nil { + log.Fatalf("Could not insert example: %v", errExample1) + } + _, errExample2 := engine.Insert(&example2) + if errExample2 != nil { + log.Fatalf("Could not insert example: %v", errExample2) + } + fmt.Println("Examples inserted!") + + // Query + serializedOreStringQuery := serialize("tree", "examples", "encrypted_text") + jsonQueryData, _ := json.Marshal(serializedOreStringQuery) + + var example Example + + has, queryErr := engine.Where("cs_ore_64_8_v1(encrypted_text) > cs_ore_64_8_v1(?)", jsonQueryData).Get(&example) + if queryErr != nil { + log.Fatalf("Could not retrieve ore example: %v", queryErr) + } + if has { + fmt.Println("Example ore range query retrieved:", example) + fmt.Println("") + fmt.Println("") + } else { + fmt.Println("Example not found") + } +} + +func OreIntRangeQuery(engine *xorm.Engine) { + fmt.Println("Ore Int query") + fmt.Println("") + + example1 := Example{NonEncryptedField: "expected ore in range query", EncryptedInt: 42, EncryptedText: "some string", EncryptedJsonb: generateJsonbData("test_one", "test_two", "test_three")} + example2 := Example{NonEncryptedField: "", EncryptedInt: 23, EncryptedText: "another string", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errExample1 := engine.Insert(&example1) + if errExample1 != nil { + log.Fatalf("Could not insert example: %v", errExample1) + } + _, errExample2 := engine.Insert(&example2) + if errExample2 != nil { + log.Fatalf("Could not insert example: %v", errExample2) + } + fmt.Println("Examples inserted!", example1) + fmt.Println("Examples inserted!", example2) + + serializedOreIntQuery := serialize(32, "examples", "encrypted_int") + query, _ := json.Marshal(serializedOreIntQuery) + + // Query + + // var example Example + var allExamples []Example + queryErr := engine.Where("cs_ore_64_8_v1(encrypted_int) > cs_ore_64_8_v1(?)", query).Find(&allExamples) + // has, queryErr := engine.Where("cs_ore_64_8_v1(encrypted_int) > cs_ore_64_8_v1(?)", query).Find(&allExamples) + if queryErr != nil { + log.Fatalf("Could not retrieve ore example: %v", queryErr) + } + + fmt.Println("Example ore range query retrieved:", allExamples) + fmt.Println("") + fmt.Println("") +} + +func OreBoolQuery(engine *xorm.Engine) { + fmt.Println("Ore bool query") + fmt.Println("") + + example1 := Example{EncryptedBool: false, NonEncryptedField: "", EncryptedInt: 23, EncryptedText: "test_one", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + example2 := Example{EncryptedBool: true, NonEncryptedField: "expected result ore bool query", EncryptedInt: 23, EncryptedText: "test_two", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + example3 := Example{EncryptedBool: false, NonEncryptedField: "", EncryptedInt: 23, EncryptedText: "test_three", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errExample1 := engine.Insert(&example1) + if errExample1 != nil { + log.Fatalf("Could not insert example: %v", errExample1) + } + _, errExample2 := engine.Insert(&example2) + if errExample2 != nil { + log.Fatalf("Could not insert example: %v", errExample2) + } + _, errExample3 := engine.Insert(&example3) + if errExample3 != nil { + log.Fatalf("Could not insert example: %v", errExample3) + } + fmt.Println("Example1 inserted!", example1) + fmt.Println("Example2 inserted!", example2) + fmt.Println("Example3 inserted!", example3) + + // Query + serializedOreBoolQuery := serialize(false, "examples", "encrypted_bool") + query, _ := json.Marshal(serializedOreBoolQuery) + + // var example Example + var allExamples []Example + queryErr := engine.Where("cs_ore_64_8_v1(encrypted_bool) > cs_ore_64_8_v1(?)", query).Find(&allExamples) + if queryErr != nil { + log.Fatalf("Could not retrieve ore example: %v", queryErr) + } + + fmt.Println("Example ore range query retrieved:", allExamples) + fmt.Println("") + fmt.Println("") +} + +func UniqueStringQuery(engine *xorm.Engine) { + fmt.Println("Unique string query") + fmt.Println("") + + example1 := Example{EncryptedBool: false, NonEncryptedField: "", EncryptedInt: 23, EncryptedText: "test one", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + example2 := Example{EncryptedBool: true, NonEncryptedField: "expected result unique string query", EncryptedInt: 23, EncryptedText: "test two", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + example3 := Example{EncryptedBool: false, NonEncryptedField: "", EncryptedInt: 23, EncryptedText: "test three", EncryptedJsonb: generateJsonbData("blah", "boo", "bah")} + + _, errExample1 := engine.Insert(&example1) + if errExample1 != nil { + log.Fatalf("Could not insert example: %v", errExample1) + } + _, errExample2 := engine.Insert(&example2) + if errExample2 != nil { + log.Fatalf("Could not insert example: %v", errExample2) + } + _, errExample3 := engine.Insert(&example3) + if errExample3 != nil { + log.Fatalf("Could not insert example: %v", errExample3) + } + fmt.Println("Example1 inserted!", example1) + fmt.Println("Example2 inserted!", example2) + fmt.Println("Example3 inserted!", example3) + + var allExamples []Example + serializedStringQuery := serialize("test two", "examples", "encrypted_text") + + query, _ := json.Marshal(serializedStringQuery) + queryErr := engine.Where("cs_unique_v1(encrypted_text) = cs_unique_v1($1)", query).Find(&allExamples) + if queryErr != nil { + log.Fatalf("Could not retrieve unique example: %v", queryErr) + } + + fmt.Println("Example unique query retrieved:", allExamples) + fmt.Println("") + fmt.Println("") +} + +// For testing +func generateJsonbData(value_one string, value_two string, value_three string) map[string]any { + data := map[string]any{ + "top": map[string]any{ + "nested": []any{value_one, value_two}, + }, + "bottom": value_three, + } + + return data +} diff --git a/go/xorm/go-xorm-app b/go/xorm/go-xorm-app new file mode 100755 index 00000000..a5d75dc4 Binary files /dev/null and b/go/xorm/go-xorm-app differ diff --git a/go/xorm/go.mod b/go/xorm/go.mod new file mode 100644 index 00000000..79eb3679 --- /dev/null +++ b/go/xorm/go.mod @@ -0,0 +1,28 @@ +module go-xorm-app + +go 1.21.3 + +require github.com/jackc/pgx/v5 v5.7.1 + +require ( + github.com/jackc/pgio v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect +) + +require ( + github.com/goccy/go-json v0.8.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgtype v1.14.3 + github.com/jackc/pgx v3.6.2+incompatible + github.com/json-iterator/go v1.1.12 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/text v0.18.0 // indirect + xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978 // indirect + xorm.io/xorm v1.3.9 // indirect +) diff --git a/go/xorm/go.sum b/go/xorm/go.sum new file mode 100644 index 00000000..460a2b5b --- /dev/null +++ b/go/xorm/go.sum @@ -0,0 +1,239 @@ +gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.8.1 h1:4/Wjm0JIJaTDm8K1KcGrLHJoa8EsJ13YWeX+6Kfq6uI= +github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.3 h1:h6W9cPuHsRWQFTWUZMAKMgG5jSwQI0Zurzdvlx3Plus= +github.com/jackc/pgtype v1.14.3/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978 h1:bvLlAPW1ZMTWA32LuZMBEGHAUOcATZjzHcotf3SWweM= +xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/xorm v1.3.9 h1:TUovzS0ko+IQ1XnNLfs5dqK1cJl1H5uHpWbWqAQ04nU= +xorm.io/xorm v1.3.9/go.mod h1:LsCCffeeYp63ssk0pKumP6l96WZcHix7ChpurcLNuMw= diff --git a/go/xorm/init-db/create-db.sql b/go/xorm/init-db/create-db.sql new file mode 100644 index 00000000..2b4227fb --- /dev/null +++ b/go/xorm/init-db/create-db.sql @@ -0,0 +1 @@ +CREATE DATABASE gotest; \ No newline at end of file diff --git a/go/xorm/main.go b/go/xorm/main.go new file mode 100644 index 00000000..1f1975f5 --- /dev/null +++ b/go/xorm/main.go @@ -0,0 +1,318 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "strconv" + + _ "github.com/jackc/pgx/stdlib" // PostgreSQL driver + "xorm.io/xorm" + "xorm.io/xorm/names" +) + +// To setup postgres: +// Run: docker compose up +// To run examples +// Run: go run . + +// Create types for encrypted column +// +// EQL expects a json format that looks like this: +// '{"k":"pt","p":"a string representation of the plaintext that is being encrypted","i":{"t":"table","c":"column"},"v":1}' +// +// Creating a go struct to represent this shape to use for serialization. +type TableColumn struct { + T string `json:"t"` // This maps T to t in the json + C string `json:"c"` +} + +type EncryptedColumn struct { + K string `json:"k"` + P string `json:"p"` + I TableColumn `json:"i"` + V int `json:"v"` +} + +// Creating custom types for encrypted fields +// This way we can use the conversion interface to convert from the type used in the app, to +// the underlying jsonb shape that EQL expects +// '{"k":"pt","p":"a string representation of the plaintext that is being encrypted","i":{"t":"table","c":"column"},"v":1}' +// And then be able to convert back from the EQL jsonb shape back to the expected type to be used in the Go app. +type EncryptedText string +type EncryptedJsonb map[string]interface{} +type EncryptedInt int +type EncryptedBool bool + +// type EncryptedDate time.Time + +type Example struct { + Id int64 `xorm:"pk autoincr"` + NonEncryptedField string `xorm:"varchar(100)"` + EncryptedText EncryptedText `json:"encrypted_text" xorm:"jsonb 'encrypted_text'"` + EncryptedJsonb EncryptedJsonb `json:"encrypted_jsonb" xorm:"jsonb 'encrypted_jsonb'"` + EncryptedInt EncryptedInt `json:"encrypted_int" xorm:"jsonb 'encrypted_int'"` + EncryptedBool EncryptedBool `json:"encrypted_bool" xorm:"jsonb 'encrypted_bool'"` + // EncryptedDate EncryptedDate `json:"encrypted_date" xorm:"jsonb 'encrypted_date'"` +} + +func (Example) TableName() string { + return "examples" +} + +// Using the conversion interface so Encrypted* structs are converted to json when being inserted +// and converting back to the original type when retrieved. +// TODO: move these out to a separate module + +// encrypted text conversion +func (et *EncryptedText) FromDB(data []byte) error { + var jsonData map[string]interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return err + } + fmt.Println("json data", jsonData) + if pValue, ok := jsonData["p"].(string); ok { + *et = EncryptedText(pValue) + return nil + } + + return fmt.Errorf("invalid format: missing 'p' field in JSONB") +} + +func (et EncryptedText) ToDB() ([]byte, error) { + val := serialize(string(et), "examples", "encrypted_text") + return json.Marshal(val) +} + +// Encrypted jsonb conversion +func (ej *EncryptedJsonb) FromDB(data []byte) error { + var jsonData map[string]interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return err + } + + if pValue, ok := jsonData["p"].(string); ok { + var pData map[string]interface{} + if err := json.Unmarshal([]byte(pValue), &pData); err != nil { + return fmt.Errorf("error unmarshaling 'p' JSON string: %v", err) + } + + *ej = EncryptedJsonb(pData) + return nil + } + + return fmt.Errorf("invalid format: missing 'p' field in JSONB") +} + +func (ej EncryptedJsonb) ToDB() ([]byte, error) { + val := serialize(map[string]any(ej), "examples", "encrypted_jsonb") + return json.Marshal(val) +} + +// encrypted int conversion +func (ei *EncryptedInt) FromDB(data []byte) error { + var jsonData map[string]interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return err + } + + if pValue, ok := jsonData["p"].(string); ok { + parsedValue, err := strconv.Atoi(pValue) // Convert string to int + if err != nil { + return fmt.Errorf("invalid number format in 'p' field: %v", err) + } + *ei = EncryptedInt(parsedValue) + return nil + } + + return fmt.Errorf("invalid format: missing 'p' field") +} + +func (ei EncryptedInt) ToDB() ([]byte, error) { + val := serialize(int(ei), "examples", "encrypted_int") + return json.Marshal(val) +} + +// Encrypted bool converesion +func (eb *EncryptedBool) FromDB(data []byte) error { + var jsonData map[string]interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return err + } + + if pValue, ok := jsonData["p"].(string); ok { + parsedValue, err := strconv.ParseBool(pValue) + if err != nil { + return fmt.Errorf("invalid boolean format in 'p' field: %v", err) + } + *eb = EncryptedBool(parsedValue) + return nil + } + + return fmt.Errorf("invalid format: missing 'p' field or unsupported type") +} + +func (eb EncryptedBool) ToDB() ([]byte, error) { + val := serialize(bool(eb), "examples", "encrypted_bool") + + return json.Marshal(val) +} + +// Converts a plaintext value to a string and returns the EncryptedColumn struct to use to insert into the db. +func serialize(value any, table string, column string) EncryptedColumn { + str, err := convertToString(value) + if err != nil { + fmt.Println("Error:", err) + } + + data := EncryptedColumn{"pt", str, TableColumn{table, column}, 1} + + return data +} + +func convertToString(value any) (string, error) { + switch v := value.(type) { + case string: + return v, nil + case int: + return fmt.Sprintf("%d", v), nil + case float64: + return fmt.Sprintf("%f", v), nil + case map[string]any: + jsonData, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("error marshaling JSON: %v", err) + } + return string(jsonData), nil + case bool: + return strconv.FormatBool(v), nil + default: + return "", fmt.Errorf("unsupported type: %T", v) + } +} + +func setupDb() { + connStr := "user=postgres password=postgres port=5432 host=localhost dbname=postgres sslmode=disable" + engine, err := xorm.NewEngine("pgx", connStr) + + if err != nil { + log.Fatalf("Could not connect to the database: %v", err) + } + + var exists bool + _, err = engine.SQL("SELECT EXISTS (SELECT datname FROM pg_catalog.pg_database WHERE datname = 'gotest')").Get(&exists) + if err != nil { + log.Fatalf("Error: %v", err) + } + + if exists { + _, err = engine.Exec("DROP DATABASE gotest WITH (FORCE);") + if err != nil { + log.Fatalf("Could not drop database: %v", err) + } + fmt.Println("Database 'gotest' dropped successfully!") + + _, err = engine.Exec("CREATE DATABASE gotest;") + if err != nil { + log.Fatalf("Could not create database: %v", err) + } + fmt.Println("Database 'gotest' recreated!") + } else { + fmt.Println("Database 'gotest' doesn't exist. Creating...") + _, err = engine.Exec("CREATE DATABASE gotest;") + if err != nil { + log.Fatalf("Could not create database: %v", err) + } + fmt.Println("Database 'gotest' created successfully!") + } + + engine.Close() +} + +func createTable() { + connStr := "user=postgres password=postgres port=5432 host=localhost dbname=gotest sslmode=disable" + engine, err := xorm.NewEngine("pgx", connStr) + + if err != nil { + log.Fatalf("Could not connect to gotest database: %v", err) + } + + // need to map from struct to postgres snake case lowercase + engine.SetMapper(names.SnakeMapper{}) + engine.ShowSQL(true) + + err = engine.Sync(new(Example)) + if err != nil { + log.Fatalf("Could not create examples table: %v", err) + } + + fmt.Println("Examples table synced successfully!") + engine.Close() +} + +func installEql() { + connStr := "user=postgres password=postgres port=5432 host=localhost dbname=gotest sslmode=disable" + // Install Eql, custom types, indexes and constraints + // To install our custom types we need to use the database/sql package due to an issue + // with how xorm interprets `?`. + // https://gitea.com/xorm/xorm/issues/2483 + engine, err := sql.Open("pgx", connStr) + if err != nil { + log.Fatalf("Could not connect to the database: %v", err) + } + InstallEql(engine) + AddIndexes(engine) + AddConstraint(engine) + + // Refresh config with + engine.Exec("SELECT cs_refresh_encrypt_config();") + engine.Close() + +} + +func main() { + // Recreate gotest db on each run + setupDb() + + // Connect to go test directly and create table + createTable() + + // Install EQL and add config + installEql() + + // Connect to proxy + proxyConnStr := "user=postgres password=postgres port=6432 host=localhost dbname=gotest sslmode=disable" + proxyEngine, err := xorm.NewEngine("pgx", proxyConnStr) + + if err != nil { + log.Fatalf("Could not connect to the database: %v", err) + } + // Query on unencrypted column: where clause + WhereQuery(proxyEngine) + + // Query on encrypted columns. + // MATCH + MatchQueryLongString(proxyEngine) + MatchQueryEmail(proxyEngine) + + // JSONB data query + JsonbQuerySimple(proxyEngine) + JsonbQueryDeepNested(proxyEngine) + + // ORE + // String + OreStringRangeQuery(proxyEngine) + // Int + OreIntRangeQuery(proxyEngine) + // Bool + OreBoolQuery(proxyEngine) + // Date - todo + + // UNIQUE + // String + UniqueStringQuery(proxyEngine) + // Int + // Bool + +} diff --git a/go/xorm/migrations.go b/go/xorm/migrations.go new file mode 100644 index 00000000..c2c61095 --- /dev/null +++ b/go/xorm/migrations.go @@ -0,0 +1,416 @@ +package main + +import ( + "database/sql" + "log" + "os" +) + +func InstallEql(engine *sql.DB) { + log.Println("Start installing custom types and function") + installCsCustomTypes(engine) + + installDsl(engine) + +} + +// These are our base custom types and functions for ore search. +func installCsCustomTypes(engine *sql.DB) { + sqlBlock := ` + CREATE EXTENSION IF NOT EXISTS pgcrypto; + + CREATE TYPE ore_64_8_v1_term AS ( + bytes bytea + ); + + CREATE TYPE ore_64_8_v1 AS ( + terms ore_64_8_v1_term[] + ); + + CREATE OR REPLACE FUNCTION compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term) returns integer AS $$ + DECLARE + eq boolean := true; + unequal_block smallint := 0; + hash_key bytea; + target_block bytea; + + left_block_size CONSTANT smallint := 16; + right_block_size CONSTANT smallint := 32; + right_offset CONSTANT smallint := 136; -- 8 * 17 + + indicator smallint := 0; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + IF bit_length(a.bytes) != bit_length(b.bytes) THEN + RAISE EXCEPTION 'Ciphertexts are different lengths'; + END IF; + + FOR block IN 0..7 LOOP + -- Compare each PRP (byte from the first 8 bytes) and PRF block (8 byte + -- chunks of the rest of the value). + -- NOTE: + -- * Substr is ordinally indexed (hence 1 and not 0, and 9 and not 8). + -- * We are not worrying about timing attacks here; don't fret about + -- the OR or !=. + IF + substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1) + OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * BLOCK, left_block_size) + THEN + -- set the first unequal block we find + IF eq THEN + unequal_block := block; + END IF; + eq = false; + END IF; + END LOOP; + + IF eq THEN + RETURN 0::integer; + END IF; + + -- Hash key is the IV from the right CT of b + hash_key := substr(b.bytes, right_offset + 1, 16); + + -- first right block is at right offset + nonce_size (ordinally indexed) + target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size); + + indicator := ( + get_bit( + encrypt( + substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size), + hash_key, + 'aes-ecb' + ), + 0 + ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2; + + IF indicator = 1 THEN + RETURN 1::integer; + ELSE + RETURN -1::integer; + END IF; + END; + $$ LANGUAGE plpgsql; + + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) = 0 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) <> 0 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) = -1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) != 1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) = 1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1_term(a, b) != -1 + $$ LANGUAGE SQL; + + CREATE OPERATOR = ( + PROCEDURE="ore_64_8_v1_term_eq", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES + ); + + CREATE OPERATOR <> ( + PROCEDURE="ore_64_8_v1_term_neq", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES + ); + + CREATE OPERATOR > ( + PROCEDURE="ore_64_8_v1_term_gt", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel + ); + + CREATE OPERATOR < ( + PROCEDURE="ore_64_8_v1_term_lt", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel + ); + + CREATE OPERATOR <= ( + PROCEDURE="ore_64_8_v1_term_lte", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel + ); + + CREATE OPERATOR >= ( + PROCEDURE="ore_64_8_v1_term_gte", + LEFTARG=ore_64_8_v1_term, + RIGHTARG=ore_64_8_v1_term, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel + ); + + CREATE OPERATOR FAMILY ore_64_8_v1_term_btree_ops USING btree; + CREATE OPERATOR CLASS ore_64_8_v1_term_btree_ops DEFAULT FOR TYPE ore_64_8_v1_term USING btree FAMILY ore_64_8_v1_term_btree_ops AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term); + + -- Compare the "head" of each array and recurse if necessary + -- This function assumes an empty string is "less than" everything else + -- so if a is empty we return -1, if be is empty and a isn't, we return 1. + -- If both are empty we return 0. This cases probably isn't necessary as equality + -- doesn't always make sense but it's here for completeness. + -- If both are non-empty, we compare the first element. If they are equal + -- we need to consider the next block so we recurse, otherwise we return the comparison result. + CREATE OR REPLACE FUNCTION compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]) returns integer AS $$ + DECLARE + cmp_result integer; + BEGIN + IF (array_length(a, 1) = 0 OR a IS NULL) AND (array_length(b, 1) = 0 OR b IS NULL) THEN + RETURN 0; + END IF; + IF array_length(a, 1) = 0 OR a IS NULL THEN + RETURN -1; + END IF; + IF array_length(b, 1) = 0 OR a IS NULL THEN + RETURN 1; + END IF; + + cmp_result := compare_ore_64_8_v1_term(a[1], b[1]); + IF cmp_result = 0 THEN + -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array. + RETURN compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]); + END IF; + + RETURN cmp_result; + END + $$ LANGUAGE plpgsql; + + -- This function uses lexicographic comparison + CREATE OR REPLACE FUNCTION compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1) returns integer AS $$ + DECLARE + cmp_result integer; + BEGIN + -- Recursively compare blocks bailing as soon as we can make a decision + RETURN compare_ore_array(a.terms, b.terms); + END + $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) = 0 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) <> 0 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) = -1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) != 1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) = 1 + $$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v1(a, b) != -1 + $$ LANGUAGE SQL; + + CREATE OPERATOR = ( + PROCEDURE="ore_64_8_v1_eq", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES + ); + + CREATE OPERATOR <> ( + PROCEDURE="ore_64_8_v1_neq", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES + ); + + CREATE OPERATOR > ( + PROCEDURE="ore_64_8_v1_gt", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel + ); + + CREATE OPERATOR < ( + PROCEDURE="ore_64_8_v1_lt", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel + ); + + CREATE OPERATOR <= ( + PROCEDURE="ore_64_8_v1_lte", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel + ); + + CREATE OPERATOR >= ( + PROCEDURE="ore_64_8_v1_gte", + LEFTARG=ore_64_8_v1, + RIGHTARG=ore_64_8_v1, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel + ); + + CREATE OPERATOR FAMILY ore_64_8_v1_btree_ops USING btree; + CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING btree FAMILY ore_64_8_v1_btree_ops AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1); + + ` + _, err := engine.Exec(sqlBlock) + if err != nil { + log.Fatalf("Could not execute migration: %v", err) + } + + log.Println("Custom types installed!") +} + +// Installing EQL +func installDsl(engine *sql.DB) { + path := "./cipherstash-encrypt-dsl.sql" + sql, err := os.ReadFile(path) + if err != nil { + log.Fatalf("Failed to read SQL file: %v", err) + } + + _, err = engine.Exec(string(sql)) + if err != nil { + log.Fatalf("Failed to execute SQL query: %v", err) + } +} + +// We need to add constraints on any column that is encrypted. +// This checks that all the required json fields are present, and that the data has been encrypted +// by the proxy before inserting. +// If this is not the case, then we will receive a postgres constraint violation. +func AddConstraint(engine *sql.DB) { + sql := ` + ALTER TABLE examples ADD CONSTRAINT encrypted_text_encrypted_check + CHECK ( cs_check_encrypted_v1(encrypted_text) ); + + ALTER TABLE examples ADD CONSTRAINT encrypted_jsonb_encrypted_check + CHECK ( cs_check_encrypted_v1(encrypted_jsonb) ); + + ALTER TABLE examples ADD CONSTRAINT encrypted_int_encrypted_check + CHECK ( cs_check_encrypted_v1(encrypted_int) ); + + ALTER TABLE examples ADD CONSTRAINT encrypted_bool_encrypted_check + CHECK ( cs_check_encrypted_v1(encrypted_bool) ); + ` + + _, err := engine.Exec(sql) + if err != nil { + log.Fatalf("Failed to execute SQL query to add constraint: %v", err) + } + + log.Println("constraints added") +} + +// This adds the indexes for each column. +// This configuration is needed to determine how the data is encrypted and how you can query +func AddIndexes(engine *sql.DB) { + sql := ` + SELECT cs_add_index_v1('examples', 'encrypted_text', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); + SELECT cs_add_index_v1('examples', 'encrypted_text', 'match', 'text'); + SELECT cs_add_index_v1('examples', 'encrypted_text', 'ore', 'text'); + SELECT cs_add_index_v1('examples', 'encrypted_int', 'ore', 'int'); + SELECT cs_add_index_v1('examples', 'encrypted_bool', 'ore', 'boolean'); + SELECT cs_add_index_v1('examples', 'encrypted_jsonb', 'ste_vec', 'jsonb', '{"prefix": "some-prefix"}'); + + CREATE UNIQUE INDEX ON examples(cs_unique_v1(encrypted_text)); + CREATE INDEX ON examples USING GIN (cs_match_v1(encrypted_text)); + CREATE INDEX ON examples (cs_ore_64_8_v1(encrypted_text)); + CREATE INDEX ON examples USING GIN (cs_ste_vec_v1(encrypted_jsonb)); + CREATE INDEX ON examples (cs_ore_64_8_v1(encrypted_int)); + CREATE INDEX ON examples (cs_ore_64_8_v1(encrypted_bool)); + + SELECT cs_encrypt_v1(); + SELECT cs_activate_v1(); + ` + + _, err := engine.Exec(sql) + if err != nil { + log.Fatalf("Error adding indexes: %v", err) + } + + log.Println("config updated") +} diff --git a/go/xorm/run.sh b/go/xorm/run.sh new file mode 100644 index 00000000..d58e2c0d --- /dev/null +++ b/go/xorm/run.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# exits when a command fails +# exits when script tries to use undeclared variables +# exit status is the value of the last command to exit with a non-zero status, or zero if all commands exit successfully +set -euo pipefail + +if [[ -n "${DEBUG_RUN_SH:-}" ]]; then + set -x # trace what gets executed (useful for debugging) +fi + +if [ "${BASH_SOURCE[0]}" != "./run.sh" ]; then + echo "Please run this script as ./run.sh" + exit 1 +fi + +subproject_start_postgres() { + docker compose up -d +} + +subproject_stop_postgres() { + docker compose down +} + +subproject_start_proxy() { + cd ../../../packages/cipherstash-proxy + cargo run +} + +subcommand="${1:-test}" +case $subcommand in + start_postgres) + subproject_start_postgres + ;; + + stop_postgres) + subproject_stop_postgres + ;; + + install_eql) + subproject_install_eql + ;; + + start_proxy) + subproject_start_proxy + ;; + + *) + echo "Unknown run subcommand '$subcommand'" + exit 1 + ;; +esac