From 50112e7650a727a96a5fed491fd0a5e339f9fc11 Mon Sep 17 00:00:00 2001 From: Viktoria Date: Tue, 24 Jan 2023 12:40:36 +0300 Subject: [PATCH 1/4] Add to clickhouse README.md database creation --- database/clickhouse/README.md | 1 + .../migrations/003_create_database.down.sql | 10 +++ .../migrations/003_create_database.up.sql | 81 +++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 database/clickhouse/examples/migrations/003_create_database.down.sql create mode 100644 database/clickhouse/examples/migrations/003_create_database.up.sql diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md index 96ad79f1c..a0a6b7ca3 100644 --- a/database/clickhouse/README.md +++ b/database/clickhouse/README.md @@ -23,3 +23,4 @@ * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`: * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication). * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster. +* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `deafault` table, so you can't use `USE ` inside migration. In this case you can not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql)) \ No newline at end of file diff --git a/database/clickhouse/examples/migrations/003_create_database.down.sql b/database/clickhouse/examples/migrations/003_create_database.down.sql new file mode 100644 index 000000000..8e49cfbfa --- /dev/null +++ b/database/clickhouse/examples/migrations/003_create_database.down.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS driver_ratings ON CLUSTER cluster_1; +DROP TABLE IF EXISTS user_ratings ON CLUSTER cluster_1; +DROP TABLE IF EXISTS orders ON CLUSTER cluster_1; +DROP TABLE IF EXISTS driver_ratings_queue ON CLUSTER cluster_1; +DROP TABLE IF EXISTS user_ratings_queue ON CLUSTER cluster_1; +DROP TABLE IF EXISTS orders_queue ON CLUSTER cluster_1; +DROP VIEW IF EXISTS user_ratings_queue_mv ON CLUSTER cluster_1; +DROP VIEW IF EXISTS driver_ratings_queue_mv ON CLUSTER cluster_1; +DROP VIEW IF EXISTS orders_queue_mv ON CLUSTER cluster_1; +DROP DATABASE IF EXISTS analytics ON CLUSTER cluster_1; diff --git a/database/clickhouse/examples/migrations/003_create_database.up.sql b/database/clickhouse/examples/migrations/003_create_database.up.sql new file mode 100644 index 000000000..b49d66913 --- /dev/null +++ b/database/clickhouse/examples/migrations/003_create_database.up.sql @@ -0,0 +1,81 @@ +CREATE DATABASE IF NOT EXISTS analytics ON CLUSTER cluster_1; + +CREATE TABLE IF NOT EXISTS analytics.driver_ratings ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree +PARTITION BY driverID +ORDER BY (inserted_time); + +CREATE TABLE analytics.driver_ratings_queue ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'driver-ratings', + kafka_group_name = 'rating_readers', + kafka_format = 'Avro', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.driver_ratings AS +SELECT rate, userID, driverID, orderID +FROM analytics.driver_ratings_queue; + +CREATE TABLE IF NOT EXISTS analytics.user_ratings ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree + PARTITION BY userID + ORDER BY (inserted_time); + +CREATE TABLE analytics.user_ratings_queue ON CLUSTER cluster_1( + rate UInt8, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'user-ratings', + kafka_group_name = 'rating_readers', + kafka_format = 'JSON', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.user_ratings AS +SELECT rate, userID, driverID, orderID +FROM analytics.user_ratings_queue; + +CREATE TABLE IF NOT EXISTS analytics.orders ON CLUSTER cluster_1( + from_place String, + to_place String, + userID Int64, + driverID String, + orderID String, + inserted_time DateTime DEFAULT now() +) ENGINE = ReplicatedMergeTree + PARTITION BY driverID + ORDER BY (inserted_time); + +CREATE TABLE analytics.orders_queue ON CLUSTER cluster_1( + from_place String, + to_place String, + userID Int64, + driverID String, + orderID String +) ENGINE = Kafka +SETTINGS kafka_broker_list = 'broker:9092', + kafka_topic_list = 'orders', + kafka_group_name = 'order_readers', + kafka_format = 'Avro', + kafka_max_block_size = 1048576; + +CREATE MATERIALIZED VIEW analytics.orders_queue_mv ON CLUSTER cluster_1 TO orders AS +SELECT from_place, to_place, userID, driverID, orderID +FROM analytics.orders_queue; From 90a3ac4f870766aa363b6872998e2cec8ddaffbb Mon Sep 17 00:00:00 2001 From: Viktoria Date: Tue, 24 Jan 2023 13:22:17 +0300 Subject: [PATCH 2/4] Remove cluster adaptation for tables to pass tests --- .../migrations/003_create_database.down.sql | 20 +++++++------- .../migrations/003_create_database.up.sql | 26 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/database/clickhouse/examples/migrations/003_create_database.down.sql b/database/clickhouse/examples/migrations/003_create_database.down.sql index 8e49cfbfa..76d3cf32d 100644 --- a/database/clickhouse/examples/migrations/003_create_database.down.sql +++ b/database/clickhouse/examples/migrations/003_create_database.down.sql @@ -1,10 +1,10 @@ -DROP TABLE IF EXISTS driver_ratings ON CLUSTER cluster_1; -DROP TABLE IF EXISTS user_ratings ON CLUSTER cluster_1; -DROP TABLE IF EXISTS orders ON CLUSTER cluster_1; -DROP TABLE IF EXISTS driver_ratings_queue ON CLUSTER cluster_1; -DROP TABLE IF EXISTS user_ratings_queue ON CLUSTER cluster_1; -DROP TABLE IF EXISTS orders_queue ON CLUSTER cluster_1; -DROP VIEW IF EXISTS user_ratings_queue_mv ON CLUSTER cluster_1; -DROP VIEW IF EXISTS driver_ratings_queue_mv ON CLUSTER cluster_1; -DROP VIEW IF EXISTS orders_queue_mv ON CLUSTER cluster_1; -DROP DATABASE IF EXISTS analytics ON CLUSTER cluster_1; +DROP TABLE IF EXISTS driver_ratings; +DROP TABLE IF EXISTS user_ratings; +DROP TABLE IF EXISTS orders; +DROP TABLE IF EXISTS driver_ratings_queue; +DROP TABLE IF EXISTS user_ratings_queue; +DROP TABLE IF EXISTS orders_queue; +DROP VIEW IF EXISTS user_ratings_queue_mv; +DROP VIEW IF EXISTS driver_ratings_queue_mv; +DROP VIEW IF EXISTS orders_queue_mv; +DROP DATABASE IF EXISTS analytics; diff --git a/database/clickhouse/examples/migrations/003_create_database.up.sql b/database/clickhouse/examples/migrations/003_create_database.up.sql index b49d66913..de5038880 100644 --- a/database/clickhouse/examples/migrations/003_create_database.up.sql +++ b/database/clickhouse/examples/migrations/003_create_database.up.sql @@ -1,16 +1,16 @@ -CREATE DATABASE IF NOT EXISTS analytics ON CLUSTER cluster_1; +CREATE DATABASE IF NOT EXISTS analytics; -CREATE TABLE IF NOT EXISTS analytics.driver_ratings ON CLUSTER cluster_1( +CREATE TABLE IF NOT EXISTS analytics.driver_ratings( rate UInt8, userID Int64, driverID String, orderID String, inserted_time DateTime DEFAULT now() -) ENGINE = ReplicatedMergeTree +) ENGINE = MergeTree PARTITION BY driverID ORDER BY (inserted_time); -CREATE TABLE analytics.driver_ratings_queue ON CLUSTER cluster_1( +CREATE TABLE analytics.driver_ratings_queue( rate UInt8, userID Int64, driverID String, @@ -22,21 +22,21 @@ SETTINGS kafka_broker_list = 'broker:9092', kafka_format = 'Avro', kafka_max_block_size = 1048576; -CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.driver_ratings AS +CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv TO analytics.driver_ratings AS SELECT rate, userID, driverID, orderID FROM analytics.driver_ratings_queue; -CREATE TABLE IF NOT EXISTS analytics.user_ratings ON CLUSTER cluster_1( +CREATE TABLE IF NOT EXISTS analytics.user_ratings( rate UInt8, userID Int64, driverID String, orderID String, inserted_time DateTime DEFAULT now() -) ENGINE = ReplicatedMergeTree +) ENGINE = MergeTree PARTITION BY userID ORDER BY (inserted_time); -CREATE TABLE analytics.user_ratings_queue ON CLUSTER cluster_1( +CREATE TABLE analytics.user_ratings_queue( rate UInt8, userID Int64, driverID String, @@ -48,22 +48,22 @@ SETTINGS kafka_broker_list = 'broker:9092', kafka_format = 'JSON', kafka_max_block_size = 1048576; -CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv ON CLUSTER cluster_1 TO analytics.user_ratings AS +CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv TO analytics.user_ratings AS SELECT rate, userID, driverID, orderID FROM analytics.user_ratings_queue; -CREATE TABLE IF NOT EXISTS analytics.orders ON CLUSTER cluster_1( +CREATE TABLE IF NOT EXISTS analytics.orders( from_place String, to_place String, userID Int64, driverID String, orderID String, inserted_time DateTime DEFAULT now() -) ENGINE = ReplicatedMergeTree +) ENGINE = MergeTree PARTITION BY driverID ORDER BY (inserted_time); -CREATE TABLE analytics.orders_queue ON CLUSTER cluster_1( +CREATE TABLE analytics.orders_queue( from_place String, to_place String, userID Int64, @@ -76,6 +76,6 @@ SETTINGS kafka_broker_list = 'broker:9092', kafka_format = 'Avro', kafka_max_block_size = 1048576; -CREATE MATERIALIZED VIEW analytics.orders_queue_mv ON CLUSTER cluster_1 TO orders AS +CREATE MATERIALIZED VIEW analytics.orders_queue_mv TO orders AS SELECT from_place, to_place, userID, driverID, orderID FROM analytics.orders_queue; From 64755d0b20af559e5adf22d5f1ed45a121e7f5ad Mon Sep 17 00:00:00 2001 From: Viktoria Date: Tue, 24 Jan 2023 14:37:31 +0300 Subject: [PATCH 3/4] Update README.md --- database/clickhouse/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md index a0a6b7ca3..14de52931 100644 --- a/database/clickhouse/README.md +++ b/database/clickhouse/README.md @@ -23,4 +23,4 @@ * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`: * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication). * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster. -* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `deafault` table, so you can't use `USE ` inside migration. In this case you can not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql)) \ No newline at end of file +* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `deafault` table, so you can't use `USE ` inside migration. In this case you may not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql)) \ No newline at end of file From 3b02b182eee9d36f27c79989bce31e13b3d2fc01 Mon Sep 17 00:00:00 2001 From: Viktoria <69359798+no-name16@users.noreply.github.com> Date: Sun, 19 Mar 2023 07:44:50 +0100 Subject: [PATCH 4/4] Correct a spelling mistake --- database/clickhouse/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md index 14de52931..a3cea9298 100644 --- a/database/clickhouse/README.md +++ b/database/clickhouse/README.md @@ -23,4 +23,4 @@ * Clickhouse cluster mode is not officially supported, since it's not tested right now, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`: * When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified. See the docs regarding [replicated table engines](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#table_engines-replication). * When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster. -* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `deafault` table, so you can't use `USE ` inside migration. In this case you may not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql)) \ No newline at end of file +* If you want to create database inside the migration, you should know, that table which will manage migrations `schema-migrations table` will be in `default` table, so you can't use `USE ` inside migration. In this case you may not specify the database in the connection string (example you can find [here](examples/migrations/003_create_database.up.sql))