Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Local ClickHouse cluster configuration #33

Merged
merged 11 commits into from
Aug 10, 2022
Merged

Local ClickHouse cluster configuration #33

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
367f5d5
Local ClickHouse cluster configuration
slvrtrn Aug 9, 2022
88a5c80
Log base http adapter commands
slvrtrn Aug 10, 2022
9d20262
Log base http adapter commands
slvrtrn Aug 10, 2022
d3179bd
Fix GHA
slvrtrn Aug 10, 2022
8f23c3e
Fix GHA
slvrtrn Aug 10, 2022
79f8a5a
Fix GHA
slvrtrn Aug 10, 2022
83564d5
Merge remote-tracking branch 'origin/clickhouse-cloud-ci' into local-…
slvrtrn Aug 10, 2022
dc29eec
Revert GHA
slvrtrn Aug 10, 2022
97db905
Fix `command` and related tests, use more flexible test env, rename c…
slvrtrn Aug 10, 2022
e7bef46
Remove duplicate settings
slvrtrn Aug 10, 2022
7e0306a
Lowercase test env variable
slvrtrn Aug 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 85 additions & 0 deletions .docker/clickhouse/cluster/server1_config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
<?xml version="1.0"?>
<clickhouse>

<http_port>8123</http_port>
<interserver_http_port>9009</interserver_http_port>
<interserver_http_host>clickhouse1</interserver_http_host>

<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>default</default_database>

<mark_cache_size>5368709120</mark_cache_size>

<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<access_control_path>/var/lib/clickhouse/access/</access_control_path>

<logger>
<level>debug</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<console>1</console>
</logger>

<remote_servers>
<test_cluster>
<shard>
<replica>
<host>clickhouse1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse2</host>
<port>9000</port>
</replica>
</shard>
</test_cluster>
</remote_servers>

<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>

<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
</coordination_settings>

<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse1</hostname>
<port>9000</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse2</hostname>
<port>9000</port>
</server>
</raft_configuration>
</keeper_server>

<zookeeper>
<node>
<host>clickhouse1</host>
<port>9181</port>
</node>
<node>
<host>clickhouse2</host>
<port>9181</port>
</node>
</zookeeper>

<distributed_ddl>
<path>/clickhouse/test_cluster/task_queue/ddl</path>
</distributed_ddl>

</clickhouse>
7 changes: 7 additions & 0 deletions .docker/clickhouse/cluster/server1_macros.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<clickhouse>
<macros>
<cluster>test_cluster</cluster>
<replica>clickhouse1</replica>
<shard>1</shard>
</macros>
</clickhouse>
85 changes: 85 additions & 0 deletions .docker/clickhouse/cluster/server2_config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
<?xml version="1.0"?>
<clickhouse>

<http_port>8123</http_port>
<interserver_http_port>9009</interserver_http_port>
<interserver_http_host>clickhouse2</interserver_http_host>

<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>default</default_database>

<mark_cache_size>5368709120</mark_cache_size>

<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<access_control_path>/var/lib/clickhouse/access/</access_control_path>

<logger>
<level>debug</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<console>1</console>
</logger>

<remote_servers>
<test_cluster>
<shard>
<replica>
<host>clickhouse1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse2</host>
<port>9000</port>
</replica>
</shard>
</test_cluster>
</remote_servers>

<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>2</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>

<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
</coordination_settings>

<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse1</hostname>
<port>9000</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse2</hostname>
<port>9000</port>
</server>
</raft_configuration>
</keeper_server>

<zookeeper>
<node>
<host>clickhouse1</host>
<port>9181</port>
</node>
<node>
<host>clickhouse2</host>
<port>9181</port>
</node>
</zookeeper>

<distributed_ddl>
<path>/clickhouse/test_cluster/task_queue/ddl</path>
</distributed_ddl>

</clickhouse>
7 changes: 7 additions & 0 deletions .docker/clickhouse/cluster/server2_macros.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<clickhouse>
<macros>
<cluster>test_cluster</cluster>
<replica>clickhouse2</replica>
<shard>1</shard>
</macros>
</clickhouse>
33 changes: 33 additions & 0 deletions .docker/clickhouse/cluster/users.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0"?>
<clickhouse>

<profiles>
<default>
<load_balancing>random</load_balancing>
</default>
</profiles>

<users>
<default>
<password></password>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
</default>
</users>

<quotas>
<default>
<interval>
<duration>3600</duration>
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</clickhouse>
11 changes: 11 additions & 0 deletions .docker/nginx/local.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
upstream clickhouse_cluster {
server clickhouse1:8123;
server clickhouse2:8123;
}

server {
listen 8123;
location / {
proxy_pass http://clickhouse_cluster;
}
}
2 changes: 1 addition & 1 deletion .github/workflows/run-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,6 @@ jobs:
CLICKHOUSE_CLOUD_HOST: ${{ secrets.CLICKHOUSE_CLOUD_HOST }}
CLICKHOUSE_CLOUD_USERNAME: ${{ secrets.CLICKHOUSE_CLOUD_USERNAME }}
CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.CLICKHOUSE_CLOUD_PASSWORD }}
CLICKHOUSE_CLOUD_ENABLED: 'true'
CLICKHOUSE_TEST_ENVIRONMENT: 'cloud'
run: |
npm test
90 changes: 68 additions & 22 deletions __tests__/integration/command.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,65 +1,71 @@
import { expect } from 'chai';
import { createClient, type ClickHouseClient } from '../../src';
import type { ResponseJSON } from '../../src/clickhouse_types';
import type { ResponseJSON } from '../../src';
import { type ClickHouseClient } from '../../src';
import {
createTestClient,
getClickHouseTestEnvironment,
TestEnv,
} from '../utils/client';
import { guid } from '../utils';

describe('command', () => {
let client: ClickHouseClient;
beforeEach(() => {
client = createTestClient();
});
afterEach(async () => {
await client.command({ query: 'DROP TABLE example' });
await client.close();
});

it('sends a command to execute', async () => {
client = createClient();
const ddl =
'CREATE TABLE example (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory';
const { ddl, tableName, engine } = getDDL();

await client.command({ query: ddl });
const commandResult = await client.command({
query: ddl,
format: 'TabSeparated',
});
await commandResult.text();

const result = await client.select({
query: `SELECT * from system.tables where name = 'example'`,
const selectResult = await client.select({
query: `SELECT * from system.tables where name = '${tableName}'`,
format: 'JSON',
});

const { data, rows } = await result.json<
const { data, rows } = await selectResult.json<
ResponseJSON<{ name: string; engine: string; create_table_query: string }>
>();

expect(rows).to.equal(1);
const table = data[0];
expect(table.name).equal('example');
expect(table.engine).equal('Memory');
expect(table.name).equal(tableName);
expect(table.engine).equal(engine);
expect(table.create_table_query).to.be.a('string');
});

it('does not swallow ClickHouse error', (done) => {
client = createClient();

const ddl =
'CREATE TABLE example (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory';

const { ddl, tableName } = getDDL();
Promise.resolve()
.then(() => client.command({ query: ddl }))
.then(() => client.command({ query: ddl }))
.catch((e: any) => {
expect(e.code).to.equal('57');
expect(e.type).to.equal('TABLE_ALREADY_EXISTS');
// TODO remove whitespace from end
expect(e.message).equal('Table default.example already exists. ');
expect(e.message).equal(`Table default.${tableName} already exists. `);
done();
});
});

it.skip('can specify a parameterized query', async () => {
client = createClient();
await client.command({
query:
'CREATE TABLE {table_name: String} (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory',
const commandResult = await client.command({
query: '',
query_params: {
table_name: 'example',
},
});
await commandResult.text();

// FIXME: use different DDL based on the TestEnv
const result = await client.select({
query: `SELECT * from system.tables where name = 'example'`,
format: 'JSON',
Expand All @@ -74,3 +80,43 @@ describe('command', () => {
expect(table.name).to.equal('example');
});
});

function getDDL(): {
ddl: string;
tableName: string;
engine: string;
} {
const env = getClickHouseTestEnvironment();
const tableName = `command_test_${guid()}`;
switch (env) {
// ENGINE can be omitted in the cloud statements:
// it will use ReplicatedMergeTree and will add ON CLUSTER as well
case TestEnv.Cloud: {
const ddl = `
CREATE TABLE ${tableName}
(id UInt64, name String, sku Array(UInt8), timestamp DateTime)
ORDER BY (id)
`;
return { ddl, tableName, engine: 'ReplicatedMergeTree' };
}
case TestEnv.LocalSingleNode: {
const ddl = `
CREATE TABLE ${tableName}
(id UInt64, name String, sku Array(UInt8), timestamp DateTime)
ENGINE = MergeTree()
ORDER BY (id)
`;
return { ddl, tableName, engine: 'MergeTree' };
}

case TestEnv.LocalCluster: {
const ddl = `
CREATE TABLE ${tableName} ON CLUSTER '{cluster}'
(id UInt64, name String, sku Array(UInt8), timestamp DateTime)
ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}')
ORDER BY (id)
`;
return { ddl, tableName, engine: 'ReplicatedMergeTree' };
}
}
}
Loading