-
Notifications
You must be signed in to change notification settings - Fork 46
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #497 from Altinity/2.0.1
Release 2.0.1
- Loading branch information
Showing
30 changed files
with
943 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ### Supported ClickHouse Table Engine Types. | ||
|
|
||
| The sink connector supports the following ClickHouse Engine types: | ||
| - **ReplacingMergeTree**: See [Architecture] (doc/mutable_data.md) for more information. | ||
| - ReplacingMergeTree is a variant of MergeTree that allows for updates and deletes. | ||
| - It is the default engine type used with the sink connector when tables are auto created using | ||
| - `auto.create.tables` set to `true`. | ||
| - **ReplicatedReplacingMergeTree**: | ||
| - ReplicatedReplacingMergeTree is a variant of ReplicatedMergeTree that allows for updates and deletes. | ||
| - To enable this engine type, set the `"auto.create.tables.replicated` to `true`, sink connector will | ||
| create tables with this engine type. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,14 @@ | ||
| <clickhouse replace="true"> | ||
| <timezone>America/Chicago</timezone> | ||
| <zookeeper> | ||
| <node index="1"> | ||
| <host>zookeeper</host> | ||
| <port>2181</port> | ||
| </node> | ||
| <session_timeout_ms>15000</session_timeout_ms> | ||
| </zookeeper> | ||
| <macros> | ||
| <replica>clickhouse</replica> | ||
| <shard>02</shard> | ||
| </macros> | ||
| </clickhouse> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
124 changes: 124 additions & 0 deletions
124
...ight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresInitialDockerIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| package com.altinity.clickhouse.debezium.embedded; | ||
|
|
||
| import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; | ||
| import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; | ||
| import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; | ||
| import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; | ||
| import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; | ||
| import com.clickhouse.jdbc.ClickHouseConnection; | ||
| import org.junit.Assert; | ||
| import org.junit.jupiter.api.DisplayName; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.testcontainers.Testcontainers; | ||
| import org.testcontainers.clickhouse.ClickHouseContainer; | ||
| import org.testcontainers.containers.Network; | ||
| import org.testcontainers.containers.PostgreSQLContainer; | ||
| import org.testcontainers.junit.jupiter.Container; | ||
| import org.testcontainers.utility.DockerImageName; | ||
|
|
||
| import java.sql.ResultSet; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Properties; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.atomic.AtomicReference; | ||
|
|
||
| import static com.altinity.clickhouse.debezium.embedded.PostgresProperties.getDefaultProperties; | ||
|
|
||
| public class PostgresInitialDockerIT { | ||
|
|
||
| @Container | ||
| public static ClickHouseContainer clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") | ||
| .asCompatibleSubstituteFor("clickhouse")) | ||
| .withInitScript("init_clickhouse_it.sql") | ||
| .withUsername("ch_user") | ||
| .withPassword("password") | ||
| .withExposedPorts(8123); | ||
|
|
||
| public static DockerImageName myImage = DockerImageName.parse("debezium/postgres:15-alpine").asCompatibleSubstituteFor("postgres"); | ||
|
|
||
| @Container | ||
| public static PostgreSQLContainer postgreSQLContainer = (PostgreSQLContainer) new PostgreSQLContainer(myImage) | ||
| .withInitScript("init_postgres.sql") | ||
| .withDatabaseName("public") | ||
| .withUsername("root") | ||
| .withPassword("root") | ||
| .withExposedPorts(5432) | ||
| .withCommand("postgres -c wal_level=logical") | ||
| .withNetworkAliases("postgres").withAccessToHost(true); | ||
|
|
||
|
|
||
|
|
||
| public Properties getProperties() throws Exception { | ||
|
|
||
| Properties properties = getDefaultProperties(postgreSQLContainer, clickHouseContainer); | ||
| properties.put("plugin.name", "decoderbufs"); | ||
| properties.put("plugin.path", "/"); | ||
| properties.put("table.include.list", "public.tm"); | ||
| properties.put("slot.max.retries", "6"); | ||
| properties.put("slot.retry.delay.ms", "5000"); | ||
| properties.put("database.allowPublicKeyRetrieval", "true"); | ||
| properties.put("table.include.list", "public.tm,public.tm2"); | ||
|
|
||
| return properties; | ||
| } | ||
|
|
||
| @Test | ||
| @DisplayName("Integration Test - Validates PostgreSQL replication when the plugin is set to DecoderBufs") | ||
| public void testDecoderBufsPlugin() throws Exception { | ||
| Network network = Network.newNetwork(); | ||
|
|
||
| postgreSQLContainer.withNetwork(network).start(); | ||
| clickHouseContainer.withNetwork(network).start(); | ||
| Thread.sleep(10000); | ||
|
|
||
| Testcontainers.exposeHostPorts(postgreSQLContainer.getFirstMappedPort()); | ||
| AtomicReference<DebeziumChangeEventCapture> engine = new AtomicReference<>(); | ||
|
|
||
| ExecutorService executorService = Executors.newFixedThreadPool(1); | ||
| executorService.execute(() -> { | ||
| try { | ||
|
|
||
| engine.set(new DebeziumChangeEventCapture()); | ||
| engine.get().setup(getProperties(), new SourceRecordParserService(), | ||
| new MySQLDDLParserService(new ClickHouseSinkConnectorConfig(new HashMap<>())), false); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| }); | ||
|
|
||
| Thread.sleep(10000);// | ||
| Thread.sleep(50000); | ||
|
|
||
| String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), | ||
| "public"); | ||
| ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", | ||
| clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); | ||
|
|
||
| BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), | ||
| "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); | ||
| Map<String, String> tmColumns = writer.getColumnsDataTypesForTable("tm"); | ||
| Assert.assertTrue(tmColumns.size() == 22); | ||
| Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); | ||
| Assert.assertTrue(tmColumns.get("secid").equalsIgnoreCase("Nullable(UUID)")); | ||
| //Assert.assertTrue(tmColumns.get("am").equalsIgnoreCase("Nullable(Decimal(21,5))")); | ||
| Assert.assertTrue(tmColumns.get("created").equalsIgnoreCase("Nullable(DateTime64(6))")); | ||
|
|
||
|
|
||
| int tmCount = 0; | ||
| ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from tm").executeQuery(); | ||
| while(chRs.next()) { | ||
| tmCount = chRs.getInt(1); | ||
| } | ||
|
|
||
| Assert.assertTrue(tmCount == 2); | ||
|
|
||
| if(engine.get() != null) { | ||
| engine.get().stop(); | ||
| } | ||
| // Files.deleteIfExists(tmpFilePath); | ||
| executorService.shutdown(); | ||
|
|
||
| } | ||
| } |
Oops, something went wrong.