From c9d1a21ee0f2086b35dca972db661647a98eb7e8 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 11:44:41 -0400 Subject: [PATCH 1/8] Update OSS C*, DSE versions and fix minor readme docs --- README.md | 12 ++++++------ gradle.properties | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 342fb007..811891ae 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,18 @@ The DataStax CDC for Apache Cassandra requires: * DataStax Change Agent for Apache Cassandra, which is an event producer deployed as a JVM agent on each Cassandra data node. -* Datastax Source Connector for Apache Pulsar, which is source connector deployed in your streaming platform. +* DataStax Source Connector for Apache Pulsar, which is source connector deployed in your streaming platform. -![Cassandra-source-connector](docs/modules/ROOT/assets/images/cassandra-source-connector.png) +![Cassandra-source-connector](docs/docs-src/core/modules/ROOT/assets/images/cassandra-source-connector.png) Supported streaming platform: * Apache Pulsar 2.8.1+ -* Datastax Luna Streaming 2.8.0.1.1.40+ +* DataStax Luna Streaming 2.8.0.1.1.40+ Supported Cassandra version: * Cassandra 3.11+ * Cassandra 4.0+ -* Datastax Enterprise Server 6.8.16+ +* [DataStax Enterprise (DSE)](https://www.datastax.com/products/datastax-enterprise) 6.8.16+ Note: Only Cassandra 4.0 and DSE 6.8.16+ support the near realtime CDC allowing to replicate data as soon as they are synced on disk. @@ -49,7 +49,7 @@ You can collect Cassandra/DSE and Pulsar metrics into Prometheus, and build a Gr * The mutation sent throughput from a Cassandra node * The pulsar events and data topic rate in -![CDC Dashboard](docs/modules/ROOT/assets/images/cdc-dashboard.png) +![CDC Dashboard](docs/docs-src/core/modules/ROOT/assets/images/cdc-dashboard.png) ## Limitations @@ -58,7 +58,7 @@ You can collect Cassandra/DSE and Pulsar metrics into Prometheus, and build a Gr * Does not manage TTLs * Does not support range deletes * Does not sync data available before starting the CDC agent. -* CQL column names must not match a Pulsar primitive type name (ex: INT32) +* CQL column names must not match a [Pulsar primitive type](https://pulsar.apache.org/docs/next/schema-understand/#primitive-type) name (ex: INT32) ## Supported data types diff --git a/gradle.properties b/gradle.properties index fc13f10f..e08066d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,9 +10,9 @@ releasesRepoUrl=https://repo.datastax.com/artifactory/datastax-public-releases-l avroVersion=1.10.2 lombokVersion=1.18.20 ossDriverVersion=4.11.0 -cassandra3Version=3.11.10 -cassandra4Version=4.0.4 -dse4Version=6.8.23 +cassandra3Version=3.11.14 +cassandra4Version=4.1.1 +dse4Version=6.8.33 pulsarGroup=org.apache.pulsar pulsarVersion=2.8.3 From ebf370f26c9943820cd0e8153a98d79c210e0bf6 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 12:25:19 -0400 Subject: [PATCH 2/8] Update CDC with DSE example with latest versioned artefacts --- .../core/modules/ROOT/pages/cdcExample.adoc | 46 ++++++------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/docs/docs-src/core/modules/ROOT/pages/cdcExample.adoc b/docs/docs-src/core/modules/ROOT/pages/cdcExample.adoc index ca8dd7b9..ddaed442 100644 --- a/docs/docs-src/core/modules/ROOT/pages/cdcExample.adoc +++ b/docs/docs-src/core/modules/ROOT/pages/cdcExample.adoc @@ -2,18 +2,18 @@ Capture schema changes in your C* tables and pass them to Apache Pulsar(R) with DataStax Change Data Capture (CDC). This doc will guide you through installing, configuring, and using CDC with C* or DSE in a VM-based deployment. -This installation requires: +This installation requires the following. Latest version artifacts are available here at [![GitHub release](https://img.shields.io/github/v/release/datastax/cdc-apache-cassandra.svg)](https://github.com/datastax/cdc-apache-cassandra/releases/latest): * C* or DSE environment -** https://downloads.datastax.com/#enterprise[DSE 6.8.21] +** https://downloads.datastax.com/#enterprise[DSE 6.8.16+] ** https://cassandra.apache.org/_/download.html[OSS C*] * CDC Agent -** https://github.com/datastax/cdc-apache-cassandra/releases/download/v1.0.5/agent-dse4-pulsar-1.0.5-all.jar[DSE] -** https://github.com/datastax/cdc-apache-cassandra/releases/download/v1.0.5/agent-c4-pulsar-1.0.5-all.jar[OSS C*] +** DSE - use `agent-dse4--all.jar` +** OSS C* - use `agent-c4--all.jar` * Pulsar -** https://github.com/datastax/cdc-apache-cassandra/releases/download/v1.0.5/agent-dse4-pulsar-1.0.5-all.jar[DataStax Luna Streaming 2.8.3_1.0.7 Core] +** DataStax Luna Streaming - use `agent-dse4--all.jar` * Pulsar C* source connector (CSC) -** https://github.com/datastax/cdc-apache-cassandra/releases/download/v1.0.5/pulsar-cassandra-source-1.0.5.nar[pulsar-cassandra-source-1.0.5.nar] +** Pulsar Cassandra Source NAR - use `pulsar-cassandra-source-.nar` == Installing and configuring @@ -27,7 +27,7 @@ bin/pulsar standalone + [NOTE] ==== -We recommend using the latest CDC agent version (at least version 1.04+) to support C* collection data types. +We recommend using the latest CDC agent version (at least version `1.0.4`+) to support C* collection data types. ==== . Install C*/DSE with your preferred https://docs.datastax.com/en/install/6.8/install/installWhichOne.html[installation method^]. @@ -45,16 +45,16 @@ export CDC_PULSAR_AUTH_PARAMS="file://" export CDC_TLS_TRUST_CERTS_FILE_PATH=”” # DSE CDC -JVM_OPTS="$JVM_OPTS -javaagent:/home/automaton/cdc104/agent-dse4-pulsar-1.0.5-all.jar" +JVM_OPTS="$JVM_OPTS -javaagent:/home/automaton/cdc104/agent-dse4--all.jar" ---- + -For CDC agent versions *after 1.03*, the CDC agent Pulsar connection parameters are provided as system environment variables (see *DSE CDC* in the example above). +For CDC agent versions *after 1.0.3*, the CDC agent Pulsar connection parameters are provided as system environment variables (see *DSE CDC* in the example above). + -For CDC agent versions *before 1.03*, the CDC agent Pulsar connection parameters are also provided as extra JVM options, as below: +For CDC agent versions *before 1.0.3*, the CDC agent Pulsar connection parameters are also provided as extra JVM options, as below: + [source,bash] ---- -export JVM_EXTRA_OPTS="-javaagent:/path/to/agent-c4-luna--all.jar=pulsarServiceUrl=pulsar://pulsar:6650" +export JVM_EXTRA_OPTS="-javaagent:/path/to/agent-c4--all.jar=pulsarServiceUrl=pulsar://pulsar:6650" ---- . Set the `cassandra.yaml` configuration: @@ -95,7 +95,7 @@ Key-value-avro:: ---- $ pulsar-admin source create \ --name \ - --archive /pathto/to/pulsar-cassandra-source-1.0.5.nar \ + --archive /path/to/pulsar-cassandra-source-.nar \ --tenant public \ --namespace default \ --destination-topic-name . \ @@ -119,7 +119,7 @@ Key-value-json:: ---- $ pulsar-admin source create \ --name \ - --archive /pathto/to/pulsar-cassandra-source-1.0.5.nar \ + --archive /path/to/pulsar-cassandra-source-.nar \ --tenant public \ --namespace default \ --destination-topic-name persistent://public/default/data-.
\ @@ -136,14 +136,14 @@ $ pulsar-admin source create \ ---- -- + -Json:: +JSON:: + -- [source,bash] ---- $ pulsar-admin source create \ --name \ - --archive /pathto/to/pulsar-cassandra-source-1.0.5.nar \ + --archive /path/to/pulsar-cassandra-source-.nar \ --tenant public \ --namespace default \ --destination-topic-name persistent://public/default/data-.
\ @@ -185,19 +185,3 @@ pulsar-client consume -s mysub -st auto_consume -n 0 persistent://public/default For more on monitoring your {cdc_cass} deployment, see xref:monitor.adoc[Monitor {cdc_cass}]. + For using CDC with Astra DB, see https://docs.datastax.com/en/astra-streaming/docs/astream-cdc.html[CDC for Astra DB]. - - - - - - - - - - - - - - - - From 1b5c3f6e9cb2465aed2c8044c4e20acc5ec44fdf Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 12:43:22 -0400 Subject: [PATCH 3/8] Update CI to leverage Luna Streaming 2.10_4.1 --- .github/workflows/backfill-ci.yaml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backfill-ci.yaml b/.github/workflows/backfill-ci.yaml index 87665f89..0bb946f5 100644 --- a/.github/workflows/backfill-ci.yaml +++ b/.github/workflows/backfill-ci.yaml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: jdk: ['11'] # TODO: Enable java 17 tests https://issues.apache.org/jira/browse/CASSANDRA-16895 - pulsarImage: ['datastax/lunastreaming:2.10_3.4', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] + pulsarImage: ['datastax/lunastreaming:2.10_4.1', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] cassandraFamily: ['c3', 'c4', 'dse4'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f907e55..226caa1b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: matrix: module: ['agent', 'agent-c3', 'agent-c4', 'agent-dse4', 'connector'] jdk: ['11', '17'] - pulsarImage: ['datastax/lunastreaming:2.10_3.4', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] + pulsarImage: ['datastax/lunastreaming:2.10_4.1', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] steps: - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} From ba056f8f42fdf2f04bd48e5e7e4b685d26e22db6 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 12:48:32 -0400 Subject: [PATCH 4/8] Downgrade OSS C* from 4.1.1 to 4.0.8 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e08066d4..4971931e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,7 @@ avroVersion=1.10.2 lombokVersion=1.18.20 ossDriverVersion=4.11.0 cassandra3Version=3.11.14 -cassandra4Version=4.1.1 +cassandra4Version=4.0.8 dse4Version=6.8.33 pulsarGroup=org.apache.pulsar From 5e82b757f35f731e95a8ff213b210c3407aab9f4 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 14:40:40 -0400 Subject: [PATCH 5/8] Revert "Downgrade OSS C* from 4.1.1 to 4.0.8" This reverts commit ba056f8f42fdf2f04bd48e5e7e4b685d26e22db6. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 4971931e..e08066d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,7 @@ avroVersion=1.10.2 lombokVersion=1.18.20 ossDriverVersion=4.11.0 cassandra3Version=3.11.14 -cassandra4Version=4.0.8 +cassandra4Version=4.1.1 dse4Version=6.8.33 pulsarGroup=org.apache.pulsar From 9551d687c34d10cacba1b94326fbc0ef045de3f6 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 5 Apr 2023 15:20:50 -0400 Subject: [PATCH 6/8] Downgrade OSS C* from 4.1.1 to 4.0.8 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 9d06231b..132d7b17 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,7 @@ avroVersion=1.10.2 lombokVersion=1.18.20 ossDriverVersion=4.15.0 cassandra3Version=3.11.14 -cassandra4Version=4.1.1 +cassandra4Version=4.0.8 dse4Version=6.8.33 pulsarGroup=org.apache.pulsar From 1cea53fc3e52801f3b388ad51de3d41ecd979c28 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 6 Apr 2023 13:25:02 -0400 Subject: [PATCH 7/8] Revert "Update CI to leverage Luna Streaming 2.10_4.1" This reverts commit 1b5c3f6e9cb2465aed2c8044c4e20acc5ec44fdf. --- .github/workflows/backfill-ci.yaml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backfill-ci.yaml b/.github/workflows/backfill-ci.yaml index bd22fb68..0fc77f7e 100644 --- a/.github/workflows/backfill-ci.yaml +++ b/.github/workflows/backfill-ci.yaml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: jdk: ['11'] # TODO: Enable java 17 tests https://issues.apache.org/jira/browse/CASSANDRA-16895 - pulsarImage: ['datastax/lunastreaming:2.10_4.1', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] + pulsarImage: ['datastax/lunastreaming:2.10_3.4', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] cassandraFamily: ['c3', 'c4', 'dse4'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ac2db52..a922430c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: matrix: module: ['agent', 'agent-c3', 'agent-c4', 'agent-dse4', 'connector'] jdk: ['11', '17'] - pulsarImage: ['datastax/lunastreaming:2.10_4.1', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] + pulsarImage: ['datastax/lunastreaming:2.10_3.4', 'apachepulsar/pulsar:2.10.3', 'apachepulsar/pulsar:2.11.0'] steps: - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} From 3f98183c23c4b31392bfbb66c650ac683be3ef19 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 6 Apr 2023 15:45:55 -0400 Subject: [PATCH 8/8] Downgrade OSS C* 3.x, 4.x and DSE versions per the maintaining team suggestion --- gradle.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index 132d7b17..f1bd4347 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,9 +10,9 @@ releasesRepoUrl=https://repo.datastax.com/artifactory/datastax-public-releases-l avroVersion=1.10.2 lombokVersion=1.18.20 ossDriverVersion=4.15.0 -cassandra3Version=3.11.14 -cassandra4Version=4.0.8 -dse4Version=6.8.33 +cassandra3Version=3.11.10 +cassandra4Version=4.0.4 +dse4Version=6.8.23 pulsarGroup=org.apache.pulsar pulsarVersion=2.8.3