diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index be159888fa5da..a77b29128e117 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -435,7 +435,7 @@ steps: - id: testdrive-alloydb label: "testdrive with AlloyDB" - depends_on: build-aarch64 + depends_on: build-x86_64 timeout_in_minutes: 60 parallelism: 2 inputs: [test/testdrive] diff --git a/ci/release-qualification/pipeline.template.yml b/ci/release-qualification/pipeline.template.yml index a337cfe38531b..4cd11c0fffb8c 100644 --- a/ci/release-qualification/pipeline.template.yml +++ b/ci/release-qualification/pipeline.template.yml @@ -279,22 +279,74 @@ steps: - group: "MySQL: other versions" key: mysql-versions steps: - - id: mysql-cdc-5_7 - label: "MySQL CDC w/ 5.7" - depends_on: build-x86_64 + - id: mysql-cdc-8_0 + label: "MySQL CDC w/ 8.0" + depends_on: build-aarch64 timeout_in_minutes: 60 agents: - # no matching manifest of MySQL 5.7.x for linux/arm64/v8 in the manifest list entries - # Increased memory usage following new source syntax - queue: hetzner-x86-64-12cpu-24gb + queue: hetzner-aarch64-4cpu-8gb plugins: - ./ci/plugins/mzcompose: composition: mysql-cdc - args: [ "--mysql-version=5.7.44" ] - skip: "Fails when restarted" + args: [ "--mysql-version=8.0.42" ] - - id: mysql-cdc-8_0 - label: "MySQL CDC w/ 8.0" + - id: mysql-cdc-8_1 + label: "MySQL CDC w/ 8.1" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=8.1.0" ] + + - id: mysql-cdc-8_2 + label: "MySQL CDC w/ 8.2" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=8.2.0" ] + + - id: mysql-cdc-8_3 + label: "MySQL CDC w/ 8.3" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=8.3.0" ] + + - id: mysql-cdc-8_4 + label: "MySQL CDC w/ 8.4" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=8.4.9" ] + + - id: mysql-cdc-9_0 + label: "MySQL CDC w/ 9.0" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.0.1" ] + + - id: mysql-cdc-9_1 + label: "MySQL CDC w/ 9.1" depends_on: build-aarch64 timeout_in_minutes: 60 agents: @@ -302,12 +354,77 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: mysql-cdc - args: [ "--mysql-version=8.0.40" ] - skip: "Doesn't support new RESET BINARY LOGS AND GTIDS syntax" + args: [ "--mysql-version=9.1.0" ] + + - id: mysql-cdc-9_2 + label: "MySQL CDC w/ 9.2" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.2.0" ] + + - id: mysql-cdc-9_3 + label: "MySQL CDC w/ 9.3" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.3.0" ] + + - id: mysql-cdc-9_4 + label: "MySQL CDC w/ 9.4" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.4.0" ] + + - id: mysql-cdc-9_5 + label: "MySQL CDC w/ 9.5" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.5.0" ] + + - id: mysql-cdc-9_6 + label: "MySQL CDC w/ 9.6" + depends_on: build-aarch64 + timeout_in_minutes: 60 + agents: + queue: hetzner-aarch64-4cpu-8gb + plugins: + - ./ci/plugins/mzcompose: + composition: mysql-cdc + args: [ "--mysql-version=9.6.0" ] - group: "Postgres: other versions" key: postgres-versions steps: + - id: pg-cdc-17 + label: "Postgres CDC w/ 17" + depends_on: build-aarch64 + timeout_in_minutes: 60 + inputs: [test/pg-cdc] + plugins: + - ./ci/plugins/mzcompose: + composition: pg-cdc + args: [ "--pg-version=17.10" ] + agents: + queue: hetzner-aarch64-4cpu-8gb - id: pg-cdc-16 label: "Postgres CDC w/ 16" depends_on: build-aarch64 @@ -316,7 +433,7 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: pg-cdc - args: [ "--pg-version=16.6" ] + args: [ "--pg-version=16.14" ] agents: queue: hetzner-aarch64-4cpu-8gb - id: pg-cdc-15 @@ -327,7 +444,7 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: pg-cdc - args: [ "--pg-version=15.10" ] + args: [ "--pg-version=15.18" ] agents: queue: hetzner-aarch64-4cpu-8gb - id: pg-cdc-14 @@ -338,7 +455,7 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: pg-cdc - args: [ "--pg-version=14.15" ] + args: [ "--pg-version=14.23" ] agents: queue: hetzner-aarch64-4cpu-8gb - id: pg-cdc-13 diff --git a/misc/python/materialize/cloudtest/k8s/mysql.py b/misc/python/materialize/cloudtest/k8s/mysql.py index 1bba0d3a36ab2..1745428da1bae 100644 --- a/misc/python/materialize/cloudtest/k8s/mysql.py +++ b/misc/python/materialize/cloudtest/k8s/mysql.py @@ -59,7 +59,7 @@ def __init__(self, namespace: str, apply_node_selectors: bool) -> None: ports = [V1ContainerPort(container_port=3306, name="sql")] container = V1Container( name="mysql", - image=self.image("mysql", tag="9.4.0", release_mode=True, org=None), + image=self.image("mysql", tag="9.7.0", release_mode=True, org=None), args=[ "--log-bin=mysql-bin", "--gtid_mode=ON", diff --git a/misc/python/materialize/mzcompose/services/mysql.py b/misc/python/materialize/mzcompose/services/mysql.py index b72a610dd9108..f0536a759bc84 100644 --- a/misc/python/materialize/mzcompose/services/mysql.py +++ b/misc/python/materialize/mzcompose/services/mysql.py @@ -37,7 +37,7 @@ def create_mysql_server_args( class MySql(Service): DEFAULT_ROOT_PASSWORD = "p@ssw0rd" - DEFAULT_VERSION = "9.5.0" + DEFAULT_VERSION = "9.7.0" DEFAULT_ADDITIONAL_ARGS = create_mysql_server_args(server_id="1", is_master=True) diff --git a/test/mysql-cdc-old-syntax/mzcompose.py b/test/mysql-cdc-old-syntax/mzcompose.py index ad24dca97ab25..785f54d7da17b 100644 --- a/test/mysql-cdc-old-syntax/mzcompose.py +++ b/test/mysql-cdc-old-syntax/mzcompose.py @@ -89,6 +89,13 @@ def get_targeted_mysql_version(parser: WorkflowArgumentParser) -> str: return args.mysql_version +def reset_binlog_stmt(mysql_version: str) -> str: + # RESET BINARY LOGS AND GTIDS was introduced in MySQL 8.2; + # RESET MASTER was removed in MySQL 8.4. + major_minor = tuple(int(p) for p in mysql_version.split(".")[:2]) + return "RESET MASTER" if major_minor < (8, 2) else "RESET BINARY LOGS AND GTIDS" + + def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: def process(name: str) -> None: if name in ("default", "migration"): @@ -160,6 +167,7 @@ def workflow_replica_connection(c: Composition, parser: WorkflowArgumentParser) c.up("materialized", "mysql", "mysql-replica") c.run_testdrive_files( f"--var=mysql-root-password={MySql.DEFAULT_ROOT_PASSWORD}", + f"--var=reset-binlog={reset_binlog_stmt(mysql_version)}", "override/10-replica-connection.td", ) diff --git a/test/mysql-cdc-old-syntax/override/10-replica-connection.td b/test/mysql-cdc-old-syntax/override/10-replica-connection.td index 27cac1ffb6e98..768c6cc89b100 100644 --- a/test/mysql-cdc-old-syntax/override/10-replica-connection.td +++ b/test/mysql-cdc-old-syntax/override/10-replica-connection.td @@ -22,8 +22,7 @@ $ mysql-connect name=mysql-replica url=mysql://root@mysql-replica password=${arg # state when we create our source. # $ mysql-execute name=mysql -DROP DATABASE IF EXISTS public; -RESET BINARY LOGS AND GTIDS; +DROP DATABASE IF EXISTS public; ${arg.reset-binlog}; CREATE DATABASE public; USE public; CREATE TABLE t1 (f1 INTEGER); diff --git a/test/mysql-cdc-old-syntax/toast-columns.td b/test/mysql-cdc-old-syntax/toast-columns.td index e98fdfbfb7a83..18f46b734e634 100644 --- a/test/mysql-cdc-old-syntax/toast-columns.td +++ b/test/mysql-cdc-old-syntax/toast-columns.td @@ -23,8 +23,9 @@ $ set-sql-timeout duration=1s $ mysql-connect name=mysql url=mysql://root@mysql password=${arg.mysql-root-password} -# Insert data pre-snapshot by generating 16kB of uncompressible data to force -# TOASTed storage by concatenating 1024 random MD5 hashes, each being 128bit +# Insert data pre-snapshot by generating 32kB of uncompressible data to force +# TOASTed storage by concatenating 512 random SHA-256 hashes, each being 256 bit. +# Avoid md5()/sha1() which were moved to the classic_hashing component in MySQL 9.6. $ mysql-execute name=mysql DROP DATABASE IF EXISTS public; CREATE DATABASE public; @@ -36,13 +37,13 @@ SET SESSION group_concat_max_len = 32768; # necessary because the limit is not respected by group_concat CREATE TABLE temp_rand (rand_value TEXT); -INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024; +INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512; -INSERT INTO t1 SELECT 1, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand; +INSERT INTO t1 SELECT 1, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand; DELETE FROM temp_rand; -INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024; -INSERT INTO t1 SELECT 2, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand; +INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512; +INSERT INTO t1 SELECT 2, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand; DROP TABLE temp_rand; diff --git a/test/mysql-cdc/mzcompose.py b/test/mysql-cdc/mzcompose.py index 0ebb9d96aa5dc..a8f7619fe3dd7 100644 --- a/test/mysql-cdc/mzcompose.py +++ b/test/mysql-cdc/mzcompose.py @@ -93,6 +93,13 @@ def get_targeted_mysql_version(parser: WorkflowArgumentParser) -> str: return args.mysql_version +def reset_binlog_stmt(mysql_version: str) -> str: + # RESET BINARY LOGS AND GTIDS was introduced in MySQL 8.2; + # RESET MASTER was removed in MySQL 8.4. + major_minor = tuple(int(p) for p in mysql_version.split(".")[:2]) + return "RESET MASTER" if major_minor < (8, 2) else "RESET BINARY LOGS AND GTIDS" + + def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: def process(name: str) -> None: if name in ("default", "large-scale"): @@ -164,6 +171,7 @@ def workflow_replica_connection(c: Composition, parser: WorkflowArgumentParser) c.up("materialized", "mysql", "mysql-replica") c.run_testdrive_files( f"--var=mysql-root-password={MySql.DEFAULT_ROOT_PASSWORD}", + f"--var=reset-binlog={reset_binlog_stmt(mysql_version)}", "override/10-replica-connection.td", ) diff --git a/test/mysql-cdc/override/10-replica-connection.td b/test/mysql-cdc/override/10-replica-connection.td index 08676402887cd..c8615bee649d4 100644 --- a/test/mysql-cdc/override/10-replica-connection.td +++ b/test/mysql-cdc/override/10-replica-connection.td @@ -22,8 +22,7 @@ $ mysql-connect name=mysql-replica url=mysql://root@mysql-replica password=${arg # state when we create our source. # $ mysql-execute name=mysql -DROP DATABASE IF EXISTS public; -RESET BINARY LOGS AND GTIDS; +DROP DATABASE IF EXISTS public; ${arg.reset-binlog}; CREATE DATABASE public; USE public; CREATE TABLE t1 (f1 INTEGER); diff --git a/test/mysql-cdc/toast-columns.td b/test/mysql-cdc/toast-columns.td index c8eab57bff81d..93e640a4f9f4d 100644 --- a/test/mysql-cdc/toast-columns.td +++ b/test/mysql-cdc/toast-columns.td @@ -23,8 +23,9 @@ $ set-sql-timeout duration=1s $ mysql-connect name=mysql url=mysql://root@mysql password=${arg.mysql-root-password} -# Insert data pre-snapshot by generating 16kB of uncompressible data to force -# TOASTed storage by concatenating 1024 random MD5 hashes, each being 128bit +# Insert data pre-snapshot by generating 32kB of uncompressible data to force +# TOASTed storage by concatenating 512 random SHA-256 hashes, each being 256 bit. +# Avoid md5()/sha1() which were moved to the classic_hashing component in MySQL 9.6. $ mysql-execute name=mysql DROP DATABASE IF EXISTS public; CREATE DATABASE public; @@ -36,13 +37,13 @@ SET SESSION group_concat_max_len = 32768; # necessary because the limit is not respected by group_concat CREATE TABLE temp_rand (rand_value TEXT); -INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024; +INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512; -INSERT INTO t1 SELECT 1, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand; +INSERT INTO t1 SELECT 1, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand; DELETE FROM temp_rand; -INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 1024; -INSERT INTO t1 SELECT 2, group_concat(md5(rand_value) SEPARATOR '') FROM temp_rand; +INSERT INTO temp_rand SELECT rand() FROM mysql.time_zone t1, mysql.time_zone t2 LIMIT 512; +INSERT INTO t1 SELECT 2, group_concat(sha2(rand_value, 256) SEPARATOR '') FROM temp_rand; DROP TABLE temp_rand; diff --git a/test/mysql/Dockerfile b/test/mysql/Dockerfile index 572d6b53d726a..7fa7d516a0f67 100644 --- a/test/mysql/Dockerfile +++ b/test/mysql/Dockerfile @@ -9,7 +9,7 @@ MZFROM test-certs AS certs -FROM mysql:9.5.0 AS seed +FROM mysql:9.7.0 AS seed ENV MYSQL_ROOT_HOST=% @@ -20,7 +20,7 @@ RUN chmod +x /usr/local/bin/seed-mysql-datadir.sh \ && chown -R mysql:mysql /var/lib/mysql-seed /var/lib/mysql-files \ && gosu mysql /usr/local/bin/seed-mysql-datadir.sh -FROM mysql:9.5.0 +FROM mysql:9.7.0 COPY --from=seed /var/lib/mysql-seed /var/lib/mysql-seed COPY mysql-entrypoint.sh /usr/local/bin/mysql-entrypoint.sh diff --git a/test/pg-cdc/pg-cdc.td b/test/pg-cdc/pg-cdc.td index 8d8b96d1e747d..dbd0e46c84a2e 100644 --- a/test/pg-cdc/pg-cdc.td +++ b/test/pg-cdc/pg-cdc.td @@ -38,6 +38,7 @@ CREATE SCHEMA public; DROP PUBLICATION IF EXISTS mz_source; CREATE PUBLICATION mz_source FOR ALL TABLES; +DO $do$ BEGIN IF current_setting('server_version_num')::int >= 180000 THEN EXECUTE 'ALTER PUBLICATION mz_source SET (publish_generated_columns = ''stored'')'; END IF; END $do$; CREATE TABLE pk_table (pk INTEGER PRIMARY KEY, f2 TEXT); INSERT INTO pk_table VALUES (1, 'one'); diff --git a/test/postgres/Dockerfile b/test/postgres/Dockerfile index 322de70c56e18..af29b72f890ec 100644 --- a/test/postgres/Dockerfile +++ b/test/postgres/Dockerfile @@ -9,11 +9,12 @@ MZFROM test-certs as certs -FROM postgres:17.7 +FROM postgres:18.4 ENV POSTGRES_PASSWORD=postgres +ENV PGDATA=/var/lib/postgresql/data -RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-17-cron eatmydata \ +RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-18-cron eatmydata \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/*