From 9d25b00c76c94392d9420bfda1cf6ec3f9e78af4 Mon Sep 17 00:00:00 2001 From: Akihiro Okuno Date: Tue, 25 Nov 2025 18:30:25 +0900 Subject: [PATCH] Support ScalarDB Analytics 3.17 --- scalardb-analytics-sample/README.md | 10 ++++++++-- .../config/data-sources/postgres.json | 16 ++++++---------- .../config/data-sources/scalardb.json | 6 +----- .../config/scalardb-analytics-server.properties | 2 +- .../config/spark-defaults.conf | 2 +- scalardb-analytics-sample/docker-compose.yml | 7 +++---- .../docker/Dockerfile.spark | 11 ++++++----- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/scalardb-analytics-sample/README.md b/scalardb-analytics-sample/README.md index 5b7176f..cae543c 100644 --- a/scalardb-analytics-sample/README.md +++ b/scalardb-analytics-sample/README.md @@ -20,10 +20,16 @@ docker compose run --rm scalardb-analytics-cli catalog create --catalog sample_c ```bash # Register ScalarDB data source -docker compose run --rm scalardb-analytics-cli data-source register --data-source-json /config/data-sources/scalardb.json +docker compose run --rm scalardb-analytics-cli data-source register \ + --catalog sample_catalog \ + --data-source scalardb \ + --provider-file /config/data-sources/scalardb.json # Register PostgreSQL data source -docker compose run --rm scalardb-analytics-cli data-source register --data-source-json /config/data-sources/postgres.json +docker compose run --rm scalardb-analytics-cli data-source register \ + --catalog sample_catalog \ + --data-source postgres \ + --provider-file /config/data-sources/postgres.json ``` ### 4. Run Spark SQL diff --git a/scalardb-analytics-sample/config/data-sources/postgres.json b/scalardb-analytics-sample/config/data-sources/postgres.json index 6979c53..64815e8 100644 --- a/scalardb-analytics-sample/config/data-sources/postgres.json +++ b/scalardb-analytics-sample/config/data-sources/postgres.json @@ -1,12 +1,8 @@ { - "catalog": "sample_catalog", - "name": "postgres", - "type": "postgres", - "provider": { - "host": "postgres", - "port": 5432, - "username": "postgres", - "password": "postgres", - "database": "sampledb" - } + "type": "postgresql", + "host": "postgres", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "sampledb" } diff --git a/scalardb-analytics-sample/config/data-sources/scalardb.json b/scalardb-analytics-sample/config/data-sources/scalardb.json index b2422c4..292a093 100644 --- a/scalardb-analytics-sample/config/data-sources/scalardb.json +++ b/scalardb-analytics-sample/config/data-sources/scalardb.json @@ -1,8 +1,4 @@ { - "catalog": "sample_catalog", - "name": "scalardb", "type": "scalardb", - "provider": { - "configPath": "/etc/scalardb.properties" - } + "configs": "${file:/etc/scalardb.properties}" } diff --git a/scalardb-analytics-sample/config/scalardb-analytics-server.properties b/scalardb-analytics-sample/config/scalardb-analytics-server.properties index 3ad3a44..fbe7701 100644 --- a/scalardb-analytics-sample/config/scalardb-analytics-server.properties +++ b/scalardb-analytics-sample/config/scalardb-analytics-server.properties @@ -5,7 +5,7 @@ scalar.db.analytics.server.catalog.port=11051 scalar.db.analytics.server.metering.port=11052 # Server database configuration (for catalog metadata) -scalar.db.analytics.server.db.url=jdbc:postgresql://analytics-catalog-postgres:5432/catalogdb +scalar.db.analytics.server.db.contact_points=jdbc:postgresql://analytics-catalog-postgres:5432/catalogdb scalar.db.analytics.server.db.username=analytics scalar.db.analytics.server.db.password=analytics diff --git a/scalardb-analytics-sample/config/spark-defaults.conf b/scalardb-analytics-sample/config/spark-defaults.conf index 442ab52..ea3503d 100644 --- a/scalardb-analytics-sample/config/spark-defaults.conf +++ b/scalardb-analytics-sample/config/spark-defaults.conf @@ -1,4 +1,4 @@ -spark.jars.packages com.scalar-labs:scalardb-analytics-spark-all-3.5_2.12:3.16.2 +spark.jars.packages com.scalar-labs:scalardb-analytics-spark-all-3.5_2.12:3.17.0 spark.extraListeners com.scalar.db.analytics.spark.metering.ScalarDbAnalyticsListener # Use the ScalarDB Analytics catalog as `sample_catalog` diff --git a/scalardb-analytics-sample/docker-compose.yml b/scalardb-analytics-sample/docker-compose.yml index 2ad1176..c417bbd 100644 --- a/scalardb-analytics-sample/docker-compose.yml +++ b/scalardb-analytics-sample/docker-compose.yml @@ -26,14 +26,13 @@ services: # ScalarDB Analytics Server scalardb-analytics-server: - image: ghcr.io/scalar-labs/scalardb-analytics-server-without-licensing:3.16.2 + image: ghcr.io/scalar-labs/scalardb-analytics-server-without-licensing:3.17.0 platform: linux/amd64 expose: - 11051 # Catalog service port - 11052 # Metering service port volumes: - ./config/scalardb-analytics-server.properties:/scalardb-analytics-server/server.properties:ro - - ./config/scalardb.properties:/etc/scalardb.properties:ro networks: - scalar-network depends_on: @@ -47,10 +46,11 @@ services: # ScalarDB Analytics CLI scalardb-analytics-cli: - image: ghcr.io/scalar-labs/scalardb-analytics-cli:3.16.2 + image: ghcr.io/scalar-labs/scalardb-analytics-cli:3.17.0 volumes: - ./config/scalardb-analytics-cli.properties:/config/client.properties:ro - ./config/data-sources:/config/data-sources:ro + - ./config/scalardb.properties:/etc/scalardb.properties networks: - scalar-network profiles: @@ -170,7 +170,6 @@ services: context: ./docker dockerfile: Dockerfile.spark volumes: - - ./config/scalardb.properties:/etc/scalardb.properties - ./config/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf - .scala_history:/root/.scala_history - spark-ivy-cache:/root/.ivy2 diff --git a/scalardb-analytics-sample/docker/Dockerfile.spark b/scalardb-analytics-sample/docker/Dockerfile.spark index b31f4ca..e4006f3 100644 --- a/scalardb-analytics-sample/docker/Dockerfile.spark +++ b/scalardb-analytics-sample/docker/Dockerfile.spark @@ -18,11 +18,12 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Download and verify Spark -RUN curl -fsSL -o spark.tgz "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - curl -fsSL -o spark.tgz.sha512 "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz.sha512" && \ - sha512sum -c spark.tgz.sha512 && \ - tar -xzf spark.tgz -C /opt && \ +RUN SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ + curl -fsSL -o "${SPARK_ARCHIVE}" "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}" && \ + curl -fsSL -o "${SPARK_ARCHIVE}.sha512" "https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.sha512" && \ + sha512sum -c "${SPARK_ARCHIVE}.sha512" && \ + tar -xzf "${SPARK_ARCHIVE}" -C /opt && \ mv "/opt/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" "${SPARK_HOME}" && \ - rm -rf spark.tgz spark.tgz.sha512 + rm -rf "${SPARK_ARCHIVE}" "${SPARK_ARCHIVE}.sha512" WORKDIR /opt/spark