Nessie server: add support for MariaDB and MySQL backends

CHANGELOG.md

@@ -14,6 +14,19 @@ as necessary. Empty sections will not end in the release notes.
 
 ### Upgrade notes
 
+#### Support for MariaDB and MySQL
+
+Nessie now includes built-in support for MariaDB, with full compatibility with MySQL servers.
+
+- Existing PostgresQL users can continue to use their current JDBC configuration, but are encouraged 
+  to update it as follows:
+  1. Specify the new configuration property `nessie.version.store.persist.jdbc.datasource=postgresql`;
+  2. Migrate any property under `quarkus.datasource.*` to `quarkus.datasource.postgresql.*`.
+- New users wishing to try MariaDB (or MySQL) should:
+  1. Specify the new configuration property `nessie.version.store.persist.jdbc.datasource=mariadb`;
+  2. Provide all the MariaDB (or MySQL) connection details using `quarkus.datasource.mariadb.*` 
+     configuration properties.
+
 ### Breaking changes
 
 - `nessie-quarkus-cli`, the low-level tool to for example export/import Nessie repositories, has been renamed

NOTICE

@@ -52,6 +52,32 @@ com.github.jnr:jnr-x86asm
  OTHER DEALINGS IN THE SOFTWARE.
 
 
+---
+com.github.waffle:waffle-jna
+
+MIT License
+
+Copyright (c) 2010-2023 The Waffle Project Contributors: https://github.com/Waffle/waffle/graphs/contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 ---
 com.google.protobuf:protobuf-java
 
@@ -1786,6 +1812,14 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 OTHER DEALINGS IN THE SOFTWARE.
 
 
+---
+org.mariadb.jdbc:mariadb-java-client
+
+GNU LESSER GENERAL PUBLIC LICENSE (LGPL) v. 2.1
+
+Copyright (c) 2012-2014 Monty Program Ab
+Copyright (c) 2015-2024 MariaDB Corporation Ab
+
 ---
 postcss
 

gradle/libs.versions.toml

@@ -103,8 +103,8 @@ junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" }
 junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
 keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version.ref = "keycloak" }
-maven-resolver-supplier = { module = "org.apache.maven.resolver:maven-resolver-supplier", version.ref = "mavenResolver" }
 mariadb-java-client = { module = "org.mariadb.jdbc:mariadb-java-client", version = "3.4.0" }
+maven-resolver-supplier = { module = "org.apache.maven.resolver:maven-resolver-supplier", version.ref = "mavenResolver" }
 micrometer-core = { module = "io.micrometer:micrometer-core", version = "1.13.0" }
 microprofile-contextpropagation-api = { module = "org.eclipse.microprofile.context-propagation:microprofile-context-propagation-api", version = "1.3" }
 microprofile-openapi = { module = "org.eclipse.microprofile.openapi:microprofile-openapi-api", version = "3.1.1" }

gradle/license/allowed-licenses.json

@@ -60,6 +60,9 @@
     {
       "moduleLicense": "GNU General Public License, Version 2 with the GNU Classpath Exception"
     },
+    {
+      "moduleLicense": "GNU Lesser General Public License Version 2.1"
+    },
     {
       "moduleLicense": "Go License"
     },

gradle/license/normalizer-bundle.json

@@ -61,6 +61,7 @@
     { "bundleName" : "lgpl21", "licenseNamePattern" : "GNU LESSER GENERAL PUBLIC LICENSE, Version 2.1" },
     { "bundleName" : "lgpl21", "licenseNamePattern" : "LGPL, version 2.1" },
     { "bundleName" : "lgpl21", "licenseNamePattern" : "LGPL 2.1" },
+    { "bundleName" : "lgpl21", "licenseNamePattern" : "LGPL-2.1" },
     { "bundleName" : "lgpl21", "licenseNamePattern" : "lgpl" },
 
     { "bundleName" : "lgpl3", "licenseNamePattern" : "Lesser General Public License, version 3 or greater" },

servers/quarkus-common/build.gradle.kts

@@ -45,6 +45,7 @@ dependencies {
   implementation("io.quarkus:quarkus-hibernate-validator")
   implementation("io.quarkus:quarkus-agroal")
   implementation("io.quarkus:quarkus-jdbc-postgresql")
+  implementation("io.quarkus:quarkus-jdbc-mariadb")
   implementation("io.quarkus:quarkus-opentelemetry")
   implementation("io.quarkus:quarkus-micrometer")
   implementation(enforcedPlatform(libs.quarkus.amazon.services.bom))

servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusJdbcConfig.java

@@ -30,6 +30,13 @@
 @ConfigMapping(prefix = "nessie.version.store.persist.jdbc")
 public interface QuarkusJdbcConfig extends JdbcBackendBaseConfig {
 
+  /**
+   * The name of the datasource to use. Must correspond to a configured datasource under {@code
+   * quarkus.datasource.<name>}. If not provided, the default datasource (PostgreSQL) will be used,
+   * configured from {@code quarkus.datasource}.
+   */
+  Optional<String> datasource();
+
   @Override
   Optional<String> catalog();
 

servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/datasource/DataSourceActivator.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.quarkus.config.datasource;
+
+import io.smallrye.config.ConfigSourceInterceptor;
+import io.smallrye.config.ConfigSourceInterceptorContext;
+import io.smallrye.config.ConfigValue;
+import org.projectnessie.quarkus.config.VersionStoreConfig.VersionStoreType;
+
+/**
+ * Activates a data source based on the current Nessie configuration under {@code
+ * nessie.version.store.persist.jdbc.datasource}, and deactivates all other data sources.
+ *
+ * <p>If the version store type is not JDBC, all data sources are deactivated.
+ */
+public class DataSourceActivator implements ConfigSourceInterceptor {
+
+  private static final int APPLICATION_PROPERTIES_CLASSPATH_ORDINAL = 250;
+
+  @Override
+  public ConfigValue getValue(ConfigSourceInterceptorContext context, String name) {
+    if (name.startsWith("quarkus.datasource.") && name.endsWith(".active")) {
+      boolean active = false;
+      if (versionStoreType(context) == VersionStoreType.JDBC) {
+        active = dataSourceName(name).equals(activeDataSourceName(context));
+      }
+      return newConfigValue(active);
+    }
+    return context.proceed(name);
+  }
+
+  private static VersionStoreType versionStoreType(ConfigSourceInterceptorContext context) {
+    ConfigValue versionStoreType = context.proceed("nessie.version.store.type");
+    if (versionStoreType == null || versionStoreType.getValue() == null) {
+      return VersionStoreType.IN_MEMORY;
+    }
+    return VersionStoreType.valueOf(versionStoreType.getValue());
+  }
+
+  private static String dataSourceName(String name) {
+    if (name.equals("quarkus.datasource.active")) {
+      return "default";
+    }
+    return name.substring("quarkus.datasource.".length(), name.length() - ".active".length());
+  }
+
+  private static String activeDataSourceName(ConfigSourceInterceptorContext context) {
+    ConfigValue nessieDataSource = context.proceed("nessie.version.store.persist.jdbc.datasource");
+    if (nessieDataSource == null || nessieDataSource.getValue() == null) {
+      return "default";
+    }
+    return nessieDataSource.getValue();
+  }
+
+  private static ConfigValue newConfigValue(boolean value) {
+    return ConfigValue.builder()
+        .withValue(value ? "true" : "false")
+        .withConfigSourceOrdinal(APPLICATION_PROPERTIES_CLASSPATH_ORDINAL + 1)
+        .build();
+  }
+}

servers/quarkus-common/src/main/java/org/projectnessie/quarkus/providers/storage/JdbcBackendBuilder.java

@@ -17,43 +17,64 @@
 
 import static org.projectnessie.quarkus.config.VersionStoreConfig.VersionStoreType.JDBC;
 
-import io.agroal.api.AgroalDataSource;
-import io.quarkus.datasource.common.runtime.DatabaseKind;
+import io.quarkus.arc.All;
+import io.quarkus.arc.InstanceHandle;
 import jakarta.enterprise.context.Dependent;
 import jakarta.inject.Inject;
-import org.eclipse.microprofile.config.inject.ConfigProperty;
+import java.util.List;
+import javax.sql.DataSource;
 import org.projectnessie.quarkus.config.QuarkusJdbcConfig;
 import org.projectnessie.quarkus.providers.versionstore.StoreType;
 import org.projectnessie.versioned.storage.common.persist.Backend;
 import org.projectnessie.versioned.storage.jdbc.JdbcBackendConfig;
 import org.projectnessie.versioned.storage.jdbc.JdbcBackendFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @StoreType(JDBC)
 @Dependent
 public class JdbcBackendBuilder implements BackendBuilder {
 
-  @SuppressWarnings("CdiInjectionPointsInspection")
-  @Inject
-  AgroalDataSource dataSource;
+  private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBackendBuilder.class);
 
   @Inject
-  @ConfigProperty(name = "quarkus.datasource.db-kind")
-  String databaseKind;
+  @All
+  @SuppressWarnings("CdiInjectionPointsInspection")
+  List<InstanceHandle<DataSource>> dataSources;
 
   @Inject QuarkusJdbcConfig config;
 
   @Override
   public Backend buildBackend() {
-    if (!DatabaseKind.isPostgreSQL(databaseKind) && !DatabaseKind.isH2(databaseKind)) {
-      throw new IllegalArgumentException(
-          "Database kind is configured to '"
-              + databaseKind
-              + "', which Nessie does not support yet, PostgreSQL, H2, MariaDb and MySQL(via mariaDb driver) are supported. "
-              + "Feel free to raise a pull request to support your database of choice.");
-    }
-
     JdbcBackendFactory factory = new JdbcBackendFactory();
-    JdbcBackendConfig c = JdbcBackendConfig.builder().from(config).dataSource(dataSource).build();
+    JdbcBackendConfig c =
+        JdbcBackendConfig.builder().from(config).dataSource(selectDataSource()).build();
     return factory.buildBackend(c);
   }
+
+  private DataSource selectDataSource() {
+    String dataSourceName = config.datasource().orElse("default");
+    DataSource dataSource = null;
+    for (InstanceHandle<DataSource> handle : dataSources) {
+      String name = handle.getBean().getName();
+      if (name == null) {
+        name = "default";
+      }
+      if (name.equals(dataSourceName)) {
+        dataSource = handle.get();
+      }
+    }
+    if (dataSource == null) {
+      throw new IllegalStateException("No data source configured with name: " + dataSourceName);
+    }
+    if (dataSourceName.equals("default")) {
+      LOGGER.warn(
+          "Legacy datasource configuration found under quarkus.datasource.*: "
+              + "please migrate to quarkus.datasource.postgresql.* and "
+              + "set nessie.version.store.persist.jdbc.datasource=postgresql");
+    } else {
+      LOGGER.info("Using data source: {}", dataSourceName);
+    }
+    return dataSource;
+  }
 }

servers/quarkus-common/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor

@@ -15,3 +15,4 @@
 #
 
 org.projectnessie.quarkus.config.OpenTelemetryConfigSourceInterceptor
+org.projectnessie.quarkus.config.datasource.DataSourceActivator

servers/quarkus-server/src/intTest/java/org/projectnessie/server/ITRestApiPersistMariaDB.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.server;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.projectnessie.quarkus.tests.profiles.QuarkusTestProfilePersistMariaDB;
+
+@QuarkusIntegrationTest
+@TestProfile(QuarkusTestProfilePersistMariaDB.class)
+@DisabledOnOs(OS.WINDOWS) // testcontainers does not support Windows
+class ITRestApiPersistMariaDB extends AbstractQuarkusSmoke {}

servers/quarkus-tests/src/main/java/org/projectnessie/quarkus/tests/profiles/RocksPersistTestResourceLifecycleManager.java → servers/quarkus-server/src/intTest/java/org/projectnessie/server/ITRestApiPersistMySQL.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022 Dremio
+ * Copyright (C) 2024 Dremio
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,11 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.projectnessie.quarkus.tests.profiles;
+package org.projectnessie.server;
 
-public class RocksPersistTestResourceLifecycleManager
-    extends AbstractRocksTestResourceLifecycleManager {
-  public RocksPersistTestResourceLifecycleManager() {
-    super("nessie.version.store.persist.rocks.database-path");
-  }
-}
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.projectnessie.quarkus.tests.profiles.QuarkusTestProfilePersistMySQL;
+
+@QuarkusIntegrationTest
+@TestProfile(QuarkusTestProfilePersistMySQL.class)
+@DisabledOnOs(OS.WINDOWS) // testcontainers does not support Windows
+class ITRestApiPersistMySQL extends AbstractQuarkusSmoke {}

servers/quarkus-server/src/main/resources/application.properties

@@ -85,15 +85,41 @@ nessie.version.store.type=IN_MEMORY
 #nessie.version.store.persist.cache-capacity-mb=0
 
 ## Transactional database configuration
-# Note: Nessie Quarkus Server is built with io.quarkus:quarkus-jdbc-postgresql only. If you need
-# another database, the corresponding Quarkus extensions + driver need to be included in the build
-# and the following setting updated.
+
+# Note: Nessie Quarkus Server comes with built-in support for Postgres and MariaDB, or any database
+# compatible with these. If you need another database, the corresponding Quarkus extension + driver
+# needs to be included in the build.
+# Select the datasource to use with the `nessie.version.store.persist.jdbc.datasource` property;
+# The possible built-in values are: "default" (deprecated), "postgresql" and "mariadb".
+#nessie.version.store.persist.jdbc.datasource=default
+
+# Default datasource configuration (deprecated; use quarkus.datasource.postgresql.* instead):
 quarkus.datasource.db-kind=postgresql
+quarkus.datasource.active=false
+quarkus.datasource.devservices.enabled=false
+#quarkus.datasource.jdbc.url=jdbc:postgresql://localhost:5432/my_database
 #quarkus.datasource.username=<your username>
 #quarkus.datasource.password=<your password>
-#quarkus.datasource.jdbc.url=jdbc:postgresql://localhost:5432/my_database
-# Nessie tests manage PostgreSQL test containers explicitly. Datasource dev services are not necessary.
-quarkus.datasource.devservices.enabled=false
+
+# Postgres datasource configuration:
+quarkus.datasource.postgresql.db-kind=postgresql
+quarkus.datasource.postgresql.active=false
+quarkus.datasource.postgresql.devservices.enabled=false
+#quarkus.datasource.postgresql.jdbc.url=jdbc:postgresql://localhost:5432/my_database
+#quarkus.datasource.postgresql.username=<your username>
+#quarkus.datasource.postgresql.password=<your password>
+
+# MariaDB (or MySQL) datasource configuration:
+quarkus.datasource.mariadb.db-kind=mariadb
+quarkus.datasource.mariadb.active=false
+quarkus.datasource.mariadb.devservices.enabled=false
+#quarkus.datasource.mariadb.username=<your username>
+#quarkus.datasource.mariadb.password=<your password>
+#quarkus.datasource.mariadb.jdbc.url=jdbc:mariadb://localhost:3306/my_database
+# Do not remove or modify the following, as these optimization flags are incompatible with Nessie;
+# see https://mariadb.com/docs/server/connect/programming-languages/java/batch.
+quarkus.datasource.mariadb.jdbc.additional-jdbc-properties.useBulkStmts=false
+quarkus.datasource.mariadb.jdbc.additional-jdbc-properties.useBulkStmtsForInserts=false
 
 ## RocksDB version store specific configuration
 #nessie.version.store.persist.rocks.database-path=nessie-rocksdb
@@ -269,3 +295,4 @@ quarkus.micrometer.binder.http-server.match-patterns=\
 # Turn off OIDC connection error in tests - DO NOT PUT THIS SETTING INTO YOUR PRODUCTION CODE,
 # because it would hide other OIDC issues as well!
 %test.quarkus.log.category."io.quarkus.oidc.common.runtime.OidcCommonUtils".level=OFF
+#%test.quarkus.test.arg-line=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005

servers/quarkus-tests/build.gradle.kts

@@ -36,6 +36,8 @@ dependencies {
   implementation("org.testcontainers:testcontainers")
   implementation("org.testcontainers:cassandra")
   implementation("org.testcontainers:postgresql")
+  implementation("org.testcontainers:mysql")
+  implementation("org.testcontainers:mariadb")
   implementation("org.testcontainers:mongodb")
   implementation(libs.docker.java.api)
   compileOnly(project(":nessie-keycloak-testcontainer"))