HSEARCH-4674 Rework Elasticsearch/OpenSearch container integration

.github/dependabot.yml

@@ -111,6 +111,44 @@ updates:
       - dockerhub
     # For dependabot to find Docker files they all should be in the same directory.
     # Dependabot is picking any files that has `dockerfile` in them.
+    #
+    # This includes only Ryuk container, hence we want it to be checked/updated at the same time when our regular java dependencies are updated
     directory: "/build/container/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"
+      day: "tuesday"
+    assignees: ["yrodiere"]
+  - package-ecosystem: "docker"
+    registries:
+      - dockerhub
+    # For dependabot to find Docker files they all should be in the same directory.
+    # Dependabot is picking any files that has `dockerfile` in them.
+    #
+    # This will only include database containers, hence we check for updates only once a month:
+    directory: "/build/container/database"
+    schedule:
+      interval: "monthly"
+    assignees: ["yrodiere"]
+    groups:
+      # This group combines all database containers dependencies.
+      database-containers:
+        patterns:
+          # Include all:
+          - "*"
+  - package-ecosystem: "docker"
+    registries:
+      - dockerhub
+    # For dependabot to find Docker files they all should be in the same directory.
+    # Dependabot is picking any files that has `dockerfile` in them.
+    directory: "/build/container/search-backend"
+    schedule:
+      interval: "daily"
+    assignees: ["yrodiere"]
+    groups:
+      # This group combines all search backends containers dependencies.
+      # We will still need to create specific tickets to address these updates and do some code changes,
+      # so we just want to get a notification from the bot that the new versions are available.
+      search-backend-containers:
+        patterns:
+          # Include all:
+          - "*"

build/container/search-backend/amazon-opensearch-serverless.Dockerfile

@@ -0,0 +1,7 @@
+# OpenSearch
+# See https://hub.docker.com/r/opensearchproject/opensearch/tags
+#
+# IMPORTANT! When updating the version of OpenSearch in this Dockerfile,
+# make sure to update `version.org.opensearch.latest` property in a POM file,
+# and to update the version in opensearch.Dockerfile as well.
+FROM docker.io/opensearchproject/opensearch:2.11.0

build/container/elasticsearch.Dockerfile → build/container/search-backend/elastic.Dockerfile

@@ -1,8 +1,7 @@
 # Elasticsearch
 # https://hub.docker.com/r/elastic/elasticsearch/tags
 #
-# IMPORTANT! When updating the version for Ryuk in this Dockerfile,
+# IMPORTANT! When updating the version of Elasticsearch in this Dockerfile,
 # make sure to update `version.org.elasticsearch.latest` property in a POM file.
 #
-
 FROM docker.io/elastic/elasticsearch:8.10.4

build/container/search-backend/opensearch.Dockerfile

@@ -0,0 +1,7 @@
+# OpenSearch
+# See https://hub.docker.com/r/opensearchproject/opensearch/tags
+#
+# IMPORTANT! When updating the version of OpenSearch in this Dockerfile,
+# make sure to update `version.org.opensearch.latest` property in a POM file,
+# and to update the version in amazon-opensearch-serverless.Dockerfile as well.
+FROM docker.io/opensearchproject/opensearch:2.11.0

build/parents/build/pom.xml

@@ -119,9 +119,9 @@
         <version.junit-jupiter>5.10.0</version.junit-jupiter>
         <version.junit-platform-suite-engine>1.10.0</version.junit-platform-suite-engine>
         <version.org.osgi.core>6.0.0</version.org.osgi.core>
+        <!-- Make sure to update Ryuk container version in `ryuk.Dockerfile` whenever updating testcontainers to align it with the lib. -->
         <version.testcontainers>1.19.1</version.testcontainers>
 
-
         <version.org.hamcrest>2.2</version.org.hamcrest>
         <version.org.mockito>5.7.0</version.org.mockito>
         <version.org.assertj.assertj-core>3.24.2</version.org.assertj.assertj-core>
@@ -215,7 +215,6 @@
              because we will be able to change the configuration without re-compiling the configuration file
              (which is located in a dependency of the integration tests modules).
          -->
-        <test.elasticsearch.distribution>elastic</test.elasticsearch.distribution>
         <test.elasticsearch.connection.uris></test.elasticsearch.connection.uris>
         <test.elasticsearch.connection.username></test.elasticsearch.connection.username>
         <test.elasticsearch.connection.password></test.elasticsearch.connection.password>
@@ -1203,10 +1202,9 @@
                             <org.hibernate.search.version>${project.version}</org.hibernate.search.version>
                             <org.hibernate.search.enable_performance_tests>${test.performance.enable}</org.hibernate.search.enable_performance_tests>
                             <org.hibernate.search.integrationtest.backend.elasticsearch.distribution>${test.elasticsearch.distribution}</org.hibernate.search.integrationtest.backend.elasticsearch.distribution>
-                            <org.hibernate.search.integrationtest.backend.elasticsearch.name>${test.searchengine.run.image.name}</org.hibernate.search.integrationtest.backend.elasticsearch.name>
-                            <org.hibernate.search.integrationtest.backend.elasticsearch.tag>${test.searchengine.run.image.tag}</org.hibernate.search.integrationtest.backend.elasticsearch.tag>
+                            <org.hibernate.search.integrationtest.backend.elasticsearch.version>${test.elasticsearch.version}</org.hibernate.search.integrationtest.backend.elasticsearch.version>
                             <org.hibernate.search.integrationtest.orm.database.kind>${test.database.run.kind}</org.hibernate.search.integrationtest.orm.database.kind>
-                            <org.hibernate.search.integrationtest.orm.project.root.directory>${rootProject.directory}</org.hibernate.search.integrationtest.orm.project.root.directory>
+                            <org.hibernate.search.integrationtest.project.root.directory>${rootProject.directory}</org.hibernate.search.integrationtest.project.root.directory>
                         </systemPropertyVariables>
                     </configuration>
                 </plugin>

build/parents/integrationtest/pom.xml

@@ -65,30 +65,32 @@
         <!--
              To run tests against a different version of Elasticsearch, see CONTRIBUTING.md.
           -->
-        <!-- Profile enabled when an Elasticsearch instance must be pulled -->
+        <!-- Profile enabled when an Elasticsearch/OpenSearch container must NOT be started as a test container, but when an URL to an "external" service is supplied! -->
         <profile>
-            <id>elasticsearch-run</id>
+            <id>search-container-do-not-start</id>
             <activation>
-                <!-- Activate by default, i.e. if test.elasticsearch.connection.uris has not been defined explicitly -->
+                <!-- Activate if test.elasticsearch.connection.uris has been defined explicitly -->
                 <property>
-                    <name>!test.elasticsearch.connection.uris</name>
+                    <name>test.elasticsearch.connection.uris</name>
                 </property>
             </activation>
             <properties>
-                <test.elasticsearch.run.image.name>${test.elasticsearch.run.elastic.image.name}</test.elasticsearch.run.image.name>
-                <test.elasticsearch.run.image.tag>${test.elasticsearch.run.elastic.image.tag}</test.elasticsearch.run.image.tag>
+                <test.elasticsearch.run.image.pull>false</test.elasticsearch.run.image.pull>
             </properties>
         </profile>
-        <!-- Profile enabled when an Elasticsearch instance must NOT be run by Maven -->
+
+        <!-- Profile enabled when Elasticsearch tests are skipped -->
         <profile>
-            <id>elasticsearch-do-not-run</id>
+            <id>elasticsearch-test-skip</id>
             <activation>
                 <!-- Activate if test.elasticsearch.connection.uris has been defined explicitly -->
                 <property>
-                    <name>test.elasticsearch.connection.uris</name>
+                    <name>test.elasticsearch.skip</name>
+                    <value>true</value>
                 </property>
             </activation>
             <properties>
+                <test.elasticsearch.run.skip.forRelevantModules>true</test.elasticsearch.run.skip.forRelevantModules>
                 <test.elasticsearch.run.image.pull>false</test.elasticsearch.run.image.pull>
             </properties>
         </profile>
@@ -112,20 +114,6 @@
             </properties>
         </profile>
 
-        <profile>
-            <id>opensearch</id>
-            <activation>
-                <property>
-                    <name>test.elasticsearch.distribution</name>
-                    <value>opensearch</value>
-                </property>
-            </activation>
-            <properties>
-                <test.elasticsearch.run.image.name>${test.elasticsearch.run.opensearch.image.name}</test.elasticsearch.run.image.name>
-                <test.elasticsearch.run.image.tag>${test.elasticsearch.run.opensearch.image.tag}</test.elasticsearch.run.image.tag>
-            </properties>
-        </profile>
-
         <!-- =============================== -->
         <!-- Database profiles               -->
         <!-- =============================== -->

pom.xml

@@ -377,24 +377,14 @@
         <!-- Container images for various integration tests -->
         <!-- The latest version of Elasticsearch tested against by default -->
         <version.org.elasticsearch.latest>8.10.4</version.org.elasticsearch.latest>
-        <test.elasticsearch.version>${version.org.elasticsearch.latest}</test.elasticsearch.version>
-
-        <test.elasticsearch.run.elastic.skip>true</test.elasticsearch.run.elastic.skip>
-        <test.elasticsearch.run.elastic.image.name>docker.io/elastic/elasticsearch</test.elasticsearch.run.elastic.image.name>
-        <test.elasticsearch.run.elastic.image.tag>${test.elasticsearch.version}</test.elasticsearch.run.elastic.image.tag>
-
-        <test.elasticsearch.run.opensearch.skip>true</test.elasticsearch.run.opensearch.skip>
-        <test.elasticsearch.run.opensearch.image.name>docker.io/opensearchproject/opensearch</test.elasticsearch.run.opensearch.image.name>
-        <test.elasticsearch.run.opensearch.image.tag>${test.elasticsearch.version}</test.elasticsearch.run.opensearch.image.tag>
+        <test.elasticsearch.version></test.elasticsearch.version>
+        <test.elasticsearch.distribution>elastic</test.elasticsearch.distribution>
 
         <!-- Docker images to be pulled. To be redefined in specific profiles of Elasticsearch/OpenSearch \
              or database specific ones. -->
         <test.database.run.kind>h2</test.database.run.kind>
 
         <test.elasticsearch.run.image.pull>true</test.elasticsearch.run.image.pull>
-        <test.elasticsearch.run.image.name></test.elasticsearch.run.image.name>
-        <test.elasticsearch.run.image.tag></test.elasticsearch.run.image.tag>
-
 
         <!-- Set empty default values to avoid Maven leaving property references (${...}) when it doesn't find a value -->
 
@@ -965,10 +955,16 @@
 
                                     images += parseImage( './build/container/ryuk.Dockerfile' )
                                     if ( isNotBlankString( '${test.database.run.kind}' ) && '${test.database.run.kind}' != 'h2') {
-                                        images += parseImage( './build/container/${test.database.run.kind}.Dockerfile' )
+                                        images += parseImage( './build/container/database/${test.database.run.kind}.Dockerfile' )
                                     }
-                                    if ( isTrueString( '${test.elasticsearch.run.image.pull}' ) && isNotBlankString( '${test.elasticsearch.run.image.name}' ) ) {
-                                        images += '${test.elasticsearch.run.image.name}:${test.elasticsearch.run.image.tag}'
+                                    if ( isTrueString( '${test.elasticsearch.run.image.pull}' ) ) {
+                                        def fromFile = parseImage( './build/container/search-backend/${test.elasticsearch.distribution}.Dockerfile')
+                                        def suppliedVersion = '${test.elasticsearch.version}'
+                                        if (isNotBlankString(suppliedVersion)) {
+                                            images += fromFile.substring(0, fromFile.lastIndexOf(':')+1) + suppliedVersion
+                                        } else {
+                                            images += fromFile
+                                        }
                                     }
                                     // Exclude images from non-dockerhub repositories
                                     new File('${containerImagesListFile}').append( images.findAll{ref -> !(ref ==~ /^(?!((.*\.)?docker\.io))[\w\d\.]+\/[\w\d\.]+\/[\w\d\.]+(:.+)?$/)}.join('\n') + '\n' )

util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/SearchBackendContainer.java

@@ -6,7 +6,12 @@
  */
 package org.hibernate.search.util.impl.integrationtest.backend.elasticsearch;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName;
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
@@ -23,12 +28,30 @@ private SearchBackendContainer() {
 	private static final GenericContainer<?> SEARCH_CONTAINER;
 
 	static {
-		String name = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.name", "" );
-		String tag = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.tag" );
-
-		SEARCH_CONTAINER = name.contains( "elastic" )
-				? elasticsearch( name, tag, ElasticsearchVersion.of( ElasticsearchDistributionName.ELASTIC, tag ) )
-				: opensearch( name, tag );
+		ElasticsearchDistributionName distributionName = ElasticsearchDistributionName
+				.of( System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.distribution", "" ) );
+		String tag = System.getProperty( "org.hibernate.search.integrationtest.backend.elasticsearch.version" );
+		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.project.root.directory", "" ) );
+
+		try {
+			DockerImageName dockerImageName = parseDockerImageName( root.resolve( "build" ).resolve( "container" )
+					.resolve( "search-backend" ).resolve( distributionName.externalRepresentation() + ".Dockerfile" ), tag );
+			switch ( distributionName ) {
+				case ELASTIC:
+					SEARCH_CONTAINER = elasticsearch( dockerImageName );
+					break;
+				case OPENSEARCH:
+				case AMAZON_OPENSEARCH_SERVERLESS:
+					SEARCH_CONTAINER = opensearch( dockerImageName );
+					break;
+				default:
+					throw new IllegalStateException( "Unknown distribution " + distributionName );
+			}
+		}
+		catch (IOException e) {
+			throw new IllegalStateException(
+					"Unable to initialize a Search Engine container [" + distributionName + ", " + tag + ", " + root + "]", e );
+		}
 	}
 
 	public static int mappedPort(int port) {
@@ -60,8 +83,8 @@ private static void startIfNeeded() {
 		}
 	}
 
-	private static GenericContainer<?> elasticsearch(String name, String tag, ElasticsearchVersion version) {
-		GenericContainer<?> container = common( name, tag )
+	private static GenericContainer<?> elasticsearch(DockerImageName dockerImageName) {
+		GenericContainer<?> container = common( dockerImageName )
 				.withEnv( "logger.level", "WARN" )
 				.withEnv( "discovery.type", "single-node" )
 				// Older images require HTTP authentication for all requests;
@@ -79,6 +102,9 @@ private static GenericContainer<?> elasticsearch(String name, String tag, Elasti
 				// See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/modules-cluster.html#disk-based-shard-allocation
 				.withEnv( "cluster.routing.allocation.disk.threshold_enabled", "false" );
 
+		ElasticsearchVersion version =
+				ElasticsearchVersion.of( ElasticsearchDistributionName.ELASTIC, dockerImageName.getVersionPart() );
+
 		// Disable a few features that we don't use and that just slow up container startup.
 		if ( version.majorOptional().orElse( Integer.MIN_VALUE ) == 8 ) {
 			if ( version.minor().orElse( Integer.MAX_VALUE ) > 7 ) {
@@ -109,8 +135,8 @@ private static GenericContainer<?> elasticsearch(String name, String tag, Elasti
 		return container;
 	}
 
-	private static GenericContainer<?> opensearch(String name, String tag) {
-		return common( name, tag )
+	private static GenericContainer<?> opensearch(DockerImageName dockerImageName) {
+		return common( dockerImageName )
 				.withEnv( "logger.level", "WARN" )
 				.withEnv( "discovery.type", "single-node" )
 				// Prevent swapping
@@ -134,11 +160,30 @@ private static GenericContainer<?> opensearch(String name, String tag) {
 	 * this resource in the end.
 	 */
 	@SuppressWarnings("resource")
-	private static GenericContainer<?> common(String image, String tag) {
-		return new GenericContainer<>( DockerImageName.parse( image ).withTag( tag ) )
+	private static GenericContainer<?> common(DockerImageName dockerImageName) {
+		return new GenericContainer<>( dockerImageName )
 				.withExposedPorts( 9200, 9300 )
 				.waitingFor( new HttpWaitStrategy().forPort( 9200 ).forStatusCode( 200 ) )
 				.withStartupTimeout( Duration.ofMinutes( 5 ) )
 				.withReuse( true );
 	}
+
+
+	private static DockerImageName parseDockerImageName(Path dockerfile, String tag) throws IOException {
+		Pattern DOCKERFILE_FROM_LINE_PATTERN = Pattern.compile( "FROM (.+)" );
+
+		DockerImageName dockerImageName = Files.lines( dockerfile )
+				.map( DOCKERFILE_FROM_LINE_PATTERN::matcher )
+				.filter( Matcher::matches )
+				.map( m -> m.group( 1 ) )
+				.map( DockerImageName::parse )
+				.findAny().orElseThrow( () -> new IllegalStateException(
+						"Dockerfile " + dockerfile + " has unexpected structure. It *must* contain a single FROM line." ) );
+
+		if ( tag != null && !tag.trim().isEmpty() ) {
+			return dockerImageName.withTag( tag );
+		}
+
+		return dockerImageName;
+	}
 }

util/internal/integrationtest/mapper/orm/src/main/java/org/hibernate/search/util/impl/integrationtest/mapper/orm/DatabaseContainer.java

@@ -41,11 +41,11 @@ private DatabaseContainer() {
 
 	static {
 		String name = System.getProperty( "org.hibernate.search.integrationtest.orm.database.kind", "" );
-		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.orm.project.root.directory", "" ) );
+		Path root = Path.of( System.getProperty( "org.hibernate.search.integrationtest.project.root.directory", "" ) );
 		DATABASE = SupportedDatabase.from( name );
 
 		DATABASE_CONTAINER = DATABASE.container(
-				root.resolve( "build" ).resolve( "container" ).resolve( name + ".Dockerfile" ),
+				root.resolve( "build" ).resolve( "container" ).resolve( "database" ).resolve( name + ".Dockerfile" ),
 				name
 		);
 	}