(*) install script for deploying Marginalia outside the codebase

The changeset also makes the control service responsible for flyway migrations.  This helps reduce the number of places the database configuration needs to be spread out.  These automatic migrations can be disabled with -DdisableFlyway=true.

The commit also adds curl to the docker container, to enable docker health checks and interdependencies.

build.gradle

@@ -67,7 +67,6 @@ idea {
         excludeDirs.add(file("$projectDir/run/dist"))
         excludeDirs.add(file("$projectDir/run/db"))
         excludeDirs.add(file("$projectDir/run/logs"))
-        excludeDirs.add(file("$projectDir/run/install"))
         excludeDirs.add(file("$projectDir/run/data"))
         excludeDirs.add(file("$projectDir/run/conf"))
         excludeDirs.add(file("$projectDir/run/test-data"))

code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java

@@ -20,7 +20,7 @@ public NodeConfigurationService(HikariDataSource dataSource) {
     public NodeConfiguration create(int id, String description, boolean acceptQueries) throws SQLException {
         try (var conn = dataSource.getConnection();
              var is = conn.prepareStatement("""
-                     INSERT INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?)
+                     INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?)
                      """)
         )
         {

code/common/service/build.gradle

@@ -23,6 +23,7 @@ dependencies {
     implementation libs.bundles.prometheus
     implementation libs.bundles.slf4j
     implementation libs.bucket4j
+    implementation libs.bundles.flyway
 
     testImplementation libs.bundles.slf4j.test
     implementation libs.bundles.mariadb

code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java

@@ -7,6 +7,7 @@
 import com.zaxxer.hikari.HikariDataSource;
 import lombok.SneakyThrows;
 import nu.marginalia.service.ServiceHomeNotConfiguredException;
+import org.flywaydb.core.Flyway;
 import org.mariadb.jdbc.Driver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,10 +29,25 @@ public class DatabaseModule extends AbstractModule {
 
     private final Properties dbProperties;
 
-    public DatabaseModule() {
+    public DatabaseModule(boolean migrate) {
         new Driver();
 
         dbProperties = loadDbProperties();
+
+        if (migrate) {
+            if (Boolean.getBoolean("disableFlyway")) {
+                logger.info("Flyway disabled");
+            }
+            else {
+                var config = Flyway.configure()
+                        .dataSource(getMariaDB())
+                        .locations("classpath:db/migration")
+                        .load();
+
+                new Flyway(config.getConfiguration()).migrate();
+            }
+        }
+
     }
 
     private Properties loadDbProperties() {

code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java

@@ -54,7 +54,7 @@ public static void main(String... args) throws Exception {
             Injector injector = Guice.createInjector(
                     new ConverterModule(),
                     new ProcessConfigurationModule("converter"),
-                    new DatabaseModule()
+                    new DatabaseModule(false)
             );
 
             var converter = injector.getInstance(ConverterMain.class);

code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java

@@ -124,7 +124,7 @@ public static void main(String... args) throws Exception {
             Injector injector = Guice.createInjector(
                     new CrawlerModule(),
                     new ProcessConfigurationModule("crawler"),
-                    new DatabaseModule()
+                    new DatabaseModule(false)
             );
             var crawler = injector.getInstance(CrawlerMain.class);
 

code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java

@@ -56,7 +56,7 @@ public static void main(String[] args) throws Exception {
             var main = Guice.createInjector(
                             new IndexConstructorModule(),
                             new ProcessConfigurationModule("index-constructor"),
-                            new DatabaseModule())
+                            new DatabaseModule(false))
                     .getInstance(IndexConstructorMain.class);
 
             instructions = main.fetchInstructions();

code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java

@@ -59,7 +59,7 @@ public static void main(String... args) {
             Injector injector = Guice.createInjector(
                     new ProcessConfigurationModule("loader"),
                     new LoaderModule(),
-                    new DatabaseModule()
+                    new DatabaseModule(false)
             );
 
             var instance = injector.getInstance(LoaderMain.class);

code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java

@@ -186,7 +186,7 @@ private void findAdjacentDtoS(int domainId, Consumer<DomainSimilarities> andThen
 
 
     public static void main(String[] args) throws SQLException {
-        DatabaseModule dm = new DatabaseModule();
+        DatabaseModule dm = new DatabaseModule(false);
 
         var dataSource = dm.provideConnection();
         var qc = new QueryClient();

code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java

@@ -20,7 +20,7 @@ public static void main(String... args) {
         init(ServiceId.Api, args);
 
         Injector injector = Guice.createInjector(
-                new DatabaseModule(),
+                new DatabaseModule(false),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Api));
         injector.getInstance(ApiMain.class);
         injector.getInstance(Initialization.class).setReady();

code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java

@@ -27,7 +27,7 @@ public static void main(String... args) {
         Injector injector = Guice.createInjector(
                 new DatingModule(),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Dating),
-                new DatabaseModule()
+                new DatabaseModule(false)
         );
 
         injector.getInstance(DatingMain.class);

code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java

@@ -27,7 +27,7 @@ public static void main(String... args) {
         Injector injector = Guice.createInjector(
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Explorer),
                 new ExplorerModule(),
-                new DatabaseModule()
+                new DatabaseModule(false)
         );
 
         injector.getInstance(ExplorerMain.class);

code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java

@@ -28,7 +28,7 @@ public static void main(String... args) {
         Injector injector = Guice.createInjector(
                 new SearchModule(),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Search),
-                new DatabaseModule()
+                new DatabaseModule(false)
         );
 
         injector.getInstance(SearchMain.class);

code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java

@@ -24,7 +24,7 @@ public static void main(String... args) {
         Injector injector = Guice.createInjector(
                 new AssistantModule(),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Assistant),
-                new DatabaseModule()
+                new DatabaseModule(false)
         );
 
         injector.getInstance(AssistantMain.class);

code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java

@@ -20,7 +20,7 @@ public static void main(String... args) {
         init(ServiceId.Control, args);
 
         Injector injector = Guice.createInjector(
-                new DatabaseModule(),
+                new DatabaseModule(true),
                 new ControlProcessModule(),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Control));
 

code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java

@@ -24,7 +24,7 @@ public static void main(String... args) {
 
         Injector injector = Guice.createInjector(
                 new ExecutorModule(),
-                new DatabaseModule(),
+                new DatabaseModule(false),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Executor)
         );
         injector.getInstance(NodeStatusWatcher.class);

code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java

@@ -24,7 +24,7 @@ public static void main(String... args) {
 
         Injector injector = Guice.createInjector(
                 new IndexModule(),
-                new DatabaseModule(),
+                new DatabaseModule(false),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Index)
         );
 

code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java

@@ -23,7 +23,7 @@ public static void main(String... args) {
 
         Injector injector = Guice.createInjector(
                 new QueryModule(),
-                new DatabaseModule(),
+                new DatabaseModule(false),
                 new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Query)
         );
 

code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java

@@ -37,7 +37,7 @@ public static void main(String... args) throws IOException {
         }
 
         Injector injector = Guice.createInjector(
-                new DatabaseModule(),
+                new DatabaseModule(false),
                 new ConverterModule()
         );
 

code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java

@@ -29,7 +29,7 @@ public class ScreenshotCaptureToolMain {
     private static final Logger logger = LoggerFactory.getLogger(ScreenshotCaptureToolMain.class);
 
     public static void main(String[] args) {
-        DatabaseModule databaseModule = new DatabaseModule();
+        DatabaseModule databaseModule = new DatabaseModule(false);
         var ds = databaseModule.provideConnection();
 
         System.setProperty(ChromeDriverService.CHROME_DRIVER_SILENT_OUTPUT_PROPERTY, "true");

code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java

@@ -15,7 +15,7 @@ public class ScreenshotLoaderMain {
     public static void main(String... args) throws IOException {
 
         org.mariadb.jdbc.Driver driver = new Driver();
-        var ds = new DatabaseModule().provideConnection();
+        var ds = new DatabaseModule(false).provideConnection();
 
         try (var tis = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(args[0])));
              var conn = ds.getConnection();

docker-service-with-dist.gradle

@@ -14,6 +14,8 @@ tasks.register('dockerFile') {
 #
 FROM ${dockerImage}
 
+RUN apt-get update && apt-get install -y curl
+
 ADD ${application.applicationName}.tar /
 ADD crawler-process.tar /dist
 ADD loader-process.tar /dist

docker-service.gradle

@@ -14,6 +14,7 @@ tasks.register('dockerFile') {
 #
 FROM ${dockerImage}
 
+RUN apt-get update && apt-get install -y curl
 ADD ${application.applicationName}.tar /
 RUN mkdir /wmsa
 

run/install.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+#
+# This script will set up a Marginalia instance in a given directory.
+# It will create a docker-compose.yml file, and a directory structure
+# with the necessary files.  It will also create a MariaDB database
+# in docker, and run the flyway migrations to set up the database.
+#
+# After the script is run, the instance can be started with
+# $ docker-compose up -d
+#
+# The instance can be stopped with
+# $ docker-compose down -v
+#
+# It is likely that you will want to edit the docker-compose.yml file
+# to change the ports that the services are exposed on, and to change
+# the volumes that are mounted.  The default configuration is provided
+# a starting point.
+
+set -e
+
+if ! command -v envsubst &> /dev/null
+then
+    echo "The envsubst command could not be found, please install it.  It is usually part of GNU gettext."
+    exit
+fi
+
+if [ -z "${1}" ]; then
+  echo "Usage: $0 <install directory>"
+  exit 1
+fi
+if [ -e "${1}" ]; then
+  echo "ERROR: Destination ${1} already exists, refusing to overwrite"
+  exit 1
+fi
+
+INSTALL_DIR=${1}
+
+echo "We're going to set up a Mariadb database in docker, please enter some details"
+
+read -p "MariaDB user (e.g. marginalia): " MARIADB_USER
+read -s -p "MariaDB password (e.g. hunter2 ;-): " MARIADB_PASSWORD
+echo
+read -s -p "MariaDB password (again): " MARIADB_PASSWORD2
+echo
+
+export MARIADB_USER
+export MARIADB_PASSWORD
+
+if [ "${MARIADB_PASSWORD}" != "${MARIADB_PASSWORD2}" ]; then
+  echo "ERROR: Passwords do not match"
+  exit 1
+fi
+
+echo "Will install to ${INSTALL_DIR}"
+read -p "Press enter to continue, or Ctrl-C to abort"
+
+pushd $(dirname $0)
+
+./setup.sh ## Ensure that the setup script has been run
+
+mkdir -p ${INSTALL_DIR}
+
+echo "** Copying files to ${INSTALL_DIR}"
+
+for dir in model data conf env; do
+  if [ ! -d ${dir} ]; then
+    echo "ERROR: ${dir} does not exist"
+    exit 1
+  fi
+  echo "Copying ${dir}/"
+  mkdir -p ${INSTALL_DIR}/${dir}
+  find  ${dir} -maxdepth 1 -type f -exec cp -v {} ${INSTALL_DIR}/{} \;
+done
+
+echo "** Copying settings files"
+cp prometheus.yml ${INSTALL_DIR}/
+
+echo "** Creating directories"
+mkdir -p ${INSTALL_DIR}/logs
+mkdir -p ${INSTALL_DIR}/db
+mkdir -p ${INSTALL_DIR}/index-1/{work,index,backup,storage,uploads}
+
+echo "** Updating settings files"
+
+envsubst < install/mariadb.env.template > ${INSTALL_DIR}/env/mariadb.env
+envsubst < install/db.properties.template > ${INSTALL_DIR}/conf/db.properties
+
+echo "** Creating docker-compose.yml"
+
+## Hack to get around envstubst substituting these values, which we want to be verbatim
+export uval="\$\$MARIADB_USER"
+export pval="\$\$MARIADB_PASSWORD"
+
+export INSTALL_DIR
+envsubst < install/docker-compose.yml.template >${INSTALL_DIR}/docker-compose.yml
+
+popd

run/install/db.properties.template

@@ -0,0 +1,3 @@
+db.user=${MARIADB_USER}
+db.pass=${MARIADB_PASSWORD}
+db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true