diff --git a/build.gradle b/build.gradle index 4bda69de6..9eb4ef2f9 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,6 @@ idea { excludeDirs.add(file("$projectDir/run/dist")) excludeDirs.add(file("$projectDir/run/db")) excludeDirs.add(file("$projectDir/run/logs")) - excludeDirs.add(file("$projectDir/run/install")) excludeDirs.add(file("$projectDir/run/data")) excludeDirs.add(file("$projectDir/run/conf")) excludeDirs.add(file("$projectDir/run/test-data")) diff --git a/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java b/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java index 89237090e..d1b2b9c91 100644 --- a/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java +++ b/code/common/config/src/main/java/nu/marginalia/nodecfg/NodeConfigurationService.java @@ -20,7 +20,7 @@ public NodeConfigurationService(HikariDataSource dataSource) { public NodeConfiguration create(int id, String description, boolean acceptQueries) throws SQLException { try (var conn = dataSource.getConnection(); var is = conn.prepareStatement(""" - INSERT INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?) + INSERT IGNORE INTO NODE_CONFIGURATION(ID, DESCRIPTION, ACCEPT_QUERIES) VALUES(?, ?, ?) """) ) { diff --git a/code/common/service/build.gradle b/code/common/service/build.gradle index 5ae203bdc..64bb7cd58 100644 --- a/code/common/service/build.gradle +++ b/code/common/service/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation libs.bundles.prometheus implementation libs.bundles.slf4j implementation libs.bucket4j + implementation libs.bundles.flyway testImplementation libs.bundles.slf4j.test implementation libs.bundles.mariadb diff --git a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java index 53688daba..ea2ffd3ba 100644 --- a/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java +++ b/code/common/service/src/main/java/nu/marginalia/service/module/DatabaseModule.java @@ -7,6 +7,7 @@ import com.zaxxer.hikari.HikariDataSource; import lombok.SneakyThrows; import nu.marginalia.service.ServiceHomeNotConfiguredException; +import org.flywaydb.core.Flyway; import org.mariadb.jdbc.Driver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,25 @@ public class DatabaseModule extends AbstractModule { private final Properties dbProperties; - public DatabaseModule() { + public DatabaseModule(boolean migrate) { new Driver(); dbProperties = loadDbProperties(); + + if (migrate) { + if (Boolean.getBoolean("disableFlyway")) { + logger.info("Flyway disabled"); + } + else { + var config = Flyway.configure() + .dataSource(getMariaDB()) + .locations("classpath:db/migration") + .load(); + + new Flyway(config.getConfiguration()).migrate(); + } + } + } private Properties loadDbProperties() { diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java index 6115cbb10..19048fc8e 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/ConverterMain.java @@ -54,7 +54,7 @@ public static void main(String... args) throws Exception { Injector injector = Guice.createInjector( new ConverterModule(), new ProcessConfigurationModule("converter"), - new DatabaseModule() + new DatabaseModule(false) ); var converter = injector.getInstance(ConverterMain.class); diff --git a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java index 58d909506..e3c2e1789 100644 --- a/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java +++ b/code/processes/crawling-process/src/main/java/nu/marginalia/crawl/CrawlerMain.java @@ -124,7 +124,7 @@ public static void main(String... args) throws Exception { Injector injector = Guice.createInjector( new CrawlerModule(), new ProcessConfigurationModule("crawler"), - new DatabaseModule() + new DatabaseModule(false) ); var crawler = injector.getInstance(CrawlerMain.class); diff --git a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java index 94f011625..af79fae19 100644 --- a/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java +++ b/code/processes/index-constructor-process/src/main/java/nu/marginalia/index/IndexConstructorMain.java @@ -56,7 +56,7 @@ public static void main(String[] args) throws Exception { var main = Guice.createInjector( new IndexConstructorModule(), new ProcessConfigurationModule("index-constructor"), - new DatabaseModule()) + new DatabaseModule(false)) .getInstance(IndexConstructorMain.class); instructions = main.fetchInstructions(); diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java index 6babfa7ed..15031887b 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/LoaderMain.java @@ -59,7 +59,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new ProcessConfigurationModule("loader"), new LoaderModule(), - new DatabaseModule() + new DatabaseModule(false) ); var instance = injector.getInstance(LoaderMain.class); diff --git a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java index fbda48569..2e3e4d6d7 100644 --- a/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java +++ b/code/processes/website-adjacencies-calculator/src/main/java/nu/marginalia/adjacencies/WebsiteAdjacenciesCalculator.java @@ -186,7 +186,7 @@ private void findAdjacentDtoS(int domainId, Consumer andThen public static void main(String[] args) throws SQLException { - DatabaseModule dm = new DatabaseModule(); + DatabaseModule dm = new DatabaseModule(false); var dataSource = dm.provideConnection(); var qc = new QueryClient(); diff --git a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java index ea3533d55..aea51959d 100644 --- a/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java +++ b/code/services-application/api-service/src/main/java/nu/marginalia/api/ApiMain.java @@ -20,7 +20,7 @@ public static void main(String... args) { init(ServiceId.Api, args); Injector injector = Guice.createInjector( - new DatabaseModule(), + new DatabaseModule(false), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Api)); injector.getInstance(ApiMain.class); injector.getInstance(Initialization.class).setReady(); diff --git a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java index 66bc65e89..00f31cb50 100644 --- a/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java +++ b/code/services-application/dating-service/src/main/java/nu/marginalia/dating/DatingMain.java @@ -27,7 +27,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new DatingModule(), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Dating), - new DatabaseModule() + new DatabaseModule(false) ); injector.getInstance(DatingMain.class); diff --git a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java index 9009b8ba7..2b47657ba 100644 --- a/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java +++ b/code/services-application/explorer-service/src/main/java/nu/marginalia/explorer/ExplorerMain.java @@ -27,7 +27,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Explorer), new ExplorerModule(), - new DatabaseModule() + new DatabaseModule(false) ); injector.getInstance(ExplorerMain.class); diff --git a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java index 80a45f5e7..30dc17237 100644 --- a/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java +++ b/code/services-application/search-service/src/main/java/nu/marginalia/search/SearchMain.java @@ -28,7 +28,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new SearchModule(), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Search), - new DatabaseModule() + new DatabaseModule(false) ); injector.getInstance(SearchMain.class); diff --git a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java index 5b8e49ed1..24f94fa83 100644 --- a/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java +++ b/code/services-core/assistant-service/src/main/java/nu/marginalia/assistant/AssistantMain.java @@ -24,7 +24,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new AssistantModule(), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Assistant), - new DatabaseModule() + new DatabaseModule(false) ); injector.getInstance(AssistantMain.class); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java index a7e31702c..a37cb9c46 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/ControlMain.java @@ -20,7 +20,7 @@ public static void main(String... args) { init(ServiceId.Control, args); Injector injector = Guice.createInjector( - new DatabaseModule(), + new DatabaseModule(true), new ControlProcessModule(), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Control)); diff --git a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java index 768fd3a47..9d9d7d2b2 100644 --- a/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java +++ b/code/services-core/executor-service/src/main/java/nu/marginalia/executor/ExecutorMain.java @@ -24,7 +24,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new ExecutorModule(), - new DatabaseModule(), + new DatabaseModule(false), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Executor) ); injector.getInstance(NodeStatusWatcher.class); diff --git a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java index e542aee6e..9e4042ea4 100644 --- a/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java +++ b/code/services-core/index-service/src/main/java/nu/marginalia/index/IndexMain.java @@ -24,7 +24,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new IndexModule(), - new DatabaseModule(), + new DatabaseModule(false), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Index) ); diff --git a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java index a3651554d..f3f3d561e 100644 --- a/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java +++ b/code/services-core/query-service/src/main/java/nu/marginalia/query/QueryMain.java @@ -23,7 +23,7 @@ public static void main(String... args) { Injector injector = Guice.createInjector( new QueryModule(), - new DatabaseModule(), + new DatabaseModule(false), new ServiceConfigurationModule(SearchServiceDescriptors.descriptors, ServiceId.Query) ); diff --git a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java index 77ee15eda..d58bf7789 100644 --- a/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java +++ b/code/tools/experiment-runner/src/main/java/nu/marginalia/tools/ExperimentRunnerMain.java @@ -37,7 +37,7 @@ public static void main(String... args) throws IOException { } Injector injector = Guice.createInjector( - new DatabaseModule(), + new DatabaseModule(false), new ConverterModule() ); diff --git a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java b/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java index eba900958..51b01cc46 100644 --- a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java +++ b/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotCaptureToolMain.java @@ -29,7 +29,7 @@ public class ScreenshotCaptureToolMain { private static final Logger logger = LoggerFactory.getLogger(ScreenshotCaptureToolMain.class); public static void main(String[] args) { - DatabaseModule databaseModule = new DatabaseModule(); + DatabaseModule databaseModule = new DatabaseModule(false); var ds = databaseModule.provideConnection(); System.setProperty(ChromeDriverService.CHROME_DRIVER_SILENT_OUTPUT_PROPERTY, "true"); diff --git a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java b/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java index f5fedca9f..e982da02a 100644 --- a/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java +++ b/code/tools/screenshot-capture-tool/src/main/java/nu/marginalia/screenshot/ScreenshotLoaderMain.java @@ -15,7 +15,7 @@ public class ScreenshotLoaderMain { public static void main(String... args) throws IOException { org.mariadb.jdbc.Driver driver = new Driver(); - var ds = new DatabaseModule().provideConnection(); + var ds = new DatabaseModule(false).provideConnection(); try (var tis = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(args[0]))); var conn = ds.getConnection(); diff --git a/docker-service-with-dist.gradle b/docker-service-with-dist.gradle index aa97377fd..4beb9ac2e 100644 --- a/docker-service-with-dist.gradle +++ b/docker-service-with-dist.gradle @@ -14,6 +14,8 @@ tasks.register('dockerFile') { # FROM ${dockerImage} +RUN apt-get update && apt-get install -y curl + ADD ${application.applicationName}.tar / ADD crawler-process.tar /dist ADD loader-process.tar /dist diff --git a/docker-service.gradle b/docker-service.gradle index 64055ce98..9ac559c61 100644 --- a/docker-service.gradle +++ b/docker-service.gradle @@ -14,6 +14,7 @@ tasks.register('dockerFile') { # FROM ${dockerImage} +RUN apt-get update && apt-get install -y curl ADD ${application.applicationName}.tar / RUN mkdir /wmsa diff --git a/run/install.sh b/run/install.sh new file mode 100755 index 000000000..9cf4e7357 --- /dev/null +++ b/run/install.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# +# This script will set up a Marginalia instance in a given directory. +# It will create a docker-compose.yml file, and a directory structure +# with the necessary files. It will also create a MariaDB database +# in docker, and run the flyway migrations to set up the database. +# +# After the script is run, the instance can be started with +# $ docker-compose up -d +# +# The instance can be stopped with +# $ docker-compose down -v +# +# It is likely that you will want to edit the docker-compose.yml file +# to change the ports that the services are exposed on, and to change +# the volumes that are mounted. The default configuration is provided +# a starting point. + +set -e + +if ! command -v envsubst &> /dev/null +then + echo "The envsubst command could not be found, please install it. It is usually part of GNU gettext." + exit +fi + +if [ -z "${1}" ]; then + echo "Usage: $0 " + exit 1 +fi +if [ -e "${1}" ]; then + echo "ERROR: Destination ${1} already exists, refusing to overwrite" + exit 1 +fi + +INSTALL_DIR=${1} + +echo "We're going to set up a Mariadb database in docker, please enter some details" + +read -p "MariaDB user (e.g. marginalia): " MARIADB_USER +read -s -p "MariaDB password (e.g. hunter2 ;-): " MARIADB_PASSWORD +echo +read -s -p "MariaDB password (again): " MARIADB_PASSWORD2 +echo + +export MARIADB_USER +export MARIADB_PASSWORD + +if [ "${MARIADB_PASSWORD}" != "${MARIADB_PASSWORD2}" ]; then + echo "ERROR: Passwords do not match" + exit 1 +fi + +echo "Will install to ${INSTALL_DIR}" +read -p "Press enter to continue, or Ctrl-C to abort" + +pushd $(dirname $0) + +./setup.sh ## Ensure that the setup script has been run + +mkdir -p ${INSTALL_DIR} + +echo "** Copying files to ${INSTALL_DIR}" + +for dir in model data conf env; do + if [ ! -d ${dir} ]; then + echo "ERROR: ${dir} does not exist" + exit 1 + fi + echo "Copying ${dir}/" + mkdir -p ${INSTALL_DIR}/${dir} + find ${dir} -maxdepth 1 -type f -exec cp -v {} ${INSTALL_DIR}/{} \; +done + +echo "** Copying settings files" +cp prometheus.yml ${INSTALL_DIR}/ + +echo "** Creating directories" +mkdir -p ${INSTALL_DIR}/logs +mkdir -p ${INSTALL_DIR}/db +mkdir -p ${INSTALL_DIR}/index-1/{work,index,backup,storage,uploads} + +echo "** Updating settings files" + +envsubst < install/mariadb.env.template > ${INSTALL_DIR}/env/mariadb.env +envsubst < install/db.properties.template > ${INSTALL_DIR}/conf/db.properties + +echo "** Creating docker-compose.yml" + +## Hack to get around envstubst substituting these values, which we want to be verbatim +export uval="\$\$MARIADB_USER" +export pval="\$\$MARIADB_PASSWORD" + +export INSTALL_DIR +envsubst < install/docker-compose.yml.template >${INSTALL_DIR}/docker-compose.yml + +popd \ No newline at end of file diff --git a/run/install/db.properties.template b/run/install/db.properties.template new file mode 100644 index 000000000..259d79086 --- /dev/null +++ b/run/install/db.properties.template @@ -0,0 +1,3 @@ +db.user=${MARIADB_USER} +db.pass=${MARIADB_PASSWORD} +db.conn=jdbc:mariadb://mariadb:3306/WMSA_prod?rewriteBatchedStatements=true diff --git a/run/install/docker-compose.yml.template b/run/install/docker-compose.yml.template new file mode 100644 index 000000000..09488f67f --- /dev/null +++ b/run/install/docker-compose.yml.template @@ -0,0 +1,226 @@ +x-svc: &service + env_file: + - "${INSTALL_DIR}/env/service.env" + volumes: + - conf:/wmsa/conf:ro + - model:/wmsa/model + - data:/wmsa/data + - logs:/var/log/wmsa + networks: + - wmsa + healthcheck: + test: curl -f http://localhost:80/internal/ping || exit 1 + start_period: 1s + interval: 5s + timeout: 5s + retries: 60 +x-p1: &partition-1 + env_file: + - "${INSTALL_DIR}/env/service.env" + healthcheck: + test: curl -f http://localhost:80/internal/ping || exit 1 + start_period: 1s + interval: 5s + timeout: 5s + retries: 60 + volumes: + - conf:/wmsa/conf:ro + - model:/wmsa/model + - data:/wmsa/data + - logs:/var/log/wmsa + - index-1:/idx + - work-1:/work + - backup-1:/backup + - samples-1:/storage + networks: + - wmsa + environment: + - "WMSA_SERVICE_NODE=1" +services: + index-service-1: + <<: *partition-1 + image: "marginalia/index-service" + container_name: "index-service-1" + depends_on: + control-service: + condition: service_healthy + executor-service-1: + <<: *partition-1 + image: "marginalia/executor-service" + container_name: "executor-service-1" + depends_on: + control-service: + condition: service_healthy + query-service: + <<: *service + image: "marginalia/query-service" + container_name: "query-service" + depends_on: + control-service: + condition: service_healthy + search-service: + <<: *service + image: "marginalia/search-service" + container_name: "search-service" + depends_on: + control-service: + condition: service_healthy + expose: + - 80 + labels: + - "traefik.enable=true" + - "traefik.http.routers.search-service.rule=PathPrefix(`/`)" + - "traefik.http.routers.search-service.entrypoints=search" + - "traefik.http.routers.search-service.middlewares=add-xpublic" + - "traefik.http.routers.search-service.middlewares=add-public" + - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1" + - "traefik.http.middlewares.add-public.addprefix.prefix=/public" + assistant-service: + <<: *service + image: "marginalia/assistant-service" + container_name: "assistant-service" + depends_on: + control-service: + condition: service_healthy + expose: + - 80 + labels: + - "traefik.enable=true" + - "traefik.http.routers.assistant-service-screenshot.rule=PathPrefix(`/screenshot`)" + - "traefik.http.routers.assistant-service-screenshot.entrypoints=search" + - "traefik.http.routers.assistant-service-screenshot.middlewares=add-xpublic" + - "traefik.http.routers.assistant-service-screenshot.middlewares=add-public" + - "traefik.http.routers.assistant-service-suggest.rule=PathPrefix(`/suggest`)" + - "traefik.http.routers.assistant-service-suggest.entrypoints=search" + - "traefik.http.routers.assistant-service-suggest.middlewares=add-xpublic" + - "traefik.http.routers.assistant-service-suggest.middlewares=add-public" + - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1" + - "traefik.http.middlewares.add-public.addprefix.prefix=/public" + api-service: + <<: *service + image: "marginalia/api-service" + container_name: "api-service" + depends_on: + control-service: + condition: service_healthy + expose: + - "80" + labels: + - "traefik.enable=true" + - "traefik.http.routers.api-service.rule=PathPrefix(`/`)" + - "traefik.http.routers.api-service.entrypoints=api" + - "traefik.http.routers.api-service.middlewares=add-xpublic" + - "traefik.http.routers.api-service.middlewares=add-public" + - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1" + - "traefik.http.middlewares.add-public.addprefix.prefix=/public" + control-service: + <<: *service + image: "marginalia/control-service" + container_name: "control-service" + depends_on: + mariadb: + condition: service_healthy + expose: + - 80 + labels: + - "traefik.enable=true" + - "traefik.http.routers.control-service.rule=PathPrefix(`/`)" + - "traefik.http.routers.control-service.entrypoints=control" + - "traefik.http.routers.control-service.middlewares=add-xpublic" + - "traefik.http.routers.control-service.middlewares=add-public" + - "traefik.http.middlewares.add-xpublic.headers.customrequestheaders.X-Public=1" + - "traefik.http.middlewares.add-public.addprefix.prefix=/public" + mariadb: + image: "mariadb:lts" + container_name: "mariadb" + env_file: "${INSTALL_DIR}/env/mariadb.env" + command: ['mysqld', '--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci'] + ports: + - "127.0.0.1:3306:3306/tcp" + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u ${uval} --password=${pval} + start_period: 5s + interval: 5s + timeout: 5s + retries: 60 + volumes: + - db:/var/lib/mysql + networks: + - wmsa + traefik: + image: "traefik:v2.10" + container_name: "traefik" + command: + #- "--log.level=DEBUG" + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.search.address=:80" + - "--entrypoints.control.address=:81" + - "--entrypoints.api.address=:82" + ports: + - "127.0.0.1:8080:80" + - "127.0.0.1:8081:81" + - "127.0.0.1:8082:82" + - "127.0.0.1:8090:8080" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + networks: + - wmsa +networks: + wmsa: +volumes: + db: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/db + logs: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/logs + model: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/model + conf: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/conf + data: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/data + samples-1: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/index-1/storage + index-1: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/index-1/index + work-1: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/index-1/work + backup-1: + driver: local + driver_opts: + type: none + o: bind + device: ${INSTALL_DIR}/index-1/backup \ No newline at end of file diff --git a/run/install/mariadb.env.template b/run/install/mariadb.env.template new file mode 100644 index 000000000..e79f38d40 --- /dev/null +++ b/run/install/mariadb.env.template @@ -0,0 +1,4 @@ +MARIADB_RANDOM_ROOT_PASSWORD=1 +MARIADB_DATABASE=WMSA_prod +MARIADB_USER=${MARIADB_USER} +MARIADB_PASSWORD=${MARIADB_PASSWORD} \ No newline at end of file diff --git a/run/readme.md b/run/readme.md index 6fe7326ba..87d193dde 100644 --- a/run/readme.md +++ b/run/readme.md @@ -6,13 +6,12 @@ bootstrap this directory structure. ## Requirements -While the system is designed to run bare metal in production, -for local development, you're strongly encouraged to use docker -or podman. These are a bit of a pain to install, but if you follow -[this guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) you're on the right track. +**Docker** - It is a bit of a pain to install, but if you follow +[this guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) you're on the right track for ubuntu-like systems. -The system requires JDK21+, and uses Java 21 preview features. Gradle complains -a bit about this since it's not currently supported, but it works anyway. +**JDK 21** - The code uses Java 21 preview features. +The civilized way of installing this is to use [SDKMAN](https://sdkman.io/); +graalce is a good distribution choice but it doesn't matter too much. ## Set up diff --git a/settings.gradle b/settings.gradle index dbc0c8552..62c008fc7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -184,6 +184,9 @@ dependencyResolutionManagement { library('zstd','com.github.luben','zstd-jni').version('1.5.2-2') library('lz4','org.lz4','lz4-java').version('1.8.0') + library('flyway.core','org.flywaydb','flyway-core').version('10.4.1') + library('flyway.mysql','org.flywaydb','flyway-mysql').version('10.4.1') + library('jsoup','org.jsoup','jsoup').version('1.15.3') library('snakeyaml','org.yaml','snakeyaml').version('1.33') // Known CVE, but this library only parses internal config files so it's fine @@ -214,6 +217,7 @@ dependencyResolutionManagement { bundle('httpcomponents', ['httpcomponents.core', 'httpcomponents.client']) bundle('parquet', ['parquet-column', 'parquet-hadoop']) bundle('junit', ['junit.jupiter', 'junit.jupiter.engine']) + bundle('flyway', ['flyway.core', 'flyway.mysql']) }