configbaker container

docker-compose-dev.yml

@@ -32,6 +32,18 @@ services:
     mem_reservation: 1024m
     privileged: false
 
+  dev_bootstrap:
+    container_name: "dev_bootstrap"
+    image: gdcc/configbaker:unstable
+    restart: "no"
+    command:
+      - bootstrap.sh
+      - dev
+    depends_on:
+      - dev_dataverse
+    networks:
+      - dataverse
+
   dev_postgres:
     container_name: "dev_postgres"
     hostname: postgres

modules/container-configbaker/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2023 Forschungszentrum Jülich GmbH
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# This build arg must be given or build will fail
+ARG SOLR_VERSION
+# We simply have this intermediate stage here without any activity to copy the default configset over
+FROM solr:${SOLR_VERSION} AS solr
+
+# Let's build us a baker
+FROM alpine:3
+
+ENV SCRIPT_DIR="/scripts" \
+    SECRETS_DIR="/secrets" \
+    SOLR_TEMPLATE="/template"
+ENV PATH="${PATH}:${SCRIPT_DIR}" \
+    BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap"
+
+ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x"
+
+RUN true && \
+  # Install necessary software and tools
+  apk add --no-cache ${APK_PACKAGES} && \
+  # Make our working directories
+  mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE}
+
+# Get in the scripts and make them executable (just in case...)
+COPY maven/scripts ${SCRIPT_DIR}/
+RUN chmod +x ${SCRIPT_DIR}/*.sh ${BOOTSTRAP_DIR}/*/*.sh
+
+# Copy the Solr config bits
+COPY --from=solr /opt/solr/server/solr/configsets/_default ${SOLR_TEMPLATE}/
+COPY maven/solr/*.xml ${SOLR_TEMPLATE}/conf/
+
+# Copy the data from scripts/api that provide the common base setup you'd get from the installer.
+# ".dockerignore" will take care of taking only the bare necessities
+COPY maven/setup ${SCRIPT_DIR}/bootstrap/base/
+
+# Set the entrypoint to tini (as a process supervisor)
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+# By default run a script that will print a help message and terminate
+CMD ["help.sh"]
+
+LABEL org.opencontainers.image.created="@git.build.time@" \
+      org.opencontainers.image.authors="Research Data Management at FZJ <forschungsdaten@fz-juelich.de>" \
+      org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \
+      org.opencontainers.image.documentation="https://guides.dataverse.org/en/latest/container/" \
+      org.opencontainers.image.source="https://github.com/IQSS/dataverse/tree/develop/modules/container-base" \
+      org.opencontainers.image.version="@project.version@" \
+      org.opencontainers.image.revision="@git.commit.id.abbrev@" \
+      org.opencontainers.image.vendor="Global Dataverse Community Consortium" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.title="Dataverse Base Image" \
+      org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software"

modules/container-configbaker/README.md

@@ -0,0 +1,3 @@
+# ConfigBaker
+
+This is a placeholder. Needs more info about it.

modules/container-configbaker/assembly.xml

@@ -0,0 +1,46 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+    <fileSets>
+        <!-- Get the scripts -->
+        <fileSet>
+            <directory>modules/container-configbaker/scripts</directory>
+            <outputDirectory>scripts</outputDirectory>
+        </fileSet>
+        <!-- Get our custom Solr files -->
+        <fileSet>
+            <directory>conf/solr/8.11.1</directory>
+            <outputDirectory>solr</outputDirectory>
+        </fileSet>
+        <!-- Get the setup scripts from the installer (selected choice only) -->
+        <fileSet>
+            <directory>scripts/api</directory>
+            <outputDirectory>setup</outputDirectory>
+            <includes>
+                <include>setup-all.sh</include>
+                <include>setup-builtin-roles.sh</include>
+                <include>setup-datasetfields.sh</include>
+                <include>setup-identity-providers.sh</include>
+
+                <include>data/licenses/*.json</include>
+                <include>data/authentication-providers/builtin.json</include>
+                <include>data/metadatablocks/*.tsv</include>
+
+                <include>data/dv-root.json</include>
+
+                <include>data/role-admin.json</include>
+                <include>data/role-curator.json</include>
+                <include>data/role-dsContributor.json</include>
+                <include>data/role-dvContributor.json</include>
+                <include>data/role-editor.json</include>
+                <include>data/role-filedownloader.json</include>
+                <include>data/role-fullContributor.json</include>
+                <include>data/role-member.json</include>
+
+                <include>data/user-admin.json</include>
+            </includes>
+            <excludes>
+                <exclude>data/metadatablocks/custom*</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
+</assembly>

modules/container-configbaker/scripts/bootstrap.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# [INFO]: Execute bootstrapping configuration of a freshly baked instance
+
+set -euo pipefail
+
+function usage() {
+  echo "Usage: $(basename "$0") [-u instanceUrl] [-t timeout] [<persona>]"
+  echo ""
+  echo "Execute initial configuration (bootstrapping) of an empty Dataverse instance."
+  echo "Known personas are: $(ls )"
+  echo ""
+  echo "Parameters:"
+  echo "instanceUrl - Location on container network where to reach your instance. Default: 'http://dataverse:8080'"
+  echo "    timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '2m'"
+  echo "    persona - Configure persona to execute. Calls ${BOOTSTRAP_DIR}/<persona>/init.sh. Default: 'base'"
+  echo ""
+  echo "This script will wait for the Dataverse instance to be available before executing the bootstrapping."
+  exit 1
+}
+
+# Set some defaults as documented
+DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"}
+TIMEOUT=${TIMEOUT:-"2m"}
+
+while getopts "u:t:" OPTION
+do
+  case "$OPTION" in
+    u) DATAVERSE_URL="$OPTARG" ;;
+    t) TIMEOUT="$OPTARG" ;;
+    \?) usage;;
+  esac
+done
+shift $((OPTIND-1))
+
+# Assign persona if present or go default
+PERSONA=${1:-"base"}
+
+# Export the URL to be reused in the actual setup scripts
+export DATAVERSE_URL
+
+# Wait for the instance to become available
+echo "Waiting for ${DATAVERSE_URL} to become ready in max ${TIMEOUT}."
+wait4x http "${DATAVERSE_URL}/api/info/version" -i 8s -t "$TIMEOUT" --expect-status-code 200 --expect-body-json data.version
+
+# Avoid bootstrapping again by checking if a metadata block has been loaded
+BLOCK_COUNT=$(curl -sSf "${DATAVERSE_URL}/api/metadatablocks" | jq ".data | length")
+if [[ $BLOCK_COUNT -gt 0 ]]; then
+  echo "Your instance has been bootstrapped already, skipping."
+  exit 0
+fi
+
+# Now execute the bootstrapping script
+echo "Now executing bootstrapping script at ${BOOTSTRAP_DIR}/${PERSONA}/init.sh."
+exec "${BOOTSTRAP_DIR}/${PERSONA}/init.sh"

modules/container-configbaker/scripts/bootstrap/base/init.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Set some defaults as documented
+DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"}
+export DATAVERSE_URL
+
+./setup-all.sh

modules/container-configbaker/scripts/bootstrap/dev/init.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Set some defaults as documented
+DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"}
+export DATAVERSE_URL
+
+echo "Running base setup-all.sh (INSECURE MODE)..."
+"${BOOTSTRAP_DIR}"/base/setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out
+
+echo "Setting system mail address..."
+curl -X PUT -d "dataverse@localhost" "${DATAVERSE_URL}/api/admin/settings/:SystemEmail"
+
+echo "Setting DOI provider to \"FAKE\"..."
+curl "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" -X PUT -d FAKE
+
+API_TOKEN=$(grep apiToken "/tmp/setup-all.sh.out" | jq ".data.apiToken" | tr -d \")
+export API_TOKEN
+
+echo "Publishing root dataverse..."
+curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish"
+
+echo "Allowing users to create dataverses and datasets in root..."
+curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" -d "{\"assignee\": \":authenticated-users\",\"role\": \"fullContributor\"}" "${DATAVERSE_URL}/api/dataverses/:root/assignments"
+
+echo "Checking Dataverse version..."
+curl "${DATAVERSE_URL}/api/info/version"
+
+echo ""
+echo "Done, your instance has been configured for development. Have a nice day!"

modules/container-configbaker/scripts/fix-fs-permissions.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# [INFO]: Fix folder permissions using 'chown' to be writeable by containers not running as root.
+
+set -euo pipefail
+
+if [[ "$(id -un)" != "root" ]]; then
+  echo "This script must be run as user root (not $(id -un)), otherwise no fix is possible."
+fi
+
+DEF_DV_PATH="/dv"
+DEF_SOLR_PATH="/var/solr"
+DEF_DV_UID="1000"
+DEF_SOLR_UID="8983"
+
+function usage() {
+  echo "Usage: $(basename "$0") (dv|solr|[1-9][0-9]{3,4}) [PATH [PATH [...]]]"
+  echo ""
+  echo "You may omit a path when using 'dv' or 'solr' as first argument:"
+  echo "  - 'dv' will default to user $DEF_DV_UID and $DEF_DV_PATH"
+  echo "  - 'solr' will default to user $DEF_SOLR_UID and $DEF_SOLR_PATH"
+  exit 1
+}
+
+# Get a target name or id
+TARGET=${1:-help}
+# Get the rest of the arguments as paths to apply the fix to
+PATHS=( "${@:2}" )
+
+ID=0
+case "$TARGET" in
+  dv)
+    ID="$DEF_DV_UID"
+    # If there is no path, add the default for our app image
+    if [[ ${#PATHS[@]} -eq 0 ]]; then
+      PATHS=( "$DEF_DV_PATH" )
+    fi
+    ;;
+  solr)
+    ID="$DEF_SOLR_UID"
+    # In case there is no path, add the default path for Solr images
+    if [[ ${#PATHS[@]} -eq 0 ]]; then
+      PATHS=( "$DEF_SOLR_PATH" )
+    fi
+    ;;
+  # If there is a digit in the argument, check if this is a valid UID (>= 1000, ...)
+  *[[:digit:]]* )
+    echo "$TARGET" | grep -q "^[1-9][0-9]\{3,4\}$" || usage
+    ID="$TARGET"
+    ;;
+  *)
+    usage
+    ;;
+esac
+
+# Check that we actually have at least 1 path
+if [[ ${#PATHS[@]} -eq 0 ]]; then
+  usage
+fi
+
+# Do what we came for
+chown -R "$ID:$ID" "${PATHS[@]}"

modules/container-configbaker/scripts/help.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# [INFO]: This script.
+
+echo -e '          ╓mαo\n         ╫   jh\n         `%╥æ╨\n           ╫µ\n          ╓@M%╗,\n         ▓`    ╫U\n         ▓²    ╫╛\n          ▓M#M╝"\n  ┌æM╝╝%φ╫┘\n┌╫"      "╫┐\n▓          ▓\n▓          ▓\n`╫µ      ¿╫"\n  "╜%%MM╜`'
+echo ""
+echo "Hello!"
+echo ""
+echo "I'm ConfigBaker, a container image with lots of tooling to 'bake' a containerized Dataverse instance!"
+echo "I can cook up an instance (initial config), put icing on your Solr search index configuration, and more!"
+echo ""
+echo "Here's a list of things I can do for you:"
+
+# Get the longest name length
+LENGTH=1
+for SCRIPT in "${SCRIPT_DIR}"/*.sh; do
+  L="$(basename "$SCRIPT" | wc -m)"
+  if [ "$L" -gt "$LENGTH" ]; then
+    LENGTH="$L"
+  fi
+done
+
+# Print script names and info, but formatted
+for SCRIPT in "${SCRIPT_DIR}"/*.sh; do
+  printf "%${LENGTH}s - " "$(basename "$SCRIPT")"
+  grep "# \[INFO\]: " "$SCRIPT" | sed -e "s|# \[INFO\]: ||"
+done
+
+echo ""
+echo "Simply execute this container with the script name (and potentially arguments) as 'command'."

pom.xml

@@ -770,6 +770,8 @@
                 <app.image.tag>unstable</app.image.tag>
                 <base.image>gdcc/base:${base.image.tag}</base.image>
                 <base.image.tag>unstable</base.image.tag>
+                <conf.image>gdcc/configbaker:${conf.image.tag}</conf.image>
+                <conf.image.tag>${app.image.tag}</conf.image.tag>
 
                 <docker.platforms></docker.platforms>
 
@@ -824,23 +826,8 @@
                                             <descriptor>assembly.xml</descriptor>
                                         </assembly>
                                     </build>
-
+                                    
                                     <run>
-                                        <!-- <autoRemove>true</autoRemove> -->
-                                        <!--
-                                        <wait>
-                                            <http>
-                                                <url>http://localhost:8080/robots.txt</url>
-                                                <method>GET</method>
-                                                <status>200..399</status>
-                                            </http>
-                                            <log>dataverse was successfully deployed</log>
-                                            <time>${ct.run.wait}</time>
-                                            <exec>
-                                                <postStart>/opt/payara/scripts/bootstrap-job.sh</postStart>
-                                            </exec>
-                                        </wait>
-                                        -->
                                     </run>
 
                                     <external>
@@ -849,6 +836,26 @@
                                         <composeFile>docker-compose-dev.yml</composeFile>
                                     </external>
                                 </image>
+
+                                <image>
+                                    <alias>dev_bootstrap</alias>
+                                    <name>${conf.image}</name>
+                                    <build>
+                                        <buildx>
+                                            <platforms>
+                                                <platform>${docker.platforms}</platform>
+                                            </platforms>
+                                        </buildx>
+                                        <dockerFile>${project.basedir}/modules/container-configbaker/Dockerfile</dockerFile>
+                                        <args>
+                                            <SOLR_VERSION>${SOLR_VERSION}</SOLR_VERSION>
+                                        </args>
+                                        <filter>@</filter>
+                                        <assembly>
+                                            <descriptor>${project.basedir}/modules/container-configbaker/assembly.xml</descriptor>
+                                        </assembly>
+                                    </build>
+                                </image>
                             </images>
                             <autoCreateCustomNetworks>true</autoCreateCustomNetworks>
                         </configuration>