Merge pull request #69 from EyeSeeTea/development

Release 1.3.0

README.md

@@ -56,10 +56,10 @@ $ d2-docker create core eyeseetea/dhis2-core:2.30 --war=dhis.war --dhis2-home=/t
 
 ### Create a base DHIS2 data image
 
-Create a dhis2-data image from a .sql.gz SQL file and the apps directory to include:
+Create a dhis2-data image from a .sql.gz SQL file and the apps and documents directory to include:
 
 ```
-$ d2-docker create data eyeseetea/dhis2-data:2.30-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps]
+$ d2-docker create data eyeseetea/dhis2-data:2.30-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] [--documents-dir=path/to/document]
 ```
 
 ### Start a DHIS2 instance
@@ -82,6 +82,15 @@ Some notes:
 -   Use option `-auth` to pass the instance authentication (`USER:PASS`). It will be used to call post-tomcat scripts.
 -   Use option `--run-sql=DIRECTORY` to run SQL files (.sql, .sql.gz or .dump files) after the DB has been initialized.
 -   Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to.
+-   Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
+
+#### Custom DHIS2 dhis.conf
+
+Copy the default [dhis.conf](https://github.com/EyeSeeTea/d2-docker/blob/master/src/d2_docker/config/DHIS2_home/dhis.conf) and use it as a template to create your own configuration. Then pass it to the `start` command:
+
+```
+$ d2-docker start --dhis-conf=dhis.conf ...
+```
 
 #### Custom Tomcat server.xml
 
@@ -125,7 +134,7 @@ _If only one d2-docker container is active, you can omit the image name._
 
 ### Commit & push an image
 
-This will update the image from the current container (SQL dump and apps):
+This will update the image from the current container (SQL dump, apps and documents):
 
 ```
 $ d2-docker commit
@@ -193,7 +202,7 @@ $ d2-docker rm eyeseetea/dhis2-data:2.30-sierra
 
 ### Copy Docker images to/from local directories
 
-You can use a Docker image or a data directory (db + apps) as source, that will create a new Docker image _eyeseetea/dhis2-data:2.30-sierra2_ and a `sierra-data/` directory:
+You can use a Docker image or a data directory (db + apps + documents) as source, that will create a new Docker image _eyeseetea/dhis2-data:2.30-sierra2_ and a `sierra-data/` directory:
 
 ```
 $ d2-docker copy eyeseetea/dhis2-data:2.30-sierra eyeseetea/dhis2-data:2.30-sierra2 sierra-data
@@ -202,10 +211,10 @@ $ docker image ls | grep 2.30-sierra2
 eyeseetea/dhis2-data      2.30-sierra2         930aced0d915        1 minutes ago      106MB
 
 $ ls sierra-data/
-apps db.sql.gz
+apps document db.sql.gz
 ```
 
-Alternatively, you can use a data directory (db + apps) as source and create Docker images from it:
+Alternatively, you can use a data directory (db + apps + documents) as source and create Docker images from it:
 
 ```
 $ d2-docker copy sierra-data eyeseetea/dhis2-data:2.30-sierra3 eyeseetea/dhis2-data:2.30-sierra4

setup.py

@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="d2_docker",
-    version="1.2.0",
+    version="1.3.0",
     description="Dockers for DHIS2 instances",
     long_description=open("README.md", encoding="utf-8").read(),
     keywords=["python"],

src/d2_docker/commands/copy.py

@@ -40,3 +40,5 @@ def copy(source, destinations, docker_dir):
             utils.copytree(source, dest)
         else:
             raise utils.D2DockerError("Not implemented")
+
+        logger.info("Done")

src/d2_docker/commands/create.py

@@ -21,6 +21,7 @@ def setup(parser):
     data_parser.add_argument("data_image", metavar="IMAGE", help="Image core name")
     data_parser.add_argument("--sql", help=".sql (plain text), .sql.gz (gzipped plain text format) or .dump database file (binary format)")
     data_parser.add_argument("--apps-dir", help="Directory containing Dhis2 apps")
+    data_parser.add_argument("--documents-dir", help="Directory containing Dhis2 documents")
 
 
 def run(args):
@@ -55,6 +56,10 @@ def create_data(args):
             dest_apps_dir = os.path.join(build_dir, "apps")
             utils.logger.debug("Copy apps: {} -> {}".format(args.apps_dir, dest_apps_dir))
             utils.copytree(args.apps_dir, dest_apps_dir)
+        if args.documents_dir:
+            dest_documents_dir = os.path.join(build_dir, "document")
+            utils.logger.debug("Copy documents: {} -> {}".format(args.documents_dir, dest_documents_dir))
+            utils.copytree(args.documents_dir, dest_documents_dir)
         if args.sql:
             utils.logger.debug("Copy DB file:  {} -> {}".format(args.sql, db_path))
             shutil.copy(args.sql, db_path)

src/d2_docker/commands/start.py

@@ -9,8 +9,10 @@
 
 def setup(parser):
     d2_docker_path = os.path.abspath(d2_docker.__path__[0])
-    server_xml_path = os.path.join(d2_docker_path, "config", "server.xml")
+    server_xml_path = utils.get_config_file("server.xml")
     server_xml_help = "Use a custom Tomcat server.xml file. Template: {0}".format(server_xml_path)
+    dhis_conf_path = utils.get_config_file("DHIS2_home/dhis.conf")
+    dhis_conf_help = "Use a custom dhis.conf file. Template: {0}".format(dhis_conf_path)
 
     parser.add_argument(
         "image_or_file", metavar="IMAGE_OR_EXPORT_FILE", help="Docker image or exported file"
@@ -24,6 +26,7 @@ def setup(parser):
         "-k", "--keep-containers", action="store_true", help="Keep existing containers"
     )
     parser.add_argument("--tomcat-server-xml", metavar="FILE", help=server_xml_help)
+    parser.add_argument("--dhis-conf", metavar="FILE", help=dhis_conf_help)
     parser.add_argument("--run-sql", metavar="DIRECTORY", help="Run .sql[.gz] files in directory")
     parser.add_argument(
         "--run-scripts",
@@ -33,6 +36,7 @@ def setup(parser):
     parser.add_argument("--pull", action="store_true", help="Force a pull from docker hub")
     parser.add_argument("-p", "--port", type=int, metavar="N", help="Set Dhis2 instance port")
     parser.add_argument("--deploy-path", type=str, help="Set Tomcat context.path")
+    parser.add_argument("--java-opts", type=str, help="Set Tomcat JAVA_OPTS")
 
 
 def run(args):
@@ -81,7 +85,7 @@ def start(args, image_name):
         bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None]
     )
 
-    deploy_path = ("/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else "")
+    deploy_path = "/" + re.sub("^/*", "", args.deploy_path) if args.deploy_path else ""
 
     with utils.stop_docker_on_interrupt(image_name, core_image):
         utils.run_docker_compose(
@@ -94,7 +98,9 @@ def start(args, image_name):
             scripts_dir=args.run_scripts,
             deploy_path=deploy_path,
             dhis2_auth=args.auth,
-            tomcat_server=args.tomcat_server_xml
+            tomcat_server=args.tomcat_server_xml,
+            dhis_conf=args.dhis_conf,
+            java_opts=args.java_opts,
         )
 
     if args.detach:

src/d2_docker/config/dhis2-core-start.sh

@@ -25,7 +25,8 @@ scripts_dir="/data/scripts"
 root_db_path="/data/db"
 post_db_path="/data/db/post"
 source_apps_path="/data/apps"
-dest_apps_path="/DHIS2_home/files/"
+source_documents_path="/data/document"
+files_path="/DHIS2_home/files/"
 tomcat_conf_dir="/usr/local/tomcat/conf"
 
 debug() {
@@ -34,6 +35,8 @@ debug() {
 
 run_sql_files() {
     base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path")
+    debug "Files in data path"
+    find "$base_db_path" >&2;
 
     find "$base_db_path" -type f \( -name '*.dump' \) |
         sort | while read -r path; do
@@ -69,10 +72,18 @@ run_post_scripts() {
 }
 
 copy_apps() {
-    debug "Copy Dhis2 apps: $source_apps_path -> $dest_apps_path"
-    mkdir -p "$dest_apps_path/apps"
+    debug "Copy Dhis2 apps: $source_apps_path -> $files_path"
+    mkdir -p "$files_path/apps"
     if test -e "$source_apps_path"; then
-        cp -Rv "$source_apps_path" "$dest_apps_path"
+        cp -Rv "$source_apps_path" "$files_path"
+    fi
+}
+
+copy_documents() {
+    debug "Copy Dhis2 documents: $source_documents_path -> $files_path"
+    mkdir -p "$files_path/document"
+    if test -e "$source_documents_path"; then
+        cp -Rv "$source_documents_path" "$files_path"
     fi
 }
 
@@ -107,6 +118,7 @@ run() {
     local host=$1 psql_port=$2
     setup_tomcat
     copy_apps
+    copy_documents
     wait_for_postgres
     run_sql_files || true
     run_pre_scripts || true

src/d2_docker/docker-compose.yml

@@ -6,14 +6,15 @@ services:
             - "com.eyeseetea.image-name=${DHIS2_DATA_IMAGE}"
         volumes:
             - home:/DHIS2_home
+            - ${DHIS_CONF}:/config/DHIS2_home/dhis.conf
             - ./config:/config
             - data:/data
             - "${TOMCAT_SERVER}:/config/override/tomcat/server.xml"
             - "${SCRIPTS_DIR}:/data/scripts"
             - "${POST_SQL_DIR}:/data/db/post"
         environment:
             CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH}"
-            JAVA_OPTS: "-Xmx7500m -Xms4000m"
+            JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}"
             LOAD_FROM_DATA: "${LOAD_FROM_DATA}"
             DEPLOY_PATH: "${DEPLOY_PATH}"
             DHIS2_AUTH: "${DHIS2_AUTH}"

src/d2_docker/images/dhis2-data/Dockerfile

@@ -6,5 +6,6 @@ COPY db/ /data/db/
 COPY run.sh /usr/local/bin/
 
 COPY apps/ /data/apps
+COPY document/ /data/document
 
 CMD ["sh", "/usr/local/bin/run.sh"]

src/d2_docker/utils.py

@@ -11,6 +11,7 @@
 from distutils import dir_util
 from pathlib import Path
 
+import d2_docker
 from .image_name import ImageName
 
 PROJECT_NAME_PREFIX = "d2-docker"
@@ -63,6 +64,14 @@ def run(command_parts, raise_on_error=True, env=None, capture_output=False, **kw
         raise D2DockerError(msg.format(cmd, exc.returncode, exc.stderr))
 
 
+@contextlib.contextmanager
+def possible_errors():
+    try:
+        yield
+    except D2DockerError as exc:
+        pass
+
+
 def get_free_port(start=8080, end=65535):
     for port in range(start, end):
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
@@ -202,6 +211,8 @@ def run_docker_compose(
     post_sql_dir=None,
     scripts_dir=None,
     deploy_path=None,
+    dhis_conf=None,
+    java_opts=None,
     dhis2_auth=None,
     tomcat_server=None,
     **kwargs,
@@ -228,15 +239,21 @@ def run_docker_compose(
         ("POST_SQL_DIR", post_sql_dir_abs),
         ("SCRIPTS_DIR", scripts_dir_abs),
         ("DEPLOY_PATH", deploy_path or ""),
+        ("JAVA_OPTS", java_opts or ""),
         ("DHIS2_AUTH", dhis2_auth or ""),
-        ("TOMCAT_SERVER", get_abs_file_for_docker_volume(tomcat_server)),
+        ("TOMCAT_SERVER", get_absfile_for_docker_volume(tomcat_server)),
+        ("DHIS_CONF", get_config_path("DHIS2_home/dhis.conf", dhis_conf)),
     ]
     env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
     yaml_path = os.path.join(os.path.dirname(__file__), "docker-compose.yml")
     return run(["docker-compose", "-f", yaml_path, "-p", project_name, *args], env=env, **kwargs)
 
 
+def get_config_path(default_filename, path):
+    return os.path.abspath(path) if path else get_config_file(default_filename)
+
+
 def get_absdir_for_docker_volume(directory):
     """Return absolute path for given directory, with fallback to empty directory."""
     if not directory:
@@ -248,7 +265,7 @@ def get_absdir_for_docker_volume(directory):
         return os.path.abspath(directory)
 
 
-def get_abs_file_for_docker_volume(file_path):
+def get_absfile_for_docker_volume(file_path):
     """Return absolute path for given file, with fallback to empty file."""
     if not file_path:
         return os.path.join(os.path.dirname(__file__), ".empty", "placeholder")
@@ -315,7 +332,7 @@ def copy_image(docker_dir, source_image, dest_image):
 
 
 def build_image_from_directory(docker_dir, data_dir, dest_image_name):
-    """Build docker image from data (db + apps) directory."""
+    """Build docker image from data (db + apps + documents) directory."""
     with tempfile.TemporaryDirectory() as temp_dir_root:
         logger.info("Create temporal directory: {}".format(temp_dir_root))
         temp_dir = os.path.join(temp_dir_root, "contents")
@@ -329,23 +346,30 @@ def build_image_from_directory(docker_dir, data_dir, dest_image_name):
 
 
 def export_data_from_image(source_image, dest_path):
+    logger.info("Export data from image: {} -> {}".format(source_image, dest_path))
     result = run(["docker", "create", source_image], capture_output=True)
     container_id = result.stdout.decode("utf8").splitlines()[0]
     mkdir_p(dest_path)
     try:
         run(["docker", "cp", container_id + ":" + "/data/db", dest_path])
-        run(["docker", "cp", container_id + ":" + "/data/apps", dest_path])
+        with possible_errors():
+            run(["docker", "cp", container_id + ":" + "/data/apps", dest_path])
+            run(["docker", "cp", container_id + ":" + "/data/document", dest_path])
     finally:
         run(["docker", "rm", "-v", container_id])
 
 
 def export_data_from_running_containers(image_name, containers, destination):
-    """Export data (db + apps) from a running Docker container to a destination directory."""
+    """Export data (db + apps + documents) from a running Docker container to a destination directory."""
     logger.info("Copy Dhis2 apps")
-    apps_source = "{}:/DHIS2_home/files/apps/".format(containers["core"])
     mkdir_p(destination)
+
+    apps_source = "{}:/DHIS2_home/files/apps/".format(containers["core"])
     run(["docker", "cp", apps_source, destination])
 
+    documents_source = "{}:/DHIS2_home/files/document/".format(containers["core"])
+    run(["docker", "cp", documents_source, destination])
+
     db_path = os.path.join(destination, "db", "db.sql.gz")
     export_database(image_name, db_path)
 
@@ -452,7 +476,12 @@ def wait_for_server(port):
 
 
 def create_core(
-    *, docker_dir, image, version=None, war=None, dhis2_home_paths=None,
+    *,
+    docker_dir,
+    image,
+    version=None,
+    war=None,
+    dhis2_home_paths=None,
 ):
     logger.info("Create core image: {}".format(image))
 
@@ -478,4 +507,9 @@ def create_core(
         run(["docker", "build", build_dir, "--tag", image])
 
 
+def get_config_file(filename):
+    d2_docker_path = os.path.abspath(d2_docker.__path__[0])
+    return os.path.join(d2_docker_path, "config", filename)
+
+
 logger = get_logger()