From d216e5212347ba4d2fd5167afc0390adcdc95164 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Fri, 10 Jun 2022 17:06:56 +0300 Subject: [PATCH 1/3] Provide an option to use a Docker image build time argument to add additional base OS packages to the stage 1 image by setting ADDITIONAL_BASE_IMAGE_PACKAGES on the "docker build" command line, without needed to edit DockerfileStage1 locally. Explain about this is docker-compose.yml and also explain there about when to use the "--no-cache" option. --- Dockerfile | 4 +++- DockerfileStage1 | 4 +++- docker-compose.yml | 22 +++++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index af400ba20f..10c2f4c0d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,6 +84,8 @@ ENV WEBWORK_URL=/webwork2 \ DEBCONF_NONINTERACTIVE_SEEN=true \ DEV=0 +ARG ADDITIONAL_BASE_IMAGE_PACKAGES + # Environment variables which depend on a prior environment variable must be set # in an ENV call after the dependencies were defined. ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \ @@ -203,7 +205,7 @@ RUN apt-get update \ zip \ iputils-ping \ imagemagick \ - jq \ + jq $ADDITIONAL_BASE_IMAGE_PACKAGES \ && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && apt-get install -y --no-install-recommends --no-install-suggests nodejs \ && apt-get clean \ diff --git a/DockerfileStage1 b/DockerfileStage1 index fa4c904438..59ad13f9b7 100644 --- a/DockerfileStage1 +++ b/DockerfileStage1 @@ -8,6 +8,8 @@ FROM ubuntu:20.04 # # We only need install time ENV variables, not those needed by the WeBWorK system +ARG ADDITIONAL_BASE_IMAGE_PACKAGES + ENV DEBIAN_FRONTEND=noninteractive \ DEBCONF_NONINTERACTIVE_SEEN=true @@ -124,7 +126,7 @@ RUN apt-get update \ zip \ iputils-ping \ imagemagick \ - jq \ + jq $ADDITIONAL_BASE_IMAGE_PACKAGES \ && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && apt-get install -y --no-install-recommends --no-install-suggests nodejs \ && apt-get clean \ diff --git a/docker-compose.yml b/docker-compose.yml index c1abb62e4a..80c191b161 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,13 @@ services: # docker build --tag webwork-base:forWW217 -f DockerfileStage1 . # and then # docker-compose build + # When rebuilding to get updated images add the "--no-cache" option to both commands. + # You can add something like + # --build-arg ADDITIONAL_BASE_IMAGE_PACKAGES="nano less" + # to the stage 1 "docker build" command to include extra OS packages in the base image. + # That approach makes them (persistent) so available immediately each time a container is + # started and is thus faster than adding them in when a container is started via the + # ADD_APT_PACKAGES setting below. depends_on: - db @@ -232,9 +239,18 @@ services: # Use to build additional locales in the running container at startup. Ex: #ADD_LOCALES: "he_IL ISO-8859-8\nhe_IL.UTF-8 UTF-8\n" - # Extra Ubuntu packages to install during startup - # Commenting the following line out will speed up container startup time. - ADD_APT_PACKAGES: nano less + # Extra Ubuntu packages to install during container startup. + # + # Uncommenting the ADD_APT_PACKAGES line and put the list of additional + # Ubuntu packages to install on the line. + # + # This is best used when you have a temporary need for additional + # packages, as using the ADD_APT_PACKAGES option will slow down the + # container startup time. + # + # OS packages you want frequently available are better added using the + # build time ADDITIONAL_BASE_IMAGE_PACKAGES setting described above. + #ADD_APT_PACKAGES: nano less # The system timezone for the container can be set using #SYSTEM_TIMEZONE: zone/city From 48e1a7869b8d8fbce13a24958681c05ab3de25ac Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Fri, 10 Jun 2022 17:48:36 +0300 Subject: [PATCH 2/3] Modify bin/OPL-update to allow bypassing upload-OPL-statistics.pl using an environment variable. (Needed for Docker.) Modify docker-entrypoint.sh to use the new bin/OPL-update to get the metadata when needed. --- bin/OPL-update | 8 ++++-- docker-config/docker-entrypoint.sh | 43 +++++++++--------------------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/bin/OPL-update b/bin/OPL-update index e6e78c421a..8921d9a1d8 100755 --- a/bin/OPL-update +++ b/bin/OPL-update @@ -49,8 +49,12 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} || print "\nLoading global statistics (if possible).\n"; do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl'; - print "\nSharing aggregated statistics\n"; - do $ENV{WEBWORK_ROOT}.'/bin/upload-OPL-statistics.pl'; + if ( $ENV{SKIP_UPLOAD_OPL_statistics} ) { + print "\nSkipping upload-OPL-statistics as requested\n"; + } else { + print "\nSharing aggregated statistics\n"; + do $ENV{WEBWORK_ROOT}.'/bin/upload-OPL-statistics.pl'; + } } print "\nDone.\n"; diff --git a/docker-config/docker-entrypoint.sh b/docker-config/docker-entrypoint.sh index add5775351..e791019dc9 100755 --- a/docker-config/docker-entrypoint.sh +++ b/docker-config/docker-entrypoint.sh @@ -59,7 +59,7 @@ if [ ! -d "$APP_ROOT/libraries/webwork-open-problem-library/OpenProblemLibrary" cd $APP_ROOT/libraries/ /usr/bin/git clone -v --progress --single-branch --branch main https://github.com/openwebwork/webwork-open-problem-library.git - # The next line forces the system to run OPL-update or load saved OPL tables below, as we just installed it + # The next line forces the system to download and install the OPL metadata later touch "$APP_ROOT/libraries/Restore_or_build_OPL_tables" fi @@ -159,32 +159,22 @@ if [ "$1" = 'apache2' ]; then if [ ! -f "$WEBWORK_ROOT/htdocs/DATA/tagging-taxonomy.json" ]; then # The next line forces the system to run OPL-update below, as the # tagging-taxonomy.json file was found to be missing. - if [ -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then - echo "The tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/." - echo "But the libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql files was seen" - echo "so the OPL tables and the JSON files will (hopefully) be restored from save versions" - else - echo "We will run OPL-update as the tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/." - echo "Check if you should be mounting webwork2/htdocs/DATA/ from outside the Docker image!" - fi + echo "We will run OPL-update as the tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/." + echo "Check if you should be mounting webwork2/htdocs/DATA/ from outside the Docker image!" touch "$APP_ROOT/libraries/Restore_or_build_OPL_tables" fi if [ -f "$APP_ROOT/libraries/Restore_or_build_OPL_tables" ]; then if [ ! -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then - # Download a saved version of the OPL sql table data to be loaded and extract - cd $WEBWORK_ROOT/bin/OPL_releases/ - make extract - cp -r webwork-open-problem-library $APP_ROOT/libraries/ - # check out commit corresponding to the release - make latest_release.tag - OPL_TAG=`cat $WEBWORK_ROOT/bin/OPL_releases/latest_release.tag` - cd $APP_ROOT/libraries/webwork-open-problem-library/ - git fetch --tags origin - echo "Checking out tag $OPL_TAG" + # Download the metadata and install it + export SKIP_UPLOAD_OPL_statistics=1 + if [ ! -d $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED ]; then + mkdir $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED + fi + wait_for_db + $WEBWORK_ROOT/bin/OPL-update + cd $APP_ROOT/libraries/webwork-open-problem-library git checkout $OPL_TAG - fi - - if [ -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then + else echo "Restoring OPL tables from the TABLE-DUMP/OPL-tables.sql file" wait_for_db $WEBWORK_ROOT/bin/restore-OPL-tables.pl @@ -199,15 +189,6 @@ if [ "$1" = 'apache2' ]; then echo "You are missing some of the JSON files including tagging-taxonomy.json" echo "Some of the library functions will not work properly" fi - else - echo "About to start OPL-update. This takes a long time - please be patient." - wait_for_db - $WEBWORK_ROOT/bin/OPL-update - # Dump the OPL tables, to allow a quick restore in the future - $WEBWORK_ROOT/bin/dump-OPL-tables.pl - # Save a copy of the generated JSON files - mkdir -p $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED - cp -a $WEBWORK_ROOT/htdocs/DATA/*.json $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED fi rm $APP_ROOT/libraries/Restore_or_build_OPL_tables fi From 4310b04fa1a8a07b42a41cdf8e67b37dee5c81e0 Mon Sep 17 00:00:00 2001 From: Nathan Wallach Date: Thu, 14 Jul 2022 17:08:41 +0300 Subject: [PATCH 3/3] Delete unnecessary git checkout by OPL_TAG. --- docker-config/docker-entrypoint.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-config/docker-entrypoint.sh b/docker-config/docker-entrypoint.sh index e791019dc9..3a7979cb7e 100755 --- a/docker-config/docker-entrypoint.sh +++ b/docker-config/docker-entrypoint.sh @@ -172,8 +172,6 @@ if [ "$1" = 'apache2' ]; then fi wait_for_db $WEBWORK_ROOT/bin/OPL-update - cd $APP_ROOT/libraries/webwork-open-problem-library - git checkout $OPL_TAG else echo "Restoring OPL tables from the TABLE-DUMP/OPL-tables.sql file" wait_for_db