Docker updates for WW 2.17: ADDITIONAL_BASE_IMAGE_PACKAGES and updated OPL metadata download approach

Dockerfile

@@ -84,6 +84,8 @@ ENV WEBWORK_URL=/webwork2 \
 	DEBCONF_NONINTERACTIVE_SEEN=true \
 	DEV=0
 
+ARG ADDITIONAL_BASE_IMAGE_PACKAGES
+
 # Environment variables which depend on a prior environment variable must be set
 # in an ENV call after the dependencies were defined.
 ENV WEBWORK_ROOT=$APP_ROOT/webwork2 \
@@ -203,7 +205,7 @@ RUN apt-get update \
 	zip \
 	iputils-ping \
 	imagemagick \
-	jq \
+	jq $ADDITIONAL_BASE_IMAGE_PACKAGES \
 	&& curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
 	&& apt-get install -y --no-install-recommends --no-install-suggests nodejs \
 	&& apt-get clean \

DockerfileStage1

@@ -8,6 +8,8 @@ FROM ubuntu:20.04
 #
 # We only need install time ENV variables, not those needed by the WeBWorK system
 
+ARG ADDITIONAL_BASE_IMAGE_PACKAGES
+
 ENV DEBIAN_FRONTEND=noninteractive \
 	DEBCONF_NONINTERACTIVE_SEEN=true
 
@@ -124,7 +126,7 @@ RUN apt-get update \
 	zip \
 	iputils-ping \
 	imagemagick \
-	jq \
+	jq $ADDITIONAL_BASE_IMAGE_PACKAGES \
 	&& curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
 	&& apt-get install -y --no-install-recommends --no-install-suggests nodejs \
 	&& apt-get clean \

bin/OPL-update

@@ -49,8 +49,12 @@ if ($ce->{problemLibrary}{showLibraryLocalStats} ||
   print "\nLoading global statistics (if possible).\n";
   do $ENV{WEBWORK_ROOT}.'/bin/load-OPL-global-statistics.pl';
 
-  print "\nSharing aggregated statistics\n";
-  do $ENV{WEBWORK_ROOT}.'/bin/upload-OPL-statistics.pl';
+  if ( $ENV{SKIP_UPLOAD_OPL_statistics} ) {
+    print "\nSkipping upload-OPL-statistics as requested\n";
+  } else {
+    print "\nSharing aggregated statistics\n";
+    do $ENV{WEBWORK_ROOT}.'/bin/upload-OPL-statistics.pl';
+  }
 }
 
 print "\nDone.\n";

docker-compose.yml

@@ -101,6 +101,13 @@ services:
         #     docker build --tag webwork-base:forWW217 -f DockerfileStage1 .
         # and then
         #     docker-compose build
+        # When rebuilding to get updated images add the "--no-cache" option to both commands.
+        # You can add something like
+        #         --build-arg ADDITIONAL_BASE_IMAGE_PACKAGES="nano less"
+        # to the stage 1 "docker build" command to include extra OS packages in the base image.
+        # That approach makes them (persistent) so available immediately each time a container is
+        # started and is thus faster than adding them in when a container is started via the
+        # ADD_APT_PACKAGES setting below.
 
     depends_on:
       - db
@@ -232,9 +239,18 @@ services:
       # Use to build additional locales in the running container at startup. Ex:
       #ADD_LOCALES: "he_IL ISO-8859-8\nhe_IL.UTF-8 UTF-8\n"
 
-      # Extra Ubuntu packages to install during startup
-      # Commenting the following line out will speed up container startup time.
-      ADD_APT_PACKAGES: nano less
+      # Extra Ubuntu packages to install during container startup.
+      #
+      # Uncommenting the ADD_APT_PACKAGES line and put the list of additional
+      # Ubuntu packages to install on the line.
+      #
+      # This is best used when you have a temporary need for additional
+      # packages, as using the ADD_APT_PACKAGES option will slow down the
+      # container startup time.
+      #
+      # OS packages you want frequently available are better added using the
+      # build time ADDITIONAL_BASE_IMAGE_PACKAGES setting described above.
+      #ADD_APT_PACKAGES: nano less
 
       # The system timezone for the container can be set using
       #SYSTEM_TIMEZONE: zone/city

docker-config/docker-entrypoint.sh

@@ -59,7 +59,7 @@ if [ ! -d "$APP_ROOT/libraries/webwork-open-problem-library/OpenProblemLibrary"
   cd $APP_ROOT/libraries/
   /usr/bin/git clone -v --progress --single-branch --branch main https://github.com/openwebwork/webwork-open-problem-library.git
 
-  # The next line forces the system to run OPL-update or load saved OPL tables below, as we just installed it
+  # The next line forces the system to download and install the OPL metadata later
   touch "$APP_ROOT/libraries/Restore_or_build_OPL_tables"
 fi
 
@@ -159,32 +159,20 @@ if [ "$1" = 'apache2' ]; then
     if [ ! -f "$WEBWORK_ROOT/htdocs/DATA/tagging-taxonomy.json"  ]; then
       # The next line forces the system to run OPL-update below, as the
       # tagging-taxonomy.json file was found to be missing.
-      if [ -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then
-        echo "The tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/."
-        echo "But the libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql files was seen"
-        echo "so the OPL tables and the JSON files will (hopefully) be restored from save versions"
-      else
-        echo "We will run OPL-update as the tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/."
-        echo "Check if you should be mounting webwork2/htdocs/DATA/ from outside the Docker image!"
-      fi
+      echo "We will run OPL-update as the tagging-taxonomy.json file is missing in webwork2/htdocs/DATA/."
+      echo "Check if you should be mounting webwork2/htdocs/DATA/ from outside the Docker image!"
       touch "$APP_ROOT/libraries/Restore_or_build_OPL_tables"
     fi
     if [ -f "$APP_ROOT/libraries/Restore_or_build_OPL_tables" ]; then
       if [ ! -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then
-        # Download a saved version of the OPL sql table data to be loaded and extract
-        cd $WEBWORK_ROOT/bin/OPL_releases/
-        make extract
-        cp -r webwork-open-problem-library $APP_ROOT/libraries/
-        # check out commit corresponding to the release
-        make latest_release.tag
-        OPL_TAG=`cat $WEBWORK_ROOT/bin/OPL_releases/latest_release.tag`
-        cd $APP_ROOT/libraries/webwork-open-problem-library/
-        git fetch --tags origin
-        echo "Checking out tag $OPL_TAG"
-        git checkout $OPL_TAG
-      fi
-
-      if [ -f "$APP_ROOT/libraries/webwork-open-problem-library/TABLE-DUMP/OPL-tables.sql" ]; then
+        # Download the metadata and install it
+        export SKIP_UPLOAD_OPL_statistics=1
+        if [ ! -d $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED ]; then
+          mkdir $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED
+        fi
+        wait_for_db
+        $WEBWORK_ROOT/bin/OPL-update
+      else
         echo "Restoring OPL tables from the TABLE-DUMP/OPL-tables.sql file"
         wait_for_db
         $WEBWORK_ROOT/bin/restore-OPL-tables.pl
@@ -199,15 +187,6 @@ if [ "$1" = 'apache2' ]; then
           echo "You are missing some of the JSON files including tagging-taxonomy.json"
           echo "Some of the library functions will not work properly"
         fi
-      else
-        echo "About to start OPL-update. This takes a long time - please be patient."
-        wait_for_db
-        $WEBWORK_ROOT/bin/OPL-update
-        # Dump the OPL tables, to allow a quick restore in the future
-        $WEBWORK_ROOT/bin/dump-OPL-tables.pl
-        # Save a copy of the generated JSON files
-        mkdir -p $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED
-        cp -a $WEBWORK_ROOT/htdocs/DATA/*.json $APP_ROOT/libraries/webwork-open-problem-library/JSON-SAVED
       fi
       rm $APP_ROOT/libraries/Restore_or_build_OPL_tables
     fi