Add icon caching to offline devfile registries

Signed-off-by: Angel Misevski <amisevsk@redhat.com>

arbitrary-users-patch/happy-path/build_happy_path_image.sh

@@ -28,7 +28,7 @@ if [ "$1" == "--push" ]; then
 fi
 
 # Build image for happy-path tests with precashed mvn dependencies
-docker build -t "${NAME_FORMAT}/happy-path:${TAG}" --no-cache --build-arg TAG=${TAG} "${SCRIPT_DIR}"/
+docker build -t "${NAME_FORMAT}/happy-path:${TAG}" --no-cache --build-arg TAG="${TAG}" "${SCRIPT_DIR}"/
 if ${PUSH_IMAGES}; then
     echo "Pushing ${NAME_FORMAT}/happy-path:${TAG}" to remote registry
     docker push "${NAME_FORMAT}/happy-path:${TAG}"

build/dockerfiles/Dockerfile

@@ -7,7 +7,7 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 FROM alpine:3.10 AS builder
-RUN apk add --no-cache py-pip jq bash git && pip install yq
+RUN apk add --no-cache py-pip jq bash wget git && pip install yq
 
 # Registry, organization, and tag to use for base images in dockerfiles. Devfiles
 # will be rewritten during build to use these values for base images.
@@ -38,7 +38,9 @@ CMD ["/usr/bin/run-httpd"]
 
 # Offline registry: download project zips and place them in /build/resources
 FROM builder AS offline-builder
-RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build
+RUN ./cache_projects.sh devfiles resources && \
+    ./cache_images.sh devfiles resources && \
+    chmod -R g+rwX /build
 
 # Offline registry: copy updated devfile.yamls and cached projects
 FROM registry AS offline-registry

build/dockerfiles/entrypoint.sh

@@ -33,6 +33,7 @@ PUBLIC_URL=${CHE_DEVFILE_REGISTRY_URL}
 
 DEFAULT_DEVFILES_DIR="/var/www/html/devfiles"
 DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}"
+INDEX_JSON="${DEVFILES_DIR}/index.json"
 
 # Regex used to break an image reference into groups:
 #   \1 - Whitespace and (optional) quotation preceding image reference
@@ -47,6 +48,7 @@ IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-
 # registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
 # The below command will fail if any path contains whitespace
 readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devfile.yaml')
+readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml')
 for devfile in "${devfiles[@]}"; do
   echo "Checking devfile $devfile"
   # Need to update each field separately in case they are not defined.
@@ -68,16 +70,17 @@ done
 if [ -n "$PUBLIC_URL" ]; then
   echo "Updating devfiles to point at internal project zip files"
   PUBLIC_URL=${PUBLIC_URL%/}
-  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}"
+  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
 else
   if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then
     echo "WARNING: environment variable 'CHE_DEVFILE_REGISTRY_URL' not configured" \
          "for an offline build of this registry. This may cause issues with importing" \
          "projects in a workspace."
     # Experimental workaround -- detect service IP for che-devfile-registry
-    # Depends on service used being named 'che-devfile-registry'
+    # Depends on service used being named 'che-devfile-registry' and only works
+    # within the cluster (i.e. browser-side retrieval won't work)
     URL="http://${CHE_DEVFILE_REGISTRY_SERVICE_HOST}:${CHE_DEVFILE_REGISTRY_SERVICE_PORT}"
-    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}"
+    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
   fi
 fi
 

build/dockerfiles/rhel.Dockerfile

@@ -98,7 +98,9 @@ CMD ["/usr/local/bin/rhel.entrypoint.sh"]
 
 # Offline devfile registry build
 FROM builder AS offline-builder
-RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build
+RUN ./cache_projects.sh devfiles resources && \
+    ./cache_images.sh devfiles resources && \
+    chmod -R g+rwX /build
 
 FROM registry AS offline-registry
 COPY --from=offline-builder /build/devfiles /var/www/html/devfiles

build/scripts/cache_images.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright (c) 2012-2018 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Arguments
+#    $1 - devfiles directory
+#    $2 - resources directory, where project zips will be stored.
+#
+# Only supports downloading projecst from GitHub.
+
+set -e
+
+DEVFILES_DIR="${1%/}"
+INDEX_JSON="${DEVFILES_DIR#./}/index.json"
+RESOURCES_DIR="${2%/}/images/"
+RESOURCES_DIR="${RESOURCES_DIR#\./}"
+TEMP_DIR="${RESOURCES_DIR%/}/temp_images"
+
+readarray -d '' metas < <(find "${DEVFILES_DIR#./}" -name 'meta.yaml' -print0)
+
+images=$(yq -S -r '.icon' "${metas[@]}" | sort | uniq)
+mkdir -p "$RESOURCES_DIR" "$TEMP_DIR"
+
+echo "Caching images referenced in devfiles"
+for image in "${images[@]}"; do
+  # Workaround for getting filenames through content-disposition: copy to temp
+  # dir and read filename before moving to /resources.
+  wget -P "${TEMP_DIR}" -nv --content-disposition "${image}"
+  file=$(find "${TEMP_DIR}" -type f)
+  filename=$(basename "${file}")
+
+  # Store downloaded image in resources dir, on subpath derived from URL (strip
+  # protocol and last portion)
+  image_dir="${image#*//}"
+  image_dir="${RESOURCES_DIR%/}/${image_dir%/*}"
+  mkdir -p "$image_dir"
+
+  cached_image="${image_dir%/}/${filename}"
+  mv "$file" "$cached_image"
+  echo "  Downloaded image $image to $cached_image"
+
+  cached_url="{{ DEVFILE_REGISTRY_URL }}/${cached_image#/}"
+  sed -i "s|${image}|${cached_url}|g" "${metas[@]}" "$INDEX_JSON"
+  echo "  Updated devfiles to point at cached image"
+done
+
+rm -rf "$TEMP_DIR"

build/scripts/cache_projects.sh

@@ -30,7 +30,7 @@ function cache_project() {
   local repo="$1"
   local branch="$2"
   local destination="$3"
-  git clone "$location" -b "$branch" --depth 1 "$TEMP_REPO" &>/dev/null
+  git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" &>/dev/null
   git archive "$branch" --remote="$TEMP_REPO" --format zip --output "$destination"
   rm -rf "$TEMP_REPO"
 }
@@ -82,7 +82,7 @@ function get_devfile_name() {
     else
       "unnamed-devfile"
     end
-  ' $devfile
+  ' "$devfile"
 }
 
 readarray -d '' devfiles < <(find "$DEVFILES_DIR" -name 'devfile.yaml' -print0)
@@ -103,12 +103,13 @@ for devfile in "${devfiles[@]}"; do
 
     location=$(echo "$project" | jq -r '.source.location')
     branch=$(echo "$project" | jq -r '.source.branch')
-    if [ -n $branch ]; then
+    if [ -n "$branch" ]; then
       branch="master"
     fi
     destination="${RESOURCES_DIR}/${devfile_name}-${project_name}-${branch}.zip"
-    echo "    Caching project to $(realpath $destination)"
-    cache_project "$location" "$branch" "$(realpath $destination)"
+    absolute_destination=$(realpath "$destination")
+    echo "    Caching project to $absolute_destination"
+    cache_project "$location" "$branch" "$absolute_destination"
 
     echo "    Updating devfile $devfile to point at cached project zip $destination"
     update_devfile "$devfile" "$project_name" "$destination"

build/scripts/check_mandatory_fields.sh

@@ -17,7 +17,7 @@ for meta in "${metas[@]}"; do
     echo "Checking devfile '${meta}'"
     unset NULL_OR_EMPTY_FIELDS
     for field in "${FIELDS[@]}"; do
-        if ! grep -q "^${field}:.*\S" $meta; then
+        if ! grep -q "^${field}:.*\S" "$meta"; then
             NULL_OR_EMPTY_FIELDS+="$field "
         fi
     done

build/scripts/index.sh

@@ -12,7 +12,7 @@ set -e
 
 readarray -d '' metas < <(find devfiles -name 'meta.yaml' -print0)
 for meta in "${metas[@]}"; do
-    META_DIR=$(dirname ${meta})
+    META_DIR=$(dirname "${meta}")
     # Workaround to include self-links, since it's not possible to
     # get filename in yq easily
     echo -e "links:\n  self: /${META_DIR}/devfile.yaml" >> "${meta}"

build/scripts/list_containers.sh

@@ -18,5 +18,5 @@ if [[ ! $1 ]]; then DIR=$(dirname "$0"); else DIR="$1"; fi
 
 # search in devfiles folder, eg., $1 = devfiles/
 echo "BEGIN list of external containers in $DIR folder:"
-yq -r '.components[].image | strings' ${DIR}/**/devfile.yaml | sort | uniq | sed "s/^/  /g"
+yq -r '.components[].image | strings' "${DIR}"/**/devfile.yaml | sort | uniq | sed "s/^/  /g"
 echo "END list of external containers in $DIR folder"