From 440813ff66d33a2ef44dbfe998e69b0f0b2b9c89 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Mon, 27 Jul 2020 17:11:59 +0200 Subject: [PATCH 1/2] [FLINK-16260] Change generate-stackbrew-library.sh to support new release.metadata file --- generate-stackbrew-library-docker.sh | 4 +- generate-stackbrew-library.sh | 153 ++++++++------------------- 2 files changed, 45 insertions(+), 112 deletions(-) diff --git a/generate-stackbrew-library-docker.sh b/generate-stackbrew-library-docker.sh index 7b39013d..c8c2a63b 100755 --- a/generate-stackbrew-library-docker.sh +++ b/generate-stackbrew-library-docker.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash exec docker run --rm \ --volume "${PWD}:/build:ro" \ - plucas/docker-flink-build \ + rmetzger/git-and-bash:latest \ /build/generate-stackbrew-library.sh diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 25d349d3..9f3a9061 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script generates a manifest compatibile with the expectations set forth # by docker-library/official-images. @@ -8,16 +8,11 @@ set -eu -declare -A aliases=( - [1.11]='latest' -) - self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -# Identify directories matching '?.?' (e.g. '1.7') and remove trailing slashes -versions=( ?.?/ ?.??/ ) -versions=( "${versions[@]%/}" ) +# remove "latest" and any "scala_" tag, unless it is the latest version +PRUNE_FROM_NON_LATEST_VERSION="^(latest$|scala\_/*)" # get the most recent commit which modified any of "$@" fileCommit() { @@ -41,21 +36,39 @@ dirCommit() { ) } -getArches() { - local repo="$1"; shift - local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' +# Inputs: +# - tags: comma-seprated list of image tags +# - latestVersion: latest version +# Output: comma-separated list of tags with "latest" removed if not latest version +pruneTags() { + local inTagsString=$1 + local latestVersion=$2 + if [[ $inTagsString =~ $latestVersion ]]; then + # tagsString contains latest version. keep "latest" tag + echo $inTagsString + else + # split list of tags, remove anything containing "latest" + IFS=', ' read -r -a inTags <<< "$inTagsString" + local outString="" + for inTag in "${inTags[@]}"; do + if [[ $inTag =~ $PRUNE_FROM_NON_LATEST_VERSION ]]; then + continue; + fi + outString="$outString, $inTag" + done + echo ${outString:2} + fi +} - eval "declare -g -A parentRepoToArches=( $( - find -name 'Dockerfile' -exec awk ' - toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|microsoft\/[^:]+)(:|$)/ { - print "'"$officialImagesUrl"'" $2 - } - ' '{}' + \ - | sort -u \ - | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' - ) )" +extractValue() { + local key="$1" + local file="$2" + local line=$(cat $file | grep "$key:") + echo $line | cut -d ':' -f 2 | xargs # remove key from line, remove whitespace } -getArches 'flink' + +# get latest flink version +latest_version=`ls -1a | grep -E "[0-9]+.[0-9]+" | sort -V -r | head -n 1` cat <<-EOH # this file is generated via https://github.com/apache/flink-docker/blob/$(fileCommit "$self")/$self @@ -65,105 +78,25 @@ Maintainers: Patrick Lucas (@patricklucas), GitRepo: https://github.com/apache/flink-docker.git EOH -# prints "$2$1$3$1...$N" -join() { - local sep="$1"; shift - local out; printf -v out "${sep//%/%%}%s" "$@" - echo "${out#$sep}" -} - -# Sorry for the style here, but it makes the nested code easier to read -for version in "${versions[@]}"; do - -# Defaults, can vary between versions -source_variants=( debian ) -scala_versions=( 2.11 2.12 ) -# Version-specific variants (example) -# if [ "$flink_release" = "x.y" ]; then -# scala_versions=( 2.10 2.11 2.12 ) -# fi - -for source_variant in "${source_variants[@]}"; do -for scala_version in "${scala_versions[@]}"; do - dir="$version/scala_${scala_version}-${source_variant}" - - # Not all variant combinations may exist - [ -f "$dir/Dockerfile" ] || continue +for dockerfile in $(find . -name "Dockerfile"); do + dir=$(dirname $dockerfile) commit="$(dirCommit "$dir")" + metadata="$dir/release.metadata" + architectures=$(extractValue "Architectures" $metadata) + tags=$(extractValue "Tags" $metadata) + tags=$(pruneTags "$tags" $latest_version) - # Extract the full Flink version from the Dockerfile - flink_version="$(git show "$commit":"$dir/Dockerfile" | grep 'dist/flink/flink-[0-9.]*' | cut -d/ -f6 | cut -d- -f2)" - - full_version=$flink_version-scala_$scala_version - - variantParent="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile")" - variantArches="${parentRepoToArches[$variantParent]}" - - # Start with the full version e.g. "1.2.0-scala_2.11" and add - # additional tags as relevant - tags=( $full_version ) - - is_latest_version= - [ "$version" = "${versions[-1]}" ] && is_latest_version=1 - - is_latest_scala= - [ "$scala_version" = "${scala_versions[-1]}" ] && is_latest_scala=1 - - add_tags=( $version ) - - # Add a scala version tag to each image - tags=( - ${tags[@]} - ${add_tags[@]/%/-scala_$scala_version} - ) - - # If this is the latest Flink release, add a tag with only the scala version - if [ -n "$is_latest_version" ]; then - tags=( - ${tags[@]} - "scala_$scala_version" - ) - fi - - # For the latest supported Scala version, add tags that omit the scala version - if [ -n "$is_latest_scala" ]; then - tags=( - ${tags[@]} - $flink_version - $version - ) - fi - - # Add -$variant suffix for non-debian-based images - if [ "$source_variant" != "debian" ]; then - tags=( ${tags[@]/%/-$source_variant} ) - fi - - # Finally, designate the 'latest' tag (or '$variant', for non-debian-based images) - if [ -n "$is_latest_scala" ]; then - alias_tag="${aliases[$version]:-}" - if [ -n "$alias_tag" ] && [ "$source_variant" != "debian" ]; then - alias_tag="$source_variant" - fi - - tags=( - ${tags[@]} - $alias_tag - ) - fi - + # newline echo # The tabs here are necessary for the heredoc to work right cat <<-EOE - Tags: $(join ', ' "${tags[@]}") - Architectures: $(join ', ' $variantArches) + Tags: $tags + Architectures: $architectures GitCommit: $commit Directory: $dir EOE done -done -done From 0264b7e7b3ef7896803405e6c4e919884626ab20 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 28 Jul 2020 15:44:46 +0200 Subject: [PATCH 2/2] address PR comments --- generate-stackbrew-library.sh | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 9f3a9061..a8f6766a 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -11,8 +11,6 @@ set -eu self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -# remove "latest" and any "scala_" tag, unless it is the latest version -PRUNE_FROM_NON_LATEST_VERSION="^(latest$|scala\_/*)" # get the most recent commit which modified any of "$@" fileCommit() { @@ -47,16 +45,9 @@ pruneTags() { # tagsString contains latest version. keep "latest" tag echo $inTagsString else - # split list of tags, remove anything containing "latest" - IFS=', ' read -r -a inTags <<< "$inTagsString" - local outString="" - for inTag in "${inTags[@]}"; do - if [[ $inTag =~ $PRUNE_FROM_NON_LATEST_VERSION ]]; then - continue; - fi - outString="$outString, $inTag" - done - echo ${outString:2} + # remove "latest" and any "scala_" tag, unless it is the latest version + # the "scala_" tag has a similar semantic as the "latest" tag in docker registries. + echo $inTagsString | sed -E 's|,(scala\|latest)[-_[:alnum:]]*||g' fi } @@ -64,7 +55,7 @@ extractValue() { local key="$1" local file="$2" local line=$(cat $file | grep "$key:") - echo $line | cut -d ':' -f 2 | xargs # remove key from line, remove whitespace + echo $line | cut -d ':' -f 2 | tr -d ' ' # remove key from line, remove whitespace } # get latest flink version