From 4eff23fd0270ed08c727f1ddb10c497559732c22 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 24 Aug 2017 14:35:00 -0700 Subject: [PATCH] Add support for multiple architectures --- .architectures-lib | 30 ++++++++++++++++++++++ .travis.yml | 10 ++++++++ Dockerfile | 48 +++++++++++++++++++++++++---------- Dockerfile.template | 42 ++++++++++++++++++++++++++++++ generate-stackbrew-library.sh | 34 +++++++++++++++++++++++++ release-architectures | 9 +++++++ update.sh | 45 ++++++++++++++++++++++++++++---- 7 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 .architectures-lib create mode 100644 Dockerfile.template create mode 100644 release-architectures diff --git a/.architectures-lib b/.architectures-lib new file mode 100644 index 00000000..724a716c --- /dev/null +++ b/.architectures-lib @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +_awkArch() { + local version="$1"; shift + local awkExpr="$1"; shift + awk "$@" "/^#|^\$/ { next } $awkExpr" "$version/release-architectures" +} + +dpkgArches() { + local version="$1"; shift + _awkArch "$version" '{ print $2 }' +} + +hasBashbrewArch() { + local version="$1"; shift + local bashbrewArch="$1"; shift + _awkArch "$version" 'BEGIN { exitCode = 1 } $1 == bashbrewArch { exitCode = 0 } END { exit exitCode }' -v bashbrewArch="$bashbrewArch" +} + +dpkgToJuliaTarArch() { + local version="$1"; shift + local dpkgArch="$1"; shift + _awkArch "$version" '$2 == dpkgArch { print $3; exit }' -v dpkgArch="$dpkgArch" +} + +dpkgToJuliaDirArch() { + local version="$1"; shift + local dpkgArch="$1"; shift + _awkArch "$version" '$2 == dpkgArch { print $4; exit }' -v dpkgArch="$dpkgArch" +} diff --git a/.travis.yml b/.travis.yml index d1814bb7..68837ea2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,22 @@ language: bash services: docker +env: + - ARCH= + - ARCH=i386 + install: - git clone https://github.com/docker-library/official-images.git ~/official-images before_script: - env | sort - image='julia' + - | + if [ -n "$ARCH" ]; then + from="$(awk '$1 == toupper("FROM") { print $2 }' Dockerfile)" + docker pull "$ARCH/$from" + docker tag "$ARCH/$from" "$from" + fi script: - travis_retry docker build -t "$image" . diff --git a/Dockerfile b/Dockerfile index 238ef811..d9a21653 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,46 @@ FROM debian:jessie RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ && rm -rf /var/lib/apt/lists/* ENV JULIA_PATH /usr/local/julia -ENV JULIA_VERSION 0.6.0 -RUN mkdir $JULIA_PATH \ - && apt-get update && apt-get install -y curl \ - && curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -o julia.tar.gz \ - && curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz.asc" -o julia.tar.gz.asc \ - && export GNUPGHOME="$(mktemp -d)" \ -# http://julialang.org/juliareleases.asc +# https://julialang.org/juliareleases.asc # Julia (Binary signing key) - && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys 3673DF529D9049477F76B37566E3C7DC03D6E495 \ - && gpg --batch --verify julia.tar.gz.asc julia.tar.gz \ - && rm -r "$GNUPGHOME" julia.tar.gz.asc \ - && tar -xzf julia.tar.gz -C $JULIA_PATH --strip-components 1 \ - && rm -rf /var/lib/apt/lists/* julia.tar.gz* +ENV JULIA_GPG 3673DF529D9049477F76B37566E3C7DC03D6E495 + +# https://julialang.org/downloads/ +ENV JULIA_VERSION 0.6.0 + +RUN set -ex; \ + \ +# https://julialang.org/downloads/#julia-command-line-version +# https://julialang-s3.julialang.org/bin/checksums/julia-0.6.0.sha256 +# this "case" statement is generated via "update.sh" + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ + amd64) tarArch='x86_64'; dirArch='x64'; sha256='3a27ea78b06f46701dc4274820d9853789db205bce56afdc7147f7bd6fa83e41' ;; \ + armhf) tarArch='arm'; dirArch='arm'; sha256='7515f5977b2aac0cea1333ef249b3983928dee76ea8eb3de9dd6a7cdfbd07d2d' ;; \ + i386) tarArch='i686'; dirArch='x86'; sha256='bfebd2ef38c25ce72dd6661cdd8a6f509800492a4d250c2908f83e791c0a444a' ;; \ + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding Julia binary release"; exit 1 ;; \ + esac; \ + \ + curl -fL -o julia.tar.gz "https://julialang-s3.julialang.org/bin/linux/${dirArch}/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-${tarArch}.tar.gz"; \ + curl -fL -o julia.tar.gz.asc "https://julialang-s3.julialang.org/bin/linux/${dirArch}/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-${tarArch}.tar.gz.asc"; \ + \ + echo "${sha256} *julia.tar.gz" | sha256sum -c -; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$JULIA_GPG"; \ + gpg --batch --verify julia.tar.gz.asc julia.tar.gz; \ + rm -rf "$GNUPGHOME" julia.tar.gz.asc; \ + \ + mkdir "$JULIA_PATH"; \ + tar -xzf julia.tar.gz -C "$JULIA_PATH" --strip-components 1; \ + rm julia.tar.gz ENV PATH $JULIA_PATH/bin:$PATH diff --git a/Dockerfile.template b/Dockerfile.template new file mode 100644 index 00000000..75e155d1 --- /dev/null +++ b/Dockerfile.template @@ -0,0 +1,42 @@ +FROM debian:jessie + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +ENV JULIA_PATH /usr/local/julia + +# https://julialang.org/juliareleases.asc +# Julia (Binary signing key) +ENV JULIA_GPG 3673DF529D9049477F76B37566E3C7DC03D6E495 + +# https://julialang.org/downloads/ +ENV JULIA_VERSION %%JULIA_VERSION%% + +RUN set -ex; \ + \ +# https://julialang.org/downloads/#julia-command-line-version +# https://julialang-s3.julialang.org/bin/checksums/julia-%%JULIA_VERSION%%.sha256 +# this "case" statement is generated via "update.sh" + %%ARCH-CASE%%; \ + \ + curl -fL -o julia.tar.gz "https://julialang-s3.julialang.org/bin/linux/${dirArch}/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-${tarArch}.tar.gz"; \ + curl -fL -o julia.tar.gz.asc "https://julialang-s3.julialang.org/bin/linux/${dirArch}/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-${tarArch}.tar.gz.asc"; \ + \ + echo "${sha256} *julia.tar.gz" | sha256sum -c -; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$JULIA_GPG"; \ + gpg --batch --verify julia.tar.gz.asc julia.tar.gz; \ + rm -rf "$GNUPGHOME" julia.tar.gz.asc; \ + \ + mkdir "$JULIA_PATH"; \ + tar -xzf julia.tar.gz -C "$JULIA_PATH" --strip-components 1; \ + rm julia.tar.gz + + +ENV PATH $JULIA_PATH/bin:$PATH + +CMD ["julia"] diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 631f230f..10339a29 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -26,6 +26,39 @@ dirCommit() { ) } +getArches() { + local repo="$1"; shift + local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' + + eval "declare -g -A parentRepoToArches=( $( + find -name 'Dockerfile' -exec awk ' + toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|microsoft\/[^:]+)(:|$)/ { + print "'"$officialImagesUrl"'" $2 + } + ' '{}' + \ + | sort -u \ + | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' + ) )" +} +getArches 'julia' + +source '.architectures-lib' + +parentArches() { + local version="$1"; shift # "1.8", etc + + local parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/Dockerfile")" + local parentArches="${parentRepoToArches[$parent]:-}" + + local arches=() + for arch in $parentArches; do + if hasBashbrewArch "$version" "$arch"; then + arches+=( "$arch" ) + fi + done + echo "${arches[*]}" +} + cat <<-EOH # this file is generated via https://github.com/docker-library/julia/blob/$(fileCommit "$self")/$self @@ -55,5 +88,6 @@ versionAliases+=( $fullVersion latest ) echo cat <<-EOE Tags: $(join ', ' "${versionAliases[@]}") + Architectures: $(join ', ' $(parentArches .)) GitCommit: $commit EOE diff --git a/release-architectures b/release-architectures new file mode 100644 index 00000000..cfb18c5b --- /dev/null +++ b/release-architectures @@ -0,0 +1,9 @@ +# see https://julialang.org/downloads/#julia-command-line-version + +# bashbrew-arch dpkg-arch julia-tar-arch julia-dir-arch +amd64 amd64 x86_64 x64 +arm32v7 armhf arm arm +i386 i386 i686 x86 + +# ppc64le appears to be "0.6.0-rc3"-only as of 2017-08-24 +#ppc64le ppc64el ppc64le ppc64le diff --git a/update.sh b/update.sh index 6bc32923..5a4fcbf7 100755 --- a/update.sh +++ b/update.sh @@ -1,9 +1,44 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash +set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -pattern='.*\/julia-([0-9]+\.[0-9]+\.[0-9]+)-linux-x86_64\.tar\.gz.*' -version=$(curl -sSL 'http://julialang.org/downloads/' | sed -rn "s/${pattern}/\1/gp") -sed -ri 's/^(ENV JULIA_VERSION) .*/\1 '"$version"'/' "Dockerfile" +source '.architectures-lib' +# see http://stackoverflow.com/a/2705678/433558 +sed_escape_rhs() { + echo "$@" | sed -e 's/[\/&]/\\&/g' | sed -e ':a;N;$!ba;s/\n/\\n/g' +} + +for version in '.'; do + pattern='.*/julia-([0-9]+\.[0-9]+\.[0-9]+)-linux-x86_64\.tar\.gz.*' + fullVersion="$(curl -fsSL 'https://julialang.org/downloads/' | sed -rn "s!${pattern}!\1!gp")" + if [ -z "$fullVersion" ]; then + echo >&2 "error: failed to determine latest release for '$version'" + exit 1 + fi + + sha256s="$(curl -fsSL "https://julialang-s3.julialang.org/bin/checksums/julia-${fullVersion}.sha256")" + + linuxArchCase='dpkgArch="$(dpkg --print-architecture)"; '$'\\\n' + linuxArchCase+=$'\t''case "${dpkgArch##*-}" in '$'\\\n' + for dpkgArch in $(dpkgArches "$version"); do + tarArch="$(dpkgToJuliaTarArch "$version" "$dpkgArch")" + dirArch="$(dpkgToJuliaDirArch "$version" "$dpkgArch")" + sha256="$(echo "$sha256s" | grep "*julia-${fullVersion}-linux-${tarArch}.tar.gz$" | cut -d' ' -f1)" + if [ -z "$sha256" ]; then + echo >&2 "error: cannot find sha256 for $fullVersion on arch $tarArch / $dirArch ($dpkgArch)" + exit 1 + fi + linuxArchCase+=$'\t\t'"$dpkgArch) tarArch='$tarArch'; dirArch='$dirArch'; sha256='$sha256' ;; "$'\\\n' + done + linuxArchCase+=$'\t\t''*) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding Julia binary release"; exit 1 ;; '$'\\\n' + linuxArchCase+=$'\t''esac' + + echo "$version: $fullVersion" + + sed -r \ + -e 's!%%JULIA_VERSION%%!'"$fullVersion"'!g' \ + -e 's!%%ARCH-CASE%%!'"$(sed_escape_rhs "$linuxArchCase")"'!g' \ + Dockerfile.template > "$version/Dockerfile" +done