Skip to content

Commit

Permalink
generate.pl: overhaul to support building slim images
Browse files Browse the repository at this point in the history 
Do a long-needed overhaul of this script allow building of `slim`
variants:

- Make the base image configurable through a new `builds` setting.
- Rename the `64bit` image to `main`, as bit-ness no longer applies as
  we already build on multiple architectures.
- Rename `Releases.yaml` to the more general `config.yml`, allowing it
  to contain the supported image variants and Configure options.
- Run through perltidy 💄

The `slim` variants installs the minimum build-deps needed to build
Perl, and removes them after (save for make and netbase, to retain being
able to install CPAN modules through cpanm.)
  • Loading branch information
@zakame
zakame committed Aug 12, 2018
1 parent b4032ce commit 0cb4fc3
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 65 deletions.
21 changes: 15 additions & 6 deletions Releases.yaml → config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,39 +1,48 @@
---
builds:
- main
- slim

options:
common: "-Duseshrplib -Dvendorprefix=/usr/local"
threaded: "-Dusethreads"

releases:
- version: 5.8.9
sha256: 1097fbcd48ceccb2bc735d119c9db399a02a8ab9f7dc53e29e47e6a8d0d72e79
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.10.1
sha256: 9385f2c8c2ca8b1dc4a7c31903f1f8dc8f2ba867dc2a9e5c93012ed6b564e826
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.12.5
sha256: 10749417fd3010aae320a34181ad4cd6a4855c1fc63403b87fa4d630b18e966c
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.14.4
sha256: eece8c2b0d491bf6f746bd1f4f1bb7ce26f6b98e91c54690c617d7af38964745
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.16.3
sha256: bb7bc735e6813b177dcfccd480defcde7eddefa173b5967eac11babd1bfa98e8
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.18.4
sha256: 1fb4d27b75cd244e849f253320260efe1750641aaff4a18ce0d67556ff1b96a5
extra_flags: "-A ccflags=-fwrapv"
test_parallel: no
buildpack_deps: jessie
debian_release: jessie

- version: 5.20.3
sha256: 1b40068166c242e34a536836286e70b78410602a80615143301e52aa2901493b
Expand Down
187 changes: 128 additions & 59 deletions generate.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,68 @@
sub die_with_sample {
die <<EOF;
The Releases.yaml file must look roughly like:
The config.yml file must look roughly like:
releases:
- version: 5.20.0
sha256: asdasdadas
---
builds:
- main
- slim
Where version is the version number of Perl and sha256 is the SHA256 of
the tar.bz2 file.
options:
common: "-Duseshrplib -Dvendorprefix=/usr/local"
threaded: "-Dusethreads"
releases:
- version: 5.20.0
sha256: asdasdadas
Where "version" is the version number of Perl and "sha256" is the SHA256
of the Perl distribution tarball.
If needed or desired, extra_flags: can be added, which will be passed
verbatim to Configure.
Run "perldoc ./generate.pl" to read the complete documentation.
EOF
}

my $yaml = do {
open my $fh, "<", "Releases.yaml" or die "Couldn't open releases";
my $docker_slim_run_install = <<'EOF';
apt-get update \
&& apt-get install -y --no-install-recommends \
bzip2 \
ca-certificates \
# cpio \
curl \
dpkg-dev \
# file \
gcc \
# g++ \
# libbz2-dev \
# libdb-dev \
libc6-dev \
# libgdbm-dev \
# liblzma-dev \
make \
netbase \
patch \
# procps \
# zlib1g-dev \
xz-utils
EOF
chomp $docker_slim_run_install;

my $docker_slim_run_purge = <<'EOF';
savedPackages="make netbase" \
&& apt-mark auto '.*' > /dev/null \
&& apt-mark manual $savedPackages \
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -fr /var/cache/apt/* /var/lib/apt/lists/*
EOF
chomp $docker_slim_run_purge;

my $config = do {
open my $fh, '<', 'config.yml' or die "Couldn't open config";
local $/;
Load <$fh>;
};
Expand All @@ -35,43 +80,41 @@ sub die_with_sample {
<DATA>;
};

my $common = join " ", qw{
-Duseshrplib
-Dvendorprefix=/usr/local
};

my %builds = (
"64bit" => "$common",
"64bit,threaded" => "-Dusethreads $common",
);
my %builds;

# sha256 taken from http://www.cpan.org/authors/id/M/MI/MIYAGAWA/CHECKSUMS
my %cpanm = (
name => "App-cpanminus-1.7044",
url => "http://www.cpan.org/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.7044.tar.gz",
sha256 => "9b60767fe40752ef7a9d3f13f19060a63389a5c23acc3e9827e19b75500f81f3",
name => "App-cpanminus-1.7044",
url => "http://www.cpan.org/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.7044.tar.gz",
sha256 => "9b60767fe40752ef7a9d3f13f19060a63389a5c23acc3e9827e19b75500f81f3",
);

die_with_sample unless defined $yaml->{releases};
die_with_sample unless ref $yaml->{releases} eq "ARRAY";
die_with_sample unless defined $config->{releases};
die_with_sample unless ref $config->{releases} eq "ARRAY";

if (! -d "downloads") {
if (!-d "downloads") {
mkdir "downloads" or die "Couldn't create a downloads directory";
}

for my $release (@{$yaml->{releases}}) {
do { die_with_sample unless $release->{$_}} for (qw(version sha256));
for my $build (@{$config->{builds}}) {
$builds{$build} = $config->{options}{common};
$builds{"$build,threaded"} = "@{$config->{options}}{qw/threaded common/}";
}

for my $release (@{$config->{releases}}) {
do { die_with_sample unless $release->{$_} }
for (qw(version sha256));

die "Bad version: $release->{version}" unless $release->{version} =~ /\A5\.\d+\.\d+\Z/;

my $patch;
$release->{type} ||= 'bz2';
my $file = "perl-$release->{version}.tar.$release->{type}";
my $url = "https://www.cpan.org/src/5.0/$file";
if (-f "downloads/$file" &&
`sha256sum downloads/$file` =~ /^\Q$release->{sha256}\E\s+\Qdownloads\/$file\E/) {
print "Skipping download of $file, already current\n";
} else {
my $url = "https://www.cpan.org/src/5.0/$file";
if (-f "downloads/$file" && `sha256sum downloads/$file` =~ /^\Q$release->{sha256}\E\s+\Qdownloads\/$file\E/) {
print "Skipping download of $file, already current\n";
}
else {
print "Downloading $url\n";
getstore($url, "downloads/$file");
}
Expand All @@ -95,20 +138,37 @@ sub die_with_sample {
die "Couldn't create a Devel::PatchPerl patch for $release->{version}" if $? != 0;
}

$release->{url} = $url;
$release->{extra_flags} = "" unless defined $release->{extra_flags};
$release->{_tag} = $release->{buildpack_deps} || "stretch";
$release->{"cpanm_dist_$_"} = $cpanm{$_} for keys %cpanm;
for my $build (keys %builds) {
$release->{url} = $url;
$release->{"cpanm_dist_$_"} = $cpanm{$_} for keys %cpanm;

$release->{extra_flags} ||= '';
$release->{debian_release} ||= 'stretch';

if ($build =~ /main/) {
$release->{image} = 'buildpack-deps';
$release->{tag} = $release->{debian_release};
}
else {
$release->{image} = 'debian';
$release->{tag} = "@{[ $release->{debian_release} ]}-slim";
}

for my $config (keys %builds) {
my $output = $template;
$output =~ s/\{\{$_\}\}/$release->{$_}/mg
for (qw(version pause extra_flags sha256 type url _tag cpanm_dist_name cpanm_dist_url cpanm_dist_sha256));
$output =~ s/\{\{args\}\}/$builds{$config}/mg;
for (qw(version pause extra_flags sha256 type url image tag cpanm_dist_name cpanm_dist_url cpanm_dist_sha256));
$output =~ s/\{\{args\}\}/$builds{$build}/mg;

my $dir = sprintf "%i.%03i.%03i-%s",
($release->{version} =~ /(\d+)\.(\d+)\.(\d+)/),
$config;
if ($build =~ /slim/) {
$output =~ s/\{\{docker_slim_run_install\}\}/$docker_slim_run_install/mg;
$output =~ s/\{\{docker_slim_run_purge\}\}/$docker_slim_run_purge/mg;
}
else {
$output =~ s/\{\{docker_slim_run_install\}\}/true/mg;
$output =~ s/\{\{docker_slim_run_purge\}\}/true/mg;
}

my $dir = sprintf "%i.%03i.%03i-%s", ($release->{version} =~ /(\d+)\.(\d+)\.(\d+)/), $build;

mkdir $dir unless -d $dir;

Expand All @@ -119,11 +179,13 @@ sub die_with_sample {
}

if (defined $release->{test_parallel} && $release->{test_parallel} eq "no") {
$output =~ s/\{\{test\}\}/make test_harness/;
} elsif (!defined $release->{test_parallel} || $release->{test_parallel} eq "yes") {
$output =~ s/\{\{test\}\}/TEST_JOBS=\$(nproc) make test_harness/;
} else {
die "test_parallel was provided for $release->{version} but is invalid; should be 'yes' or 'no'\n";
$output =~ s/\{\{test\}\}/make test_harness/;
}
elsif (!defined $release->{test_parallel} || $release->{test_parallel} eq "yes") {
$output =~ s/\{\{test\}\}/TEST_JOBS=\$(nproc) make test_harness/;
}
else {
die "test_parallel was provided for $release->{version} but is invalid; should be 'yes' or 'no'\n";
}

open my $dockerfile, ">", "$dir/Dockerfile" or die "Couldn't open $dir/Dockerfile for writing";
Expand All @@ -136,17 +198,18 @@ sub die_with_sample {
=head1 NAME
generate.pl
generate.pl - generate Dockerfiles for Perl
=head1 SYNOPSIS
generate.pl is a little helper script to reinitalize the Dockerfiles from a YAML file.
cd /path/to/docker-perl
./generate.pl
=head1 DESCRIPTION
generate.pl is meant to be run from the actual repo directory, with a Releases.yaml file
correctly configured. It starts with a 'releases' key, which contains a list of releases,
each with the following keys:
generate.pl is meant to be run from the actual repo directory, with a
config.yml file correctly configured. It contains with a 'releases'
key, which contains a list of releases, each with the following keys:
=over 4
Expand All @@ -160,23 +223,27 @@ =head1 DESCRIPTION
=item sha256
The SHA-256 of the C<.tar.bz2> file for that release.
The SHA-256 of the tarball for that release.
=back
=item OPTIONAL
=over 4
=item buildpack_deps
=item debian_release
The Docker L<buildpack-deps|https://hub.docker.com/_/buildpack-deps>
image tag which this Perl would build on.
The Docker image tag which this Perl would build on, common to both the
L<https://hub.docker.com/_/buildpack-deps|buildpack-deps> and
L<https://hub.docker.com/_/debian|debian> Docker images.
Defaults: C<stretch>
Defaults: C<stretch> for C<main> builds, C<stretch-slim> for C<slim>
builds.
=item extra_flags
Additional text to pass to C<Configure>. At the moment, this is necessary for
5.18.x so that it can get the C<-fwrapv> flag.
Additional text to pass to C<Configure>. At the moment, this is
necessary for 5.18.x so that it can get the C<-fwrapv> flag.
Default: C<"">
Expand All @@ -195,13 +262,14 @@ =head1 DESCRIPTION
=cut

__DATA__
FROM buildpack-deps:{{_tag}}
FROM {{image}}:{{tag}}
LABEL maintainer="Peter Martini <PeterCMartini@GMail.com>, Zak B. Elep <zakame@cpan.org>"
COPY *.patch /usr/src/perl/
WORKDIR /usr/src/perl
RUN curl -SL {{url}} -o perl-{{version}}.tar.{{type}} \
RUN {{docker_slim_run_install}} \
&& curl -SL {{url}} -o perl-{{version}}.tar.{{type}} \
&& echo '{{sha256}} *perl-{{version}}.tar.{{type}}' | sha256sum -c - \
&& tar --strip-components=1 -xaf perl-{{version}}.tar.{{type}} -C /usr/src/perl \
&& rm perl-{{version}}.tar.{{type}} \
Expand All @@ -217,6 +285,7 @@ =head1 DESCRIPTION
&& curl -LO {{cpanm_dist_url}} \
&& echo '{{cpanm_dist_sha256}} *{{cpanm_dist_name}}.tar.gz' | sha256sum -c - \
&& tar -xzf {{cpanm_dist_name}}.tar.gz && cd {{cpanm_dist_name}} && perl bin/cpanm . && cd /root \
&& {{docker_slim_run_purge}} \
&& rm -fr ./cpanm /root/.cpanm /usr/src/perl /usr/src/{{cpanm_dist_name}}* /tmp/*
WORKDIR /root
Expand Down

0 comments on commit 0cb4fc3

Please sign in to comment.