Skip to content
Permalink
Browse files

Use minio for integration testing.

ScalityS3 has not received any maintenance in years and is slow to start which is bad for testing.  Replace it with minio which starts quickly and ships as a single executable or a tiny container.

Minio has stricter limits on allowable characters but should still provide enough coverage to show that our encoding is working correctly.

This commit also includes the upgrade to openssl 1.1.1 in the Ubuntu 18.04 container.
  • Loading branch information...
dwsteele committed Jul 3, 2019
1 parent b9b2131 commit 1708f1d1514b3f2afb69664317df705a1fccfaf7
@@ -387,7 +387,7 @@ tlsClientRead(THIS_VOID, Buffer *buffer, bool block)
if (result <= 0)
{
// Break if the error indicates that we should not continue trying
if (!tlsError(this, SSL_get_error(this->session, (int)result))) // {uncovered - covered in next commit}
if (!tlsError(this, SSL_get_error(this->session, (int)result)))
break;
}
// Update amount of buffer used
@@ -13,8 +13,8 @@
# - docker tag pgbackrest/test:{vm}-base pgbackrest/test:{vm}-base-YYYYMMDDA
# - docker push pgbackrest/test:{vm}-base-YYYYMMDDA
# **********************************************************************************************************************************
20190424A:
u18: 72f9a453557e21dd3269b8b3fba1b4b8d6dc9d77
co6: 27a1efc146c200ad7592575808443bd086cf3e0d
co7: 5816e9219e7b5fb188331021c82559f9062de8b0
u12: 20b1b60cf8f456c03dc8ebadaf7d264fe1d9f957
20190701A:
u18: 83dfd805383db9b990f69647056638f2278f2325
co6: b3812cda61c33e959f309a9eceb23ef731fc0da2
co7: d645dbac2f094d980ac266094d0c26e84aca3ca6
u12: c919aa8562ff89713ee9ecf7dd41479798dde4d7

Large diffs are not rendered by default.

Large diffs are not rendered by default.

@@ -858,12 +858,12 @@ P00 ERROR: [055]: stop file does not exist for stanza 'db'
P00 INFO: stanza-delete command end: aborted with exception [055]

db must not exist for successful delete
> ls [TEST_PATH]/db-master/repo/backup/
> list backup
------------------------------------------------------------------------------------------------------------------------------------
db

db must not exist for successful delete
> ls [TEST_PATH]/db-master/repo/archive/
> list archive
------------------------------------------------------------------------------------------------------------------------------------
db

@@ -880,9 +880,9 @@ P00 INFO: stanza-delete command begin [BACKREST-VERSION]: --compress-level=3 -
P00 INFO: stanza-delete command end: completed successfully

db must not exist for successful delete
> ls [TEST_PATH]/db-master/repo/backup/
> list backup
------------------------------------------------------------------------------------------------------------------------------------

db must not exist for successful delete
> ls [TEST_PATH]/db-master/repo/archive/
> list archive
------------------------------------------------------------------------------------------------------------------------------------
@@ -702,12 +702,14 @@ P00 ERROR: [055]: stop file does not exist for stanza 'db'
P00 INFO: stanza-delete command end: aborted with exception [055]

db must not exist for successful delete
> ls //backup/
> list backup
------------------------------------------------------------------------------------------------------------------------------------
db

db must not exist for successful delete
> ls //archive/
> list archive
------------------------------------------------------------------------------------------------------------------------------------
db

stop db stanza (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stop
@@ -724,9 +726,9 @@ P00 INFO: http statistics:[HTTP-STATISTICS]
P00 INFO: stanza-delete command end: completed successfully

db must not exist for successful delete
> ls //backup/
> list backup
------------------------------------------------------------------------------------------------------------------------------------

db must not exist for successful delete
> ls //archive/
> list archive
------------------------------------------------------------------------------------------------------------------------------------
@@ -69,11 +69,6 @@ use constant CERT_FAKE_SERVER_KEY => CERT_FAKE
####################################################################################################################################
use constant CONTAINER_DEBUG => false;

####################################################################################################################################
# Container Debug - speeds container debugging by splitting each section into a separate intermediate container
####################################################################################################################################
use constant CONTAINER_S3_SERVER_TAG => 's3-server-20180612A';

####################################################################################################################################
# Store cache container checksums
####################################################################################################################################
@@ -303,51 +298,6 @@ sub certSetup
return $strScript;
}

####################################################################################################################################
# S3 server setup
####################################################################################################################################
sub s3ServerSetup
{
my $strOS = shift;

# Install node.js
my $strScript = sectionHeader() .
"# Install node.js\n";

if ($strOS eq VM_CO7)
{
$strScript .=
" wget -O /root/nodejs.sh https://rpm.nodesource.com/setup_6.x && \\\n" .
" bash /root/nodejs.sh && \\\n" .
" yum install -y nodejs";
}
else
{
$strScript .=
" wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \\\n" .
" bash /root/nodejs.sh && \\\n" .
" wget -qO- https://deb.nodesource.com/setup_8.x | bash - && \\\n" .
" apt-get install -y nodejs";
}

# Install Scality S3
$strScript .= sectionHeader() .
"# Install Scality S3\n";

$strScript .=
" wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \\\n" .
" mkdir /root/scalitys3 && \\\n" .
" tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \\\n" .
" cd /root/scalitys3 && \\\n" .
" npm install && \\\n" .
' sed -i "0,/,/s//,\n \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":' .
'\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/"' . " \\\n" .
' ./config.json' . " && \\\n" .
' sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json';

return $strScript;
}

####################################################################################################################################
# Entry point setup
####################################################################################################################################
@@ -426,17 +376,22 @@ sub containerBuild

#---------------------------------------------------------------------------------------------------------------------------
my $strScript = sectionHeader() .
"# Install base packages\n";
"# Install packages\n";

if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
{
if ($strOS eq VM_CO6 || $strOS eq VM_CO7)
{
$strScript .=
" yum -y install epel-release && \\\n";
}

$strScript .=
" yum -y install epel-release && \\\n" .
" yum -y update && \\\n" .
" yum -y install openssh-server openssh-clients wget sudo python-pip build-essential valgrind git \\\n" .
" perl perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL perl-YAML-LibYAML \\\n" .
" yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
" perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
" gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
" zlib-devel libxml2-devel lz4-devel";
" zlib-devel libxml2-devel lz4-devel lcov";

if ($strOS eq VM_CO6)
{
@@ -446,24 +401,16 @@ sub containerBuild
{
$strScript .= ' perl-JSON-PP';
}

if (vmCoverageC($strOS))
{
$strScript .= ' lcov';
}
}
else
{
$strScript .=
" export DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive && \\\n" .
" apt-get update && \\\n" .
" apt-get -y install wget python && \\\n" .
" wget --no-check-certificate -O /root/get-pip.py https://bootstrap.pypa.io/get-pip.py && \\\n" .
" python /root/get-pip.py && \\\n" .
" apt-get -y install openssh-server wget sudo python-pip build-essential valgrind git \\\n" .
" libdbd-pg-perl libhtml-parser-perl libio-socket-ssl-perl libxml-libxml-perl libssl-dev libperl-dev \\\n" .
" apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" .
" libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" .
" libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" .
" libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev";
" libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev lcov";

if ($strOS eq VM_U12)
{
@@ -478,11 +425,6 @@ sub containerBuild
{
$strScript .= ' clang-6.0 clang-tools-6.0';
}

if (vmCoverageC($strOS))
{
$strScript .= ' lcov';
}
}

#---------------------------------------------------------------------------------------------------------------------------
@@ -536,23 +478,25 @@ sub containerBuild

if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
{
$strScript .=
" rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG && \\\n";

if ($strOS eq VM_CO6)
{
$strScript .=
" rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG-10 && \\\n" .
" rpm -ivh \\\n" .
" http://yum.postgresql.org/9.0/redhat/rhel-6-x86_64/pgdg-centos90-9.0-5.noarch.rpm \\\n" .
" http://yum.postgresql.org/9.1/redhat/rhel-6-x86_64/pgdg-centos91-9.1-6.noarch.rpm \\\n" .
" http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/pgdg-centos92-9.2-8.noarch.rpm \\\n" .
" https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-6-x86_64/pgdg-redhat-repo-latest.noarch.rpm";
" https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-6-x86_64/" .
"pgdg-redhat-repo-latest.noarch.rpm";
}
elsif ($strOS eq VM_CO7)
{
$strScript .=
" rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG-10 && \\\n" .
" rpm -ivh \\\n" .
" http://yum.postgresql.org/9.2/redhat/rhel-7-x86_64/pgdg-centos92-9.2-3.noarch.rpm \\\n" .
" https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm";
" https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-7-x86_64/" .
"pgdg-redhat-repo-latest.noarch.rpm";
}
}
else
@@ -600,19 +544,14 @@ sub containerBuild
}
}

#---------------------------------------------------------------------------------------------------------------------------
if (!$bDeprecated)
{
$strScript .= sectionHeader() .
"# Install AWS CLI\n" .
" pip install --upgrade --no-cache-dir pip==9.0.3 && \\\n" .
" pip install --upgrade awscli";
}

#---------------------------------------------------------------------------------------------------------------------------
if (!$bDeprecated && $strOS ne VM_CO6 && $strOS ne VM_U12)
if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
{
$strScript .= s3ServerSetup($strOS);
$strScript .= sectionHeader() .
"# Cleanup\n";

$strScript .= " apt-get clean";
}

containerWrite(
@@ -685,33 +624,6 @@ sub containerBuild
"test/.vagrant/docker/${strOS}-${strPkgDevelCover}", "test/package/${strOS}-${strPkgDevelCover}");
}

# S3 image
###########################################################################################################################
if (!$bDeprecated)
{
$strImage = "${strOS}-s3-server";
$strScript = '';
$strCopy = undef;

$strScript = sectionHeader() .
"# Set worker clusters lower than the default for testing\n" .
" cd /root/scalitys3 && \\\n" .
' sed -i "s/clusters\"\: [0-9]*/clusters\"\: 2/" ./config.json';

if ($strOS ne VM_CO6 && $strOS ne VM_U12)
{
$strImageParent = containerRepo() . ":${strOS}-base";
$strScript .= "\n\nENTRYPOINT npm start --prefix /root/scalitys3";
}
else
{
$strImageParent = containerRepo() . ':' . CONTAINER_S3_SERVER_TAG;
}

containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'S3 Server', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
}

# Test image
########################################################################################################################
if (!$bDeprecated)
@@ -777,15 +689,6 @@ sub containerBuild
$strScript .=
sshSetup($strOS, TEST_USER, TEST_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER});

if (!$bDeprecated)
{
$strScript .= sectionHeader() .
"# Config AWS CLI\n" .
' sudo -i -u ' . TEST_USER . " aws configure set region us-east-1 && \\\n" .
' sudo -i -u ' . TEST_USER . " aws configure set aws_access_key_id accessKey1 && \\\n" .
' sudo -i -u ' . TEST_USER . " aws configure set aws_secret_access_key verySecretKey1";
}

$strScript .= sectionHeader() .
"# Create pgbackrest user\n" .
' ' . userCreate($strOS, BACKREST_USER, BACKREST_USER_ID, TEST_GROUP);
@@ -137,7 +137,6 @@ my $oyVm =

&VM_DB =>
[
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_94,
@@ -148,8 +147,8 @@ my $oyVm =

&VM_DB_TEST =>
[
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_94,
PG_VERSION_95,
],
@@ -266,7 +265,7 @@ my $oyVm =
[
PG_VERSION_83,
PG_VERSION_84,
PG_VERSION_92,
PG_VERSION_90,
PG_VERSION_93,
],
},
@@ -913,11 +913,11 @@ sub stanzaDelete
if (defined($self->{oLogTest}) && $self->synthetic())
{
$self->{oLogTest}->logAdd(
'ls ' . $self->repoPath() . '/backup/', $self->stanza() . ' must not exist for successful delete',
join("\n", storageRepo()->list('backup/')));
'list backup', $self->stanza() . ' must not exist for successful delete',
join("\n", storageRepo()->list('backup')));
$self->{oLogTest}->logAdd(
'ls ' . $self->repoPath() . '/archive/', $self->stanza() . ' must not exist for successful delete',
join("\n", storageRepo()->list('archive/')));
'list archive', $self->stanza() . ' must not exist for successful delete',
join("\n", storageRepo()->list('archive')));
}

# Return from function and log return values if any
@@ -11,6 +11,7 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);

use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
@@ -64,9 +65,16 @@ sub new
);

# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";

my $self = $class->SUPER::new(
HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', containerRepo() . ':' . testRunGet()->vm() . '-s3-server',
'root', testRunGet()->vm());
HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2019-06-04T01-15-58Z', 'root', 'u18',
["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro",
"${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"],
'-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' .
' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY,
'server /data --address :443 --compat', false);
bless $self, $class;

# Return from function and log return values if any

0 comments on commit 1708f1d

Please sign in to comment.
You can’t perform that action at this time.