Use minio for integration testing.

ScalityS3 has not received any maintenance in years and is slow to start which is bad for testing. Replace it with minio which starts quickly and ships as a single executable or a tiny container. Minio has stricter limits on allowable characters but should still provide enough coverage to show that our encoding is working correctly. This commit also includes the upgrade to openssl 1.1.1 in the Ubuntu 18.04 container.
pgbackrest · Jul 3, 2019 · 1708f1d · 1708f1d
1 parent b9b2131
commit 1708f1d
Show file tree

Hide file tree

Showing 14 changed files with 136 additions and 251 deletions.
diff --git a/src/common/io/tls/client.c b/src/common/io/tls/client.c
@@ -387,7 +387,7 @@ tlsClientRead(THIS_VOID, Buffer *buffer, bool block)
         if (result <= 0)
         {
             // Break if the error indicates that we should not continue trying
-            if (!tlsError(this, SSL_get_error(this->session, (int)result)))                 // {uncovered - covered in next commit}
+            if (!tlsError(this, SSL_get_error(this->session, (int)result)))
                 break;
         }
         // Update amount of buffer used

diff --git a/test/container.yaml b/test/container.yaml
@@ -13,8 +13,8 @@
 # - docker tag pgbackrest/test:{vm}-base pgbackrest/test:{vm}-base-YYYYMMDDA
 # - docker push pgbackrest/test:{vm}-base-YYYYMMDDA
 # **********************************************************************************************************************************
-20190424A:
-  u18: 72f9a453557e21dd3269b8b3fba1b4b8d6dc9d77
-  co6: 27a1efc146c200ad7592575808443bd086cf3e0d
-  co7: 5816e9219e7b5fb188331021c82559f9062de8b0
-  u12: 20b1b60cf8f456c03dc8ebadaf7d264fe1d9f957
+20190701A:
+  u18: 83dfd805383db9b990f69647056638f2278f2325
+  co6: b3812cda61c33e959f309a9eceb23ef731fc0da2
+  co7: d645dbac2f094d980ac266094d0c26e84aca3ca6
+  u12: c919aa8562ff89713ee9ecf7dd41479798dde4d7
diff --git a/test/expect/mock-all-001.log b/test/expect/mock-all-001.log
diff --git a/test/expect/mock-all-002.log b/test/expect/mock-all-002.log
diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log
@@ -858,12 +858,12 @@ P00  ERROR: [055]: stop file does not exist for stanza 'db'
 P00   INFO: stanza-delete command end: aborted with exception [055]
 
 db must not exist for successful delete
-> ls [TEST_PATH]/db-master/repo/backup/
+> list backup
 ------------------------------------------------------------------------------------------------------------------------------------
 db
 
 db must not exist for successful delete
-> ls [TEST_PATH]/db-master/repo/archive/
+> list archive
 ------------------------------------------------------------------------------------------------------------------------------------
 db
 
@@ -880,9 +880,9 @@ P00   INFO: stanza-delete command begin [BACKREST-VERSION]: --compress-level=3 -
 P00   INFO: stanza-delete command end: completed successfully
 
 db must not exist for successful delete
-> ls [TEST_PATH]/db-master/repo/backup/
+> list backup
 ------------------------------------------------------------------------------------------------------------------------------------
 
 db must not exist for successful delete
-> ls [TEST_PATH]/db-master/repo/archive/
+> list archive
 ------------------------------------------------------------------------------------------------------------------------------------
diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log
@@ -702,12 +702,14 @@ P00  ERROR: [055]: stop file does not exist for stanza 'db'
 P00   INFO: stanza-delete command end: aborted with exception [055]
 
 db must not exist for successful delete
-> ls //backup/
+> list backup
 ------------------------------------------------------------------------------------------------------------------------------------
+db
 
 db must not exist for successful delete
-> ls //archive/
+> list archive
 ------------------------------------------------------------------------------------------------------------------------------------
+db
 
 stop db stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stop
@@ -724,9 +726,9 @@ P00   INFO: http statistics:[HTTP-STATISTICS]
 P00   INFO: stanza-delete command end: completed successfully
 
 db must not exist for successful delete
-> ls //backup/
+> list backup
 ------------------------------------------------------------------------------------------------------------------------------------
 
 db must not exist for successful delete
-> ls //archive/
+> list archive
 ------------------------------------------------------------------------------------------------------------------------------------
diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm
@@ -69,11 +69,6 @@ use constant CERT_FAKE_SERVER_KEY                                   => CERT_FAKE
 ####################################################################################################################################
 use constant CONTAINER_DEBUG                                        => false;
 
-####################################################################################################################################
-# Container Debug - speeds container debugging by splitting each section into a separate intermediate container
-####################################################################################################################################
-use constant CONTAINER_S3_SERVER_TAG                                => 's3-server-20180612A';
-
 ####################################################################################################################################
 # Store cache container checksums
 ####################################################################################################################################
@@ -303,51 +298,6 @@ sub certSetup
     return $strScript;
 }
 
-####################################################################################################################################
-# S3 server setup
-####################################################################################################################################
-sub s3ServerSetup
-{
-    my $strOS = shift;
-
-    # Install node.js
-    my $strScript = sectionHeader() .
-        "# Install node.js\n";
-
-    if ($strOS eq VM_CO7)
-    {
-        $strScript .=
-            "    wget -O /root/nodejs.sh https://rpm.nodesource.com/setup_6.x && \\\n" .
-            "    bash /root/nodejs.sh && \\\n" .
-            "    yum install -y nodejs";
-    }
-    else
-    {
-        $strScript .=
-            "    wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \\\n" .
-            "    bash /root/nodejs.sh && \\\n" .
-            "    wget -qO- https://deb.nodesource.com/setup_8.x | bash - && \\\n" .
-            "    apt-get install -y nodejs";
-    }
-
-    # Install Scality S3
-    $strScript .= sectionHeader() .
-        "# Install Scality S3\n";
-
-    $strScript .=
-        "    wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \\\n" .
-        "    mkdir /root/scalitys3 && \\\n" .
-        "    tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \\\n" .
-        "    cd /root/scalitys3 && \\\n" .
-        "    npm install && \\\n" .
-        '    sed -i "0,/,/s//,\n    \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":' .
-            '\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/"' . " \\\n" .
-        '        ./config.json' . " && \\\n" .
-        '    sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json';
-
-    return $strScript;
-}
-
 ####################################################################################################################################
 # Entry point setup
 ####################################################################################################################################
@@ -426,17 +376,22 @@ sub containerBuild
 
         #---------------------------------------------------------------------------------------------------------------------------
         my $strScript = sectionHeader() .
-            "# Install base packages\n";
+            "# Install packages\n";
 
         if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
         {
+            if ($strOS eq VM_CO6 || $strOS eq VM_CO7)
+            {
+                $strScript .=
+                    "    yum -y install epel-release && \\\n";
+            }
+
             $strScript .=
-                "    yum -y install epel-release && \\\n" .
                 "    yum -y update && \\\n" .
-                "    yum -y install openssh-server openssh-clients wget sudo python-pip build-essential valgrind git \\\n" .
-                "        perl perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL perl-YAML-LibYAML \\\n" .
+                "    yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
+                "        perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
                 "        gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
-                "        zlib-devel libxml2-devel lz4-devel";
+                "        zlib-devel libxml2-devel lz4-devel lcov";
 
             if ($strOS eq VM_CO6)
             {
@@ -446,24 +401,16 @@ sub containerBuild
             {
                 $strScript .= ' perl-JSON-PP';
             }
-
-            if (vmCoverageC($strOS))
-            {
-                $strScript .= ' lcov';
-            }
         }
         else
         {
             $strScript .=
                 "    export DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive && \\\n" .
                 "    apt-get update && \\\n" .
-                "    apt-get -y install wget python && \\\n" .
-                "    wget --no-check-certificate -O /root/get-pip.py https://bootstrap.pypa.io/get-pip.py && \\\n" .
-                "    python /root/get-pip.py && \\\n" .
-                "    apt-get -y install openssh-server wget sudo python-pip build-essential valgrind git \\\n" .
-                "        libdbd-pg-perl libhtml-parser-perl libio-socket-ssl-perl libxml-libxml-perl libssl-dev libperl-dev \\\n" .
+                "    apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" .
+                "        libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" .
                 "        libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" .
-                "        libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev";
+                "        libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev lcov";
 
             if ($strOS eq VM_U12)
             {
@@ -478,11 +425,6 @@ sub containerBuild
             {
                 $strScript .= ' clang-6.0 clang-tools-6.0';
             }
-
-            if (vmCoverageC($strOS))
-            {
-                $strScript .= ' lcov';
-            }
         }
 
         #---------------------------------------------------------------------------------------------------------------------------
@@ -536,23 +478,25 @@ sub containerBuild
 
             if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
             {
+                $strScript .=
+                    "    rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG && \\\n";
+
                 if ($strOS eq VM_CO6)
                 {
                     $strScript .=
-                        "    rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG-10 && \\\n" .
                         "    rpm -ivh \\\n" .
-                        "        http://yum.postgresql.org/9.0/redhat/rhel-6-x86_64/pgdg-centos90-9.0-5.noarch.rpm \\\n" .
                         "        http://yum.postgresql.org/9.1/redhat/rhel-6-x86_64/pgdg-centos91-9.1-6.noarch.rpm \\\n" .
                         "        http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/pgdg-centos92-9.2-8.noarch.rpm \\\n" .
-                        "        https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-6-x86_64/pgdg-redhat-repo-latest.noarch.rpm";
+                        "        https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-6-x86_64/" .
+                            "pgdg-redhat-repo-latest.noarch.rpm";
                 }
                 elsif ($strOS eq VM_CO7)
                 {
                     $strScript .=
-                        "    rpm --import http://yum.postgresql.org/RPM-GPG-KEY-PGDG-10 && \\\n" .
                         "    rpm -ivh \\\n" .
                         "        http://yum.postgresql.org/9.2/redhat/rhel-7-x86_64/pgdg-centos92-9.2-3.noarch.rpm \\\n" .
-                        "        https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm";
+                        "        https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-7-x86_64/" .
+                            "pgdg-redhat-repo-latest.noarch.rpm";
                 }
             }
             else
@@ -600,19 +544,14 @@ sub containerBuild
             }
         }
 
-        #---------------------------------------------------------------------------------------------------------------------------
-        if (!$bDeprecated)
-        {
-            $strScript .= sectionHeader() .
-                "# Install AWS CLI\n" .
-                "    pip install --upgrade --no-cache-dir pip==9.0.3 && \\\n" .
-                "    pip install --upgrade awscli";
-        }
 
         #---------------------------------------------------------------------------------------------------------------------------
-        if (!$bDeprecated && $strOS ne VM_CO6 && $strOS ne VM_U12)
+        if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
         {
-            $strScript .= s3ServerSetup($strOS);
+        $strScript .=  sectionHeader() .
+            "# Cleanup\n";
+
+            $strScript .= "    apt-get clean";
         }
 
         containerWrite(
@@ -685,33 +624,6 @@ sub containerBuild
                 "test/.vagrant/docker/${strOS}-${strPkgDevelCover}", "test/package/${strOS}-${strPkgDevelCover}");
         }
 
-        # S3 image
-        ###########################################################################################################################
-        if (!$bDeprecated)
-        {
-            $strImage = "${strOS}-s3-server";
-            $strScript = '';
-            $strCopy = undef;
-
-            $strScript = sectionHeader() .
-                "# Set worker clusters lower than the default for testing\n" .
-                "    cd /root/scalitys3 && \\\n" .
-                '    sed -i "s/clusters\"\: [0-9]*/clusters\"\: 2/" ./config.json';
-
-            if ($strOS ne VM_CO6 && $strOS ne VM_U12)
-            {
-                $strImageParent = containerRepo() . ":${strOS}-base";
-                $strScript .= "\n\nENTRYPOINT npm start --prefix /root/scalitys3";
-            }
-            else
-            {
-                $strImageParent = containerRepo() . ':' . CONTAINER_S3_SERVER_TAG;
-            }
-
-            containerWrite(
-                $oStorageDocker, $strTempPath, $strOS, 'S3 Server', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
-        }
-
         # Test image
         ########################################################################################################################
         if (!$bDeprecated)
@@ -777,15 +689,6 @@ sub containerBuild
             $strScript .=
                 sshSetup($strOS, TEST_USER, TEST_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER});
 
-            if (!$bDeprecated)
-            {
-                $strScript .= sectionHeader() .
-                    "# Config AWS CLI\n" .
-                    '    sudo -i -u ' . TEST_USER . " aws configure set region us-east-1 && \\\n" .
-                    '    sudo -i -u ' . TEST_USER . " aws configure set aws_access_key_id accessKey1 && \\\n" .
-                    '    sudo -i -u ' . TEST_USER . " aws configure set aws_secret_access_key verySecretKey1";
-            }
-
             $strScript .= sectionHeader() .
                 "# Create pgbackrest user\n" .
                 '    ' . userCreate($strOS, BACKREST_USER, BACKREST_USER_ID, TEST_GROUP);

diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm
@@ -137,7 +137,6 @@ my $oyVm =
 
         &VM_DB =>
         [
-            PG_VERSION_90,
             PG_VERSION_91,
             PG_VERSION_92,
             PG_VERSION_94,
@@ -148,8 +147,8 @@ my $oyVm =
 
         &VM_DB_TEST =>
         [
-            PG_VERSION_90,
             PG_VERSION_91,
+            PG_VERSION_92,
             PG_VERSION_94,
             PG_VERSION_95,
         ],
@@ -266,7 +265,7 @@ my $oyVm =
         [
             PG_VERSION_83,
             PG_VERSION_84,
-            PG_VERSION_92,
+            PG_VERSION_90,
             PG_VERSION_93,
         ],
     },

diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
@@ -913,11 +913,11 @@ sub stanzaDelete
     if (defined($self->{oLogTest}) && $self->synthetic())
     {
         $self->{oLogTest}->logAdd(
-            'ls ' . $self->repoPath() . '/backup/', $self->stanza() . ' must not exist for successful delete',
-            join("\n", storageRepo()->list('backup/')));
+            'list backup', $self->stanza() . ' must not exist for successful delete',
+            join("\n", storageRepo()->list('backup')));
         $self->{oLogTest}->logAdd(
-            'ls ' . $self->repoPath() . '/archive/', $self->stanza() . ' must not exist for successful delete',
-            join("\n", storageRepo()->list('archive/')));
+            'list archive', $self->stanza() . ' must not exist for successful delete',
+            join("\n", storageRepo()->list('archive')));
     }
 
     # Return from function and log return values if any

diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm
@@ -11,6 +11,7 @@ use strict;
 use warnings FATAL => qw(all);
 use Carp qw(confess);
 
+use Cwd qw(abs_path);
 use Exporter qw(import);
     our @EXPORT = qw();
 use File::Basename qw(dirname);
@@ -64,9 +65,16 @@ sub new
         );
 
     # Create the host
+    my $strProjectPath = dirname(dirname(abs_path($0)));
+    my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";
+
     my $self = $class->SUPER::new(
-        HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', containerRepo() . ':' . testRunGet()->vm() . '-s3-server',
-        'root', testRunGet()->vm());
+        HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2019-06-04T01-15-58Z', 'root', 'u18',
+        ["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro",
+            "${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"],
+        '-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' .
+            ' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY,
+        'server /data --address :443 --compat', false);
     bless $self, $class;
 
     # Return from function and log return values if any