Skip to content
Permalink
Browse files

Use minio as local S3 emulator in documentation.

The documentation was relying on a ScalityS3 container built for testing which wasn't very transparent.  Instead, use the stock minio container and configure it in the documentation.

Also, install certificates and CA so that TLS verification can be enabled.
  • Loading branch information...
dwsteele committed May 27, 2019
1 parent a474ba5 commit 3e1b06acaa84399abcfaa8c684f437b63aa38de5
@@ -630,7 +630,6 @@ sub backrestConfig
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'});

if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
{
@@ -1086,15 +1085,15 @@ sub sectionChildProcess
$self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
defined($oChild->paramGet('mount', false)) ?
[$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,
$$hCacheKey{option}, $$hCacheKey{param});
$$hCacheKey{option}, $$hCacheKey{param}, $$hCacheKey{'update-hosts'});

$self->{host}{$$hCacheKey{name}} = $oHost;
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true);
$$hCacheValue{ip} = $oHost->{strIP};

# Add to the host group
my $oHostGroup = hostGroupGet();
$oHostGroup->hostAdd($oHost, {bUpdateHosts => $$hCacheKey{'update-hosts'}});
$oHostGroup->hostAdd($oHost);

# Execute initialize commands
foreach my $oExecute ($oChild->nodeList('execute', false))
@@ -0,0 +1,2 @@
*.csr
*.srl
@@ -0,0 +1,28 @@
# pgBackRest Documentation Certificates

The certificates in this directory are used for documentation generation only and should not be used for actual services.

## pgBackRest CA

Generate a CA that will be used to sign documentation certificates. It can be installed in the documentation containers to make certificates signed by it valid.

```
cd [pgbackrest-root]/doc/resource/fake-cert
openssl ecparam -genkey -name prime256v1 | openssl ec -out ca.key
openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \
-subj "/C=US/ST=All/L=All/O=pgBackRest/CN=pgbackrest.org"
```

## S3 Certificate

Mimic an S3 certificate for the `us-east-1`/`us-east-2` region to generate S3 documentation.

```
cd [pgbackrest-root]/doc/resource/fake-cert
openssl ecparam -genkey -name prime256v1 | openssl ec -out s3-server.key
openssl req -new -sha256 -nodes -out s3-server.csr -key s3-server.key -config s3.cnf
openssl x509 -req -in s3-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
-out s3-server.crt -days 99999 -extensions v3_req -extfile s3.cnf
```
@@ -0,0 +1,13 @@
-----BEGIN CERTIFICATE-----
MIIB+jCCAaCgAwIBAgIJAJDUUhiBUbmEMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT
AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr
UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDAxOTU5WhgP
MjI5MzAzMTAwMDE5NTlaMFcxCzAJBgNVBAYTAlVTMQwwCgYDVQQIDANBbGwxDDAK
BgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNrUmVzdDEXMBUGA1UEAwwOcGdiYWNr
cmVzdC5vcmcwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQYHUcSknRDL+fgFJZI
IC73Ju75yA0203IxPO35i8mVb9CcWVhEgHmS+cQ6SfY6GC7V61VB7gwzQ+XESi2p
ndhJo1MwUTAdBgNVHQ4EFgQUYMbKIlTUE6gklw8KcSC6fnlOitwwHwYDVR0jBBgw
FoAUYMbKIlTUE6gklw8KcSC6fnlOitwwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjO
PQQDAgNIADBFAiEA1Bzy17/6jQimg3ROZTrVGkRtAuzTtjgDParHFrIhSDoCIH43
OeOUaPVb0rXGPLu9rFpjPOmtFSW3lf4skheJMKyN
-----END CERTIFICATE-----
@@ -0,0 +1,5 @@
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIB5f3SxfiZ92GMpuqpfTiPO3xaVOnxRh6qVAoRtu7NOZoAoGCCqGSM49
AwEHoUQDQgAEGB1HEpJ0Qy/n4BSWSCAu9ybu+cgNNtNyMTzt+YvJlW/QnFlYRIB5
kvnEOkn2Ohgu1etVQe4MM0PlxEotqZ3YSQ==
-----END EC PRIVATE KEY-----
@@ -0,0 +1,16 @@
-----BEGIN CERTIFICATE-----
MIICbTCCAhOgAwIBAgIJAODTXyGnxWtVMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT
AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr
UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDIwODEwWhgP
MjI5MzAzMTAwMjA4MTBaMIGBMQswCQYDVQQGEwJVUzEMMAoGA1UECAwDQWxsMQww
CgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jlc3QxHDAaBgNVBAsME1VuaXQg
VGVzdGluZyBEb21haW4xIzAhBgNVBAMMGnMzLnVzLWVhc3QtMS5hbWF6b25hd3Mu
Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEe2dO1v1gE0Qj4H407i0K8tN
kASkveckACPFzXs2i/++rZY4bwUub08JcMRv0WWwnRzOoumsN26Ge454vTbjoqOB
mjCBlzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DB9BgNVHREEdjB0ghpzMy51cy1l
YXN0LTEuYW1hem9uYXdzLmNvbYIcKi5zMy51cy1lYXN0LTEuYW1hem9uYXdzLmNv
bYIaczMudXMtZWFzdC0yLmFtYXpvbmF3cy5jb22CHCouczMudXMtZWFzdC0yLmFt
YXpvbmF3cy5jb20wCgYIKoZIzj0EAwIDSAAwRQIgLiE7LuK6O/bKo70XPUi6xoDE
ew+EHO31klTOeWiS6oMCIQCHMEqSAcDF/gnG/UXnp2viHOFjnY+NZgQo76l+/2mE
iQ==
-----END CERTIFICATE-----
@@ -0,0 +1,5 @@
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIBhweMaCuhrRJy6hLV9X7QRCorDdyiUvSWEySHXZJM4DoAoGCCqGSM49
AwEHoUQDQgAEEe2dO1v1gE0Qj4H407i0K8tNkASkveckACPFzXs2i/++rZY4bwUu
b08JcMRv0WWwnRzOoumsN26Ge454vTbjog==
-----END EC PRIVATE KEY-----
@@ -0,0 +1,25 @@
[req]
default_bits = 4096
prompt = no
default_md = sha256
req_extensions = v3_req
distinguished_name = dn

[ dn ]
C=US
ST=All
L=All
O=pgBackRest
OU=Unit Testing Domain
CN = s3.us-east-1.amazonaws.com

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = s3.us-east-1.amazonaws.com
DNS.2 = *.s3.us-east-1.amazonaws.com
DNS.3 = s3.us-east-2.amazonaws.com
DNS.4 = *.s3.us-east-2.amazonaws.com
@@ -200,6 +200,7 @@
<!ATTLIST host-define from CDATA #REQUIRED>

<!ELEMENT host-add (execute*)>
<!ATTLIST host-add if CDATA "">
<!ATTLIST host-add id CDATA "">
<!ATTLIST host-add name CDATA #REQUIRED>
<!ATTLIST host-add user CDATA #REQUIRED>
@@ -54,6 +54,14 @@

<variable key="pgbackrest-repo-path">/pgbackrest</variable>

<!-- Path where CA certificates are installed -->
<variable key="ca-cert-path" if="{[os-type-is-debian]}">/usr/local/share/ca-certificates</variable>
<variable key="ca-cert-path" if="{[os-type-is-centos]}">/etc/pki/ca-trust/source/anchors</variable>

<!-- Path where fake certificates are located -->
<variable key="fake-cert-path-relative">resource/fake-cert</variable>
<variable key="fake-cert-path">{[pgbackrest-host-repo-path]}/doc/{[fake-cert-path-relative]}</variable>

<variable key="pg-version" if="{[os-type-is-debian]}">{[os-debian-pg-version]}</variable>
<variable key="pg-version" if="{[os-type-is-centos6]}">{[os-centos6-pg-version]}</variable>
<variable key="pg-version" if="{[os-type-is-centos7]}">{[os-centos7-pg-version]}</variable>
@@ -128,6 +136,15 @@
<variable key="pg-switch-wal" if="{[pg-version]} &lt; 10">pg_switch_xlog</variable>
<variable key="pg-switch-wal" if="{[pg-version]} &gt;= 10">pg_switch_wal</variable>

<!-- S3 Settings -->
<variable key="s3-local">y</variable>
<variable key="s3-bucket">demo-bucket</variable>
<variable key="s3-repo">demo-repo</variable>
<variable key="s3-region">us-east-1</variable>
<variable key="s3-endpoint">s3.{[s3-region]}.amazonaws.com</variable>
<variable key="s3-key">accessKey1</variable>
<variable key="s3-key-secret">verySecretKey1</variable>

<!-- Hosts -->
<variable key="host-image">pgbackrest/doc:{[os-type]}</variable>

@@ -228,11 +245,18 @@
echo ' StrictHostKeyChecking no' >> /root/.ssh/config &amp;&amp; \
chmod 600 /root/.ssh/*
</variable>

<variable key="minio-client-install">wget https://dl.min.io/client/mc/release/linux-amd64/mc -qO /usr/bin/mc &amp;&amp; \
chmod 755 /usr/bin/mc</variable>

<variable key="copy-ca-cert">COPY {[fake-cert-path-relative]}/ca.crt {[ca-cert-path]}/pgbackrest-ca.crt</variable>
</variable-list>

<!-- Setup hosts used to build the documentation
============================================================================================================================ -->
<host-define if="{[os-type-is-debian]}" image="{[host-image]}" from="{[os-image]}">
{[copy-ca-cert]}

# Fix root tty
RUN sed -i 's/^mesg n/tty -s \&amp;\&amp; mesg n/g' /root/.profile &amp;&amp; \

@@ -241,7 +265,10 @@

# Install base packages
RUN apt-get update &amp;&amp; \
apt-get install -y sudo ssh wget vim gnupg lsb-release 2>&amp;1
apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates 2>&amp;1

# Install CA certificate
RUN update-ca-certificates

# Install PostgreSQL
RUN RELEASE_CODENAME=`lsb_release -c | awk '{print $2}'` &amp;&amp; \
@@ -261,13 +288,21 @@
RUN adduser --disabled-password --gecos "" {[host-user]} &amp;&amp; \
echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

RUN {[minio-client-install]}

ENTRYPOINT service ssh restart &amp;&amp; bash
</host-define>

<host-define if="{[os-type-is-centos6]}" image="{[host-image]}" from="{[os-image]}">
{[copy-ca-cert]}

# Install packages
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&amp;1

# Install CA certificate
RUN update-ca-trust enable &amp;&amp; \
update-ca-trust extract

# Regenerate SSH keys
RUN rm -f /etc/ssh/ssh_host_rsa_key* &amp;&amp; \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
@@ -284,12 +319,16 @@
RUN adduser -n {[host-user]} &amp;&amp; \
echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}

RUN {[minio-client-install]}

ENTRYPOINT /usr/sbin/sshd -D
</host-define>

<host-define if="{[os-type-is-centos7]}" image="{[host-image]}" from="{[os-image]}">
ENV container docker

{[copy-ca-cert]}

RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
systemd-tmpfiles-setup.service ] || rm -f $i; done); \
rm -f /lib/systemd/system/multi-user.target.wants/*;\
@@ -305,6 +344,9 @@
# Install packages
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&amp;1

# Install CA certificate
RUN update-ca-trust extract

# Regenerate SSH keys
RUN rm -f /etc/ssh/ssh_host_rsa_key* &amp;&amp; \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key &amp;&amp; \
@@ -328,6 +370,8 @@
ln -s /usr/lib/systemd/system/systemd-user-sessions.service \
/etc/systemd/system/default.target.wants/systemd-user-sessions.service

RUN {[minio-client-install]}

CMD ["/usr/sbin/init"]
</host-define>

@@ -544,6 +588,9 @@
<section id="introduction">
<title>Introduction</title>

<!-- Create S3 server first to allow it time to boot before being used -->
<host-add if="'{[s3-local]}' eq 'y'" id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="minio/minio" os="{[os-type]}" option="-v {[fake-cert-path]}/s3-server.crt:/root/.minio/certs/public.crt:ro -v {[fake-cert-path]}/s3-server.key:/root/.minio/certs/private.key:ro -e MINIO_REGION={[s3-region]} -e MINIO_DOMAIN={[s3-endpoint]} -e MINIO_BROWSER=off -e MINIO_ACCESS_KEY={[s3-key]} -e MINIO_SECRET_KEY={[s3-key-secret]}" param="server /data --address :443 --compat" update-hosts="n"/>

<p>This user guide is intended to be followed sequentially from beginning to end &amp;mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>

<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
@@ -720,14 +767,6 @@
<section id="installation">
<title>Installation</title>

<!-- Create S3 server first to allow it time to boot before being used -->
<host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="pgbackrest/test:s3-server-20180612A" os="{[os-type]}">
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>
</host-add>

<p>A new host named <host>pg1</host> is created to contain the demo cluster and run <backrest/> examples.</p>

<host-add id="{[host-pg1-id]}" name="{[host-pg1]}" user="{[host-pg1-user]}" image="{[host-pg1-image]}" os="{[os-type]}" mount="{[host-pg1-mount]}" option="{[host-option]}"/>
@@ -1985,11 +2024,20 @@

<p><backrest/> supports locating repositories in <proper>S3-compatible</proper> object stores. The bucket used to store the repository must be created in advance &amp;mdash; <backrest/> will not do it automatically. The repository can be located in the bucket root (<path>/</path>) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.</p>

<execute-list host="{[host-s3]}" show="n">
<execute-list if="'{[s3-local]}' eq 'y'" host="{[host-pg1]}" show="n">
<title>Create the bucket</title>

<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} {[s3-bucket]}.{[s3-endpoint]} {[s3-endpoint]}" | tee -a /etc/hosts</exe-cmd>
</execute>

<execute show='n'>
<exe-cmd>mc config host add demo https://{[host-s3-ip]} {[s3-key]} {[s3-key-secret]} --insecure</exe-cmd>
</execute>

<execute show='n'>
<exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
<exe-cmd>mc mb demo/{[s3-bucket]} --insecure</exe-cmd>
</execute>
</execute-list>

@@ -2004,18 +2052,18 @@
<title>Configure <proper>S3</proper></title>

<backrest-config-option section="global" key="repo1-type">s3</backrest-config-option>
<backrest-config-option section="global" key="repo1-path">/demo-repo</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key">accessKey1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key-secret">verySecretKey1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-bucket">demo-bucket</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-endpoint">s3.amazonaws.com</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-region">us-east-1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-verify-ssl">n</backrest-config-option>
<backrest-config-option section="global" key="repo1-path">/{[s3-repo]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key">{[s3-key]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key-secret">{[s3-key-secret]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-bucket">{[s3-bucket]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-endpoint">{[s3-endpoint]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-region">{[s3-region]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-ca-file" if="{[os-type-is-centos]}">/etc/pki/tls/certs/ca-bundle.crt</backrest-config-option>

<backrest-config-option section="global" key="process-max">4</backrest-config-option>
</backrest-config>

<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>us-east-1</id> region.</admonition>
<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>{[s3-region]}</id> region.</admonition>

<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>

@@ -2029,13 +2077,13 @@
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::demo-bucket"
"arn:aws:s3:::{[s3-bucket]}"
],
"Condition": {
"StringEquals": {
"s3:prefix": [
"",
"demo-repo"
"{[s3-repo]}"
],
"s3:delimiter": [
"/"
@@ -2049,12 +2097,12 @@
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::demo-bucket"
"arn:aws:s3:::{[s3-bucket]}"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"demo-repo/*"
"{[s3-repo]}/*"
]
}
}
@@ -2067,7 +2115,7 @@
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::demo-bucket/demo-repo/*"
"arn:aws:s3:::{[s3-bucket]}/{[s3-repo]}/*"
]
}
]
@@ -2079,11 +2127,6 @@
<execute-list host="{[host-pg1]}">
<title>Create the stanza</title>

<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>

<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create</exe-cmd>
<exe-highlight>completed successfully</exe-highlight>

0 comments on commit 3e1b06a

Please sign in to comment.
You can’t perform that action at this time.