Skip to content

Commit f491aaf

Browse files
zc-devsdekobon
zc-devs
and
dekobon
committed
Add support for prefix string to dir list results
Co-authored-by: Elijah Zupancic <e.zupancic@f5.com>
1 parent 4bf8d5e commit f491aaf

11 files changed

+52
-4
lines changed

‎Dockerfile.buildkit.plus

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
1111
ENV PROXY_CACHE_VALID_NOTFOUND "1m"
1212
ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
1313
ENV CORS_ENABLED 0
14+
ENV DIRECTORY_LISTING_PATH_PREFIX ""
1415

1516
COPY plus/usr /usr
1617

‎Dockerfile.oss

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
99
ENV PROXY_CACHE_VALID_NOTFOUND "1m"
1010
ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
1111
ENV CORS_ENABLED 0
12+
ENV DIRECTORY_LISTING_PATH_PREFIX ""
1213

1314
# We modify the nginx base image by:
1415
# 1. Adding configuration files needed for proxying private S3 buckets

‎Dockerfile.plus

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
1111
ENV PROXY_CACHE_VALID_NOTFOUND "1m"
1212
ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
1313
ENV CORS_ENABLED 0
14+
ENV DIRECTORY_LISTING_PATH_PREFIX ""
1415

1516
COPY plus/etc/ssl /etc/ssl
1617
COPY plus/usr /usr

‎common/docker-entrypoint.d/00-check-for-required-env.sh

+1
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ echo "Addressing Style: ${S3_STYLE}"
134134
echo "AWS Signatures Version: v${AWS_SIGS_VERSION}"
135135
echo "DNS Resolvers: ${DNS_RESOLVERS}"
136136
echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}"
137+
echo "Directory Listing Path Prefix: ${DIRECTORY_LISTING_PATH_PREFIX}"
137138
echo "Provide Index Pages Enabled: ${PROVIDE_INDEX_PAGE}"
138139
echo "Append slash for directory enabled: ${APPEND_SLASH_FOR_POSSIBLE_DIRECTORY}"
139140
echo "Stripping the following headers from responses: x-amz-;${HEADER_PREFIXES_TO_STRIP}"

‎common/etc/nginx/include/listing.xsl

+5-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
<xsl:output method="html" encoding="utf-8" indent="yes"/>
44
<xsl:strip-space elements="*" />
55

6+
<xsl:param name="rootPath" />
7+
68
<xsl:template match="/">
79
<xsl:choose>
810
<xsl:when test="//*[local-name()='Contents'] or //*[local-name()='CommonPrefixes']">
@@ -36,7 +38,7 @@
3638
</title>
3739
</head>
3840
<body>
39-
<h1>Index of /<xsl:value-of select="$globalPrefix"/></h1>
41+
<h1>Index of /<xsl:value-of select="concat($rootPath, $globalPrefix)"/></h1>
4042
<hr/>
4143
<table id="list">
4244
<thead>
@@ -124,7 +126,8 @@
124126
encoded so that they form a valid link that NGINX can parse -->
125127
<xsl:template name="encode-uri">
126128
<xsl:param name="uri"/>
127-
<xsl:for-each select="str:split($uri, '/')">
129+
<xsl:variable name="prefixed_uri" select="concat($rootPath, $uri)" />
130+
<xsl:for-each select="str:split($prefixed_uri, '/')">
128131
<xsl:variable name="encoded" select="str:encode-uri(., 'true', 'UTF-8')" />
129132
<xsl:variable name="more-encoded" select="
130133
str:replace(

‎common/etc/nginx/nginx.conf

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ env S3_STYLE;
2323
env ALLOW_DIRECTORY_LIST;
2424
env PROVIDE_INDEX_PAGE;
2525
env APPEND_SLASH_FOR_POSSIBLE_DIRECTORY;
26+
env DIRECTORY_LISTING_PATH_PREFIX;
2627
env PROXY_CACHE_MAX_SIZE;
2728
env PROXY_CACHE_INACTIVE;
2829
env PROXY_CACHE_VALID_OK;

‎common/etc/nginx/templates/default.conf.template

+1
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ server {
177177
# Apply XSL transformation to the XML returned from S3 directory listing
178178
# results such that we can output an HTML directory contents list.
179179
xslt_stylesheet /etc/nginx/include/listing.xsl;
180+
xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}';
180181
xslt_types application/xml;
181182

182183
# We apply an output filter to the XML input received from S3 before it

‎docs/getting_started.md

+17-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ running as a Container or as a Systemd service.
2828
| `S3_STYLE` | Yes | `virtual`, `path`, `default` | `default` | The S3 host/path method. <li>`virtual` is the method that that uses DNS-style bucket+hostname:port. This is the `default` value. <li>`path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services. <br/><br/>See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information. |
2929
| `DEBUG` | No | `true`, `false` | `false` | Flag enabling AWS signatures debug output |
3030
| `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No | `true`, `false` | `false` | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`. |
31+
| `DIRECTORY_LISTING_PATH_PREFIX` | No | | | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing) |
3132
| `DNS_RESOLVERS` | No | | | DNS resolvers (separated by single spaces) to configure NGINX with |
3233
| `PROXY_CACHE_MAX_SIZE` | No | | | Limits cache size |
3334
| `PROXY_CACHE_INACTIVE` | No | | | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness |
@@ -78,6 +79,21 @@ result in log messages like:
7879
Another limitation is that when using v2 signatures with HEAD requests, the
7980
gateway will not return 200 for valid folders.
8081

82+
#### Prefixing List Results
83+
84+
The gateway can be configured to prefix all list results with a given string.
85+
This is useful if you are proxying the gateway itself and wish to relocate
86+
the path of the files returned from the listing.
87+
Using the `DIRECTORY_LISTING_PATH_PREFIX` environment variable will allow
88+
one to add that prefix in listing page's header and links.
89+
90+
For example, if one configures to `DIRECTORY_LISTING_PATH_PREFIX='main/'` and
91+
then uses HAProxy to proxy the gateway with the
92+
`http-request set-path %[path,regsub(^/main,/)]` setting, the architecture
93+
will look like the following:
94+
95+
![](./img/nginx-s3-gateway-directory-listing-path-prefix.png)
96+
8197
### Static Site Hosting
8298

8399
When `PROVIDE_INDEX_PAGE` environment variable is set to 1, the gateway will
@@ -381,4 +397,4 @@ error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 4
381397
```
382398

383399
### Error `403 Access Denied` for AWS Accounts with MFA Enabled
384-
The REST authentication method used in this container does not work with AWS IAM roles that have MFA enabled for authentication. Please use AWS IAM role credentials that do not have MFA enabled.
400+
The REST authentication method used in this container does not work with AWS IAM roles that have MFA enabled for authentication. Please use AWS IAM role credentials that do not have MFA enabled.

‎docs/img/nginx-s3-gateway-directory-listing-path-prefix.png

56.2 KB
Loading

‎settings.example

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ AWS_SIGS_VERSION=4
1212
ALLOW_DIRECTORY_LIST=false
1313
PROVIDE_INDEX_PAGE=false
1414
APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false
15+
DIRECTORY_LISTING_PATH_PREFIX=""
1516
PROXY_CACHE_MAX_SIZE=10g
1617
PROXY_CACHE_INACTIVE=60m
1718
PROXY_CACHE_VALID_OK=1h

‎standalone_ubuntu_oss_install.sh

+23-1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ echo "Addressing Style: ${S3_STYLE}"
8585
echo "AWS Signatures Version: v${AWS_SIGS_VERSION}"
8686
echo "DNS Resolvers: ${DNS_RESOLVERS}"
8787
echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}"
88+
echo "Directory Listing path prefix: ${DIRECTORY_LISTING_PATH_PREFIX}"
8889
echo "Cache size limit: ${PROXY_CACHE_MAX_SIZE}"
8990
echo "Cache inactive timeout: ${PROXY_CACHE_INACTIVE}"
9091
echo "Proxy Caching Time for Valid Response: ${PROXY_CACHE_VALID_OK}"
@@ -135,6 +136,8 @@ echo "▶ Adding environment variables to NGINX configuration file: /etc/nginx/e
135136
cat > "/etc/nginx/environment" << EOF
136137
# Enables or disables directory listing for the S3 Gateway (true=enabled, false=disabled)
137138
ALLOW_DIRECTORY_LIST=${ALLOW_DIRECTORY_LIST}
139+
# Enables or disables directory listing for the S3 Gateway (true=enabled, false=disabled)
140+
DIRECTORY_LISTING_PATH_PREFIX=${DIRECTORY_LISTING_PATH_PREFIX:-''}
138141
# AWS Authentication signature version (2=v2 authentication, 4=v4 authentication)
139142
AWS_SIGS_VERSION=${AWS_SIGS_VERSION}
140143
# Name of S3 bucket to proxy requests to
@@ -246,12 +249,31 @@ auto_envsubst() {
246249
done
247250
}
248251
252+
auto_envsubst_on_include() {
253+
local include_dir="/etc/nginx/include"
254+
local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}"
255+
local filter="${NGINX_ENVSUBST_FILTER:-}"
256+
257+
local template defined_envs relative_path output_path subdir
258+
defined_envs=$(printf '${%s} ' $(env | cut -d= -f1))
259+
if [ ! -w "$include_dir" ]; then
260+
echo "$ME: ERROR: $include_dir is not writable"
261+
return 0
262+
fi
263+
find "$include_dir" -follow -type f -name "*$suffix" -print | while read -r template; do
264+
output_path="${template%$suffix}"
265+
echo "$ME: Running envsubst on $template to $output_path"
266+
envsubst "$defined_envs" < "$template" > "$output_path"
267+
done
268+
}
269+
249270
# Attempt to read DNS Resolvers from /etc/resolv.conf
250271
if [ -z ${DNS_RESOLVERS+x} ]; then
251272
export DNS_RESOLVERS="$(cat /etc/resolv.conf | grep nameserver | cut -d' ' -f2 | xargs)"
252273
fi
253274
254275
auto_envsubst
276+
auto_envsubst_on_include
255277
EOF
256278
chmod +x /usr/local/bin/template_nginx_config.sh
257279

@@ -354,7 +376,7 @@ http {
354376
}
355377
EOF
356378

357-
download "common/etc/nginx/include/listing.xsl" "/etc/nginx/include/listing.xsl"
379+
download "common/etc/nginx/include/listing.xsl.template" "/etc/nginx/include/listing.xsl.template"
358380
download "common/etc/nginx/include/awscredentials.js" "/etc/nginx/include/awscredentials.js"
359381
download "common/etc/nginx/include/awssig2.js" "/etc/nginx/include/awssig2.js"
360382
download "common/etc/nginx/include/awssig4.js" "/etc/nginx/include/awssig4.js"

0 commit comments

Comments
 (0)