Add support for prefix string to dir list results

zc-devs · dekobon · dekobon · commit f491aaf6ecc1 · 2023-06-14T11:47:24.000-07:00
Co-authored-by: Elijah Zupancic &lt;e.zupancic@f5.com&gt;
diff --git a/Dockerfile.buildkit.plus b/Dockerfile.buildkit.plus
@@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
 ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
+ENV DIRECTORY_LISTING_PATH_PREFIX ""
 
 COPY plus/usr /usr
 
diff --git a/Dockerfile.oss b/Dockerfile.oss
@@ -9,6 +9,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
 ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
+ENV DIRECTORY_LISTING_PATH_PREFIX ""
 
 # We modify the nginx base image by:
 # 1. Adding configuration files needed for proxying private S3 buckets
diff --git a/Dockerfile.plus b/Dockerfile.plus
@@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_OK "1h"
 ENV PROXY_CACHE_VALID_NOTFOUND "1m"
 ENV PROXY_CACHE_VALID_FORBIDDEN "30s"
 ENV CORS_ENABLED 0
+ENV DIRECTORY_LISTING_PATH_PREFIX ""
 
 COPY plus/etc/ssl /etc/ssl
 COPY plus/usr /usr
diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh
@@ -134,6 +134,7 @@ echo "Addressing Style: ${S3_STYLE}"
 echo "AWS Signatures Version: v${AWS_SIGS_VERSION}"
 echo "DNS Resolvers: ${DNS_RESOLVERS}"
 echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}"
+echo "Directory Listing Path Prefix: ${DIRECTORY_LISTING_PATH_PREFIX}"
 echo "Provide Index Pages Enabled: ${PROVIDE_INDEX_PAGE}"
 echo "Append slash for directory enabled: ${APPEND_SLASH_FOR_POSSIBLE_DIRECTORY}"
 echo "Stripping the following headers from responses: x-amz-;${HEADER_PREFIXES_TO_STRIP}"
diff --git a/common/etc/nginx/include/listing.xsl b/common/etc/nginx/include/listing.xsl
@@ -3,6 +3,8 @@
     <xsl:output method="html" encoding="utf-8" indent="yes"/>
     <xsl:strip-space elements="*" />
 
+    <xsl:param name="rootPath" />
+
     <xsl:template match="/">
         <xsl:choose>
             <xsl:when test="//*[local-name()='Contents'] or //*[local-name()='CommonPrefixes']">
@@ -36,7 +38,7 @@
                 </title>
             </head>
             <body>
-                <h1>Index of /<xsl:value-of select="$globalPrefix"/></h1>
+                <h1>Index of /<xsl:value-of select="concat($rootPath, $globalPrefix)"/></h1>
                 <hr/>
                 <table id="list">
                     <thead>
@@ -124,7 +126,8 @@
          encoded so that they form a valid link that NGINX can parse -->
     <xsl:template name="encode-uri">
         <xsl:param name="uri"/>
-        <xsl:for-each select="str:split($uri, '/')">
+        <xsl:variable name="prefixed_uri" select="concat($rootPath, $uri)" />
+        <xsl:for-each select="str:split($prefixed_uri, '/')">
             <xsl:variable name="encoded" select="str:encode-uri(., 'true', 'UTF-8')" />
             <xsl:variable name="more-encoded" select="
                 str:replace(
diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf
@@ -23,6 +23,7 @@ env S3_STYLE;
 env ALLOW_DIRECTORY_LIST;
 env PROVIDE_INDEX_PAGE;
 env APPEND_SLASH_FOR_POSSIBLE_DIRECTORY;
+env DIRECTORY_LISTING_PATH_PREFIX;
 env PROXY_CACHE_MAX_SIZE;
 env PROXY_CACHE_INACTIVE;
 env PROXY_CACHE_VALID_OK;
diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template
@@ -177,6 +177,7 @@ server {
         # Apply XSL transformation to the XML returned from S3 directory listing
         # results such that we can output an HTML directory contents list.
         xslt_stylesheet /etc/nginx/include/listing.xsl;
+        xslt_string_param rootPath '${DIRECTORY_LISTING_PATH_PREFIX}';
         xslt_types application/xml;
 
         # We apply an output filter to the XML input received from S3 before it
diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -28,6 +28,7 @@ running as a Container or as a Systemd service.
 | `S3_STYLE`                            | Yes       | `virtual`, `path`, `default` | `default` | The S3 host/path method. <li>`virtual` is the method that that uses DNS-style bucket+hostname:port. This is the `default` value. <li>`path` is a method that appends the bucket name as the first directory in the URI's path. This method is used by many S3 compatible services. <br/><br/>See this [AWS blog article](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/) for further information.            |
 | `DEBUG`                               | No        | `true`, `false`              | `false`   | Flag enabling AWS signatures debug output                                                                                                                                                                                                                                                                                                                                                                                                             |
 | `APPEND_SLASH_FOR_POSSIBLE_DIRECTORY` | No        | `true`, `false`              | `false`   | Flag enabling the return a 302 with a `/` appended to the path. This is independent of the behavior selected in `ALLOW_DIRECTORY_LIST` or `PROVIDE_INDEX_PAGE`.                                                                                                                                                                                                                                                                                       |
+| `DIRECTORY_LISTING_PATH_PREFIX`       | No        |                              |           | In `ALLOW_DIRECTORY_LIST=true` mode [adds defined prefix to links](#configuring-directory-listing)                                                                                                                                                                                                                                                                                                                                                    |
 | `DNS_RESOLVERS`                       | No        |                              |           | DNS resolvers (separated by single spaces) to configure NGINX with                                                                                                                                                                                                                                                                                                                                                                                    |
 | `PROXY_CACHE_MAX_SIZE`                | No        |                              |           | Limits cache size                                                                                                                                                                                                                                                                                                                                                                                                                                     |
 | `PROXY_CACHE_INACTIVE`                | No        |                              |           | Cached data that are not accessed during the time specified by the parameter get removed from the cache regardless of their freshness                                                                                                                                                                                                                                                                                                                 |
@@ -78,6 +79,21 @@ result in log messages like:
 Another limitation is that when using v2 signatures with HEAD requests, the
 gateway will not return 200 for valid folders.
 
+#### Prefixing List Results
+
+The gateway can be configured to prefix all list results with a given string.
+This is useful if you are proxying the gateway itself and wish to relocate
+the path of the files returned from the listing.
+Using the `DIRECTORY_LISTING_PATH_PREFIX` environment variable will allow
+one to add that prefix in listing page's header and links.
+
+For example, if one configures to `DIRECTORY_LISTING_PATH_PREFIX='main/'` and 
+then uses HAProxy to proxy the gateway with the 
+`http-request set-path %[path,regsub(^/main,/)]` setting, the architecture
+will look like the following: 
+
+![](./img/nginx-s3-gateway-directory-listing-path-prefix.png)
+
 ### Static Site Hosting
 
 When `PROVIDE_INDEX_PAGE` environment variable is set to 1, the gateway will
@@ -381,4 +397,4 @@ error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 4
 ```
 
 ### Error `403 Access Denied` for AWS Accounts with MFA Enabled
-The REST authentication method used in this container does not work with AWS IAM roles that have MFA enabled for authentication. Please use AWS IAM role credentials that do not have MFA enabled. 
+The REST authentication method used in this container does not work with AWS IAM roles that have MFA enabled for authentication. Please use AWS IAM role credentials that do not have MFA enabled. 
diff --git a/docs/img/nginx-s3-gateway-directory-listing-path-prefix.png b/docs/img/nginx-s3-gateway-directory-listing-path-prefix.png
diff --git a/settings.example b/settings.example
@@ -12,6 +12,7 @@ AWS_SIGS_VERSION=4
 ALLOW_DIRECTORY_LIST=false
 PROVIDE_INDEX_PAGE=false
 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=false
+DIRECTORY_LISTING_PATH_PREFIX=""
 PROXY_CACHE_MAX_SIZE=10g
 PROXY_CACHE_INACTIVE=60m
 PROXY_CACHE_VALID_OK=1h
diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh
@@ -85,6 +85,7 @@ echo "Addressing Style: ${S3_STYLE}"
 echo "AWS Signatures Version: v${AWS_SIGS_VERSION}"
 echo "DNS Resolvers: ${DNS_RESOLVERS}"
 echo "Directory Listing Enabled: ${ALLOW_DIRECTORY_LIST}"
+echo "Directory Listing path prefix: ${DIRECTORY_LISTING_PATH_PREFIX}"
 echo "Cache size limit: ${PROXY_CACHE_MAX_SIZE}"
 echo "Cache inactive timeout: ${PROXY_CACHE_INACTIVE}"
 echo "Proxy Caching Time for Valid Response: ${PROXY_CACHE_VALID_OK}"
@@ -135,6 +136,8 @@ echo "▶ Adding environment variables to NGINX configuration file: /etc/nginx/e
 cat > "/etc/nginx/environment" << EOF
 # Enables or disables directory listing for the S3 Gateway (true=enabled, false=disabled)
 ALLOW_DIRECTORY_LIST=${ALLOW_DIRECTORY_LIST}
+# Enables or disables directory listing for the S3 Gateway (true=enabled, false=disabled)
+DIRECTORY_LISTING_PATH_PREFIX=${DIRECTORY_LISTING_PATH_PREFIX:-''}
 # AWS Authentication signature version (2=v2 authentication, 4=v4 authentication)
 AWS_SIGS_VERSION=${AWS_SIGS_VERSION}
 # Name of S3 bucket to proxy requests to
@@ -246,12 +249,31 @@ auto_envsubst() {
   done
 }
 
+auto_envsubst_on_include() {
+  local include_dir="/etc/nginx/include"
+  local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}"
+  local filter="${NGINX_ENVSUBST_FILTER:-}"
+
+  local template defined_envs relative_path output_path subdir
+  defined_envs=$(printf '${%s} ' $(env | cut -d= -f1))
+  if [ ! -w "$include_dir" ]; then
+    echo "$ME: ERROR: $include_dir is not writable"
+    return 0
+  fi
+  find "$include_dir" -follow -type f -name "*$suffix" -print | while read -r template; do
+    output_path="${template%$suffix}"
+    echo "$ME: Running envsubst on $template to $output_path"
+    envsubst "$defined_envs" < "$template" > "$output_path"
+  done
+}
+
 # Attempt to read DNS Resolvers from /etc/resolv.conf
 if [ -z ${DNS_RESOLVERS+x} ]; then
   export DNS_RESOLVERS="$(cat /etc/resolv.conf | grep nameserver | cut -d' ' -f2 | xargs)"
 fi
 
 auto_envsubst
+auto_envsubst_on_include
 EOF
 chmod +x /usr/local/bin/template_nginx_config.sh
 
@@ -354,7 +376,7 @@ http {
 }
 EOF
 
-download "common/etc/nginx/include/listing.xsl" "/etc/nginx/include/listing.xsl"
+download "common/etc/nginx/include/listing.xsl.template" "/etc/nginx/include/listing.xsl.template"
 download "common/etc/nginx/include/awscredentials.js" "/etc/nginx/include/awscredentials.js"
 download "common/etc/nginx/include/awssig2.js" "/etc/nginx/include/awssig2.js"
 download "common/etc/nginx/include/awssig4.js" "/etc/nginx/include/awssig4.js"
