s3 storage: don't proxy the layer data

S3 supports presigned URLs, so there's no need to proxy the layer data
through Pier One. We just need to check authentication and metadata,
generate the URL and redirect to it.

project.clj

@@ -14,7 +14,7 @@
                  [digest "1.4.4"]
                  [org.apache.commons/commons-compress "1.10"]
                  [org.clojure/data.codec "0.1.0"]
-                 [amazonica "0.3.57"]
+                 [amazonica "0.3.118"]
                  [org.clojure/java.jdbc "0.4.1"]
                  [org.clojure/core.async "0.2.374"]
                  [org.clojure/tools.nrepl "0.2.13"]]

src/org/zalando/stups/pierone/api_v2.clj

@@ -69,6 +69,10 @@
   [storage image tmp-file]
   (s/write-data storage image tmp-file))
 
+(defcommand get-image-url
+  [storage image]
+  (s/get-url storage image))
+
 (defcommand load-image
   [storage image]
   (s/read-data storage image))
@@ -214,12 +218,14 @@
   "Reads the binary data of an image."
   [{:keys [digest]} request db storage _ _]
   (if-let [size (:size (first (sql/cmd-get-blob-size {:image digest} {:connection db})))]
-    (let [data (load-image storage digest)]
-      (-> (resp data request :binary? true)
-          (ring/header "Docker-Content-Digest" digest)
-          (ring/header "Content-Length" size)
-          ; layers are already GZIP compressed!
-          (ring/header "Content-Encoding" "identity")))
+    (if (s/external? storage)
+      (ring/redirect (get-image-url storage digest) 307)
+      (let [data (load-image storage digest)]
+        (-> (resp data request :binary? true)
+            (ring/header "Docker-Content-Digest" digest)
+            (ring/header "Content-Length" size)
+            ; layers are already GZIP compressed!
+            (ring/header "Content-Encoding" "identity"))))
     (resp (get-error-response :BLOB_UNKNOWN {"Digest" digest}) request :status 404)))
 
 (defn read-manifest

src/org/zalando/stups/pierone/storage.clj

@@ -2,18 +2,23 @@
   (:require [org.zalando.stups.friboo.log :as log]
             [com.stuartsierra.component :as component]
             [clojure.java.io :as io]
+            [clj-time.core :as time]
             [amazonica.aws.s3 :as s3])
   (:import (com.amazonaws.services.s3.model AmazonS3Exception)
            (java.util UUID)
            (java.io File)))
 
 (defprotocol Storage
   (read-data [this image] "Returns the binary data for an image.")
+  (external? [this] "Returns true if the store should be redirected to, not read directly.")
+  (get-url [this image] "Returns an external URL for the binary data of an image.")
   (write-data [this image data] "Stores binary data for an image."))
 
 
 (def default-storage-configuration
-  {:storage-directory "target/data"})
+  {:storage-directory "target/data"
+   :storage-s3-url-expiration "30000"
+   :storage-s3-external-store "false"})
 
 
 (defrecord LocalStorage [configuration directory]
@@ -35,6 +40,10 @@
       (when (.exists file)
             (io/input-stream file))))
 
+  (external? [_] false)
+
+  (get-url [_ _] nil)
+
   (write-data [_ image data]
     (let [^File file (io/file directory image)
           ^File tmp-file (io/file directory (str image ".tmp-" (UUID/randomUUID)))]
@@ -47,15 +56,21 @@
 
   (start [this]
     (let [directory (:directory configuration)
-          bucket (:s3-bucket configuration)]
+          bucket (:s3-bucket configuration)
+          expiration (time/millis (:s3-url-expiration configuration))
+          external (:s3-external-store configuration)]
       (log/info "Using S3 storage with bucket %s and temporary directory %s." bucket directory)
       (.mkdirs (io/file directory))
-      (merge this {:directory directory
-                   :bucket    bucket})))
+      (merge this {:directory  directory
+                   :bucket     bucket
+                   :expiration expiration
+                   :external   external})))
 
   (stop [this]
-    (merge this {:directory nil
-                 :bucket    nil}))
+    (merge this {:directory  nil
+                 :bucket     nil
+                 :expiration nil
+                 :external   nil}))
 
   Storage
 
@@ -67,6 +82,16 @@
       (catch AmazonS3Exception se
         (when-not (= 404 (.getStatusCode se)) (throw se)))))
 
+  (external? [this] (:external this))
+
+  (get-url [{expiration :expiration} image]
+    (try
+      (-> (s3/generate-presigned-url bucket image (time/from-now expiration))
+          .toURI
+          .toASCIIString)
+      (catch AmazonS3Exception se
+        (when-not (= 404 (.getStatusCode se)) (throw se)))))
+
   (write-data [_ image data]
     (let [^File tmp-file (io/file directory (str image ".tmp-" (UUID/randomUUID)))]
       (io/copy data tmp-file)