Support appengine

.travis.yml

@@ -5,12 +5,14 @@ go:
 services:
   - redis-server
 env:
-  - CACHE=redis
-  - CACHE=memcache
+  - GAE=.go_appengine
+  - GAE=
 cache:
   directories:
     - vendor
     - glide-$(go version | awk '{ print $NF }' | tr '/' '-')
+    - .download
+    - .go_appengine
 install:
   - make installdeps
 script:

Makefile

@@ -1,6 +1,9 @@
 GOVERSION=$(shell go version)
 GOOS=$(word 1,$(subst /, ,$(word $(words $(GOVERSION)), $(GOVERSION))))
 GOARCH=$(word 2,$(subst /, ,$(word $(words $(GOVERSION)), $(GOVERSION))))
+ifneq ($(GAE),)
+export PATH := $(GAE):$(PATH)
+endif
 
 installdeps: glide-$(GOOS)-$(GOARCH)/glide
 	PATH=glide-$(GOOS)-$(GOARCH):$(PATH) glide install
@@ -16,8 +19,19 @@ glide-$(GOOS)-$(GOARCH)/glide:
 	@rm -rf $(GOOS)-$(GOARCH)
 
 test:
-ifdef $(CACHE)
-	$(eval TAGS := "$(TAGS) $(CACHE)")
+ifeq ($(GAE),)
+	go test -v $(shell glide-$(GOOS)-$(GOARCH)/glide novendor)
+else
+	$(MAKE) appengine_test
 endif
 
-	go test -v -tags '$(TAGS)' $(shell glide-$(GOOS)-$(GOARCH)/glide novendor)
+$(GAE)/goapp:
+	@mkdir -p .download
+	@mkdir -p $(GAE)
+	wget -q https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_$(GOOS)_$(GOARCH)-1.9.48.zip
+	@mv go_appengine_sdk_$(GOOS)_$(GOARCH)-1.9.48.zip .download/
+	@unzip .download/go_appengine_sdk_$(GOOS)_$(GOARCH)-1.9.48.zip > /dev/null
+	@mv go_appengine/* $(GAE)
+
+appengine_test: $(GAE)/goapp
+	goapp test -v $(shell glide-$(GOOS)-$(GOARCH)/glide novendor)

README.md

@@ -29,14 +29,40 @@ and transform that to below when passing to the actual sharaq app
 
 # CONFIGURATION
 
+## Listen Address
+
+```json
+{
+  "Listen": "0.0.0.0:8080"
+}
+```
+
+## Access Log
+
+See also: https://github.com/lestrrat/go-apache-logformat
+
+```json
+{
+  "AccessLog": {
+    "LogFile": "/path/to/logfile",
+    "LinkName": "/path/to/linkname.%Y%m%d",
+    "RotationTime": 86400,
+    "MaxAge": 172800,
+  }
+}
+```
+
 ## AWS (S3) Backend
 
 ```json
 {
-  "Amazon": {
-    "AccessKey": "...",
-    "SecretKey": "...",
-    "BucketName": "..."
+  "Backend": {
+    "Type": "aws",
+    "Amazon": {
+      "AccessKey": "...",
+      "SecretKey": "...",
+      "BucketName": "..."
+    }
   }
 }
 ```
@@ -83,19 +109,46 @@ For GCP (Google Storage), service keys are looked under several known locations.
 
 ```json
 {
-  "Google": {
-    "BucketName": "..."
+  "Backend": {
+    "Type": "gcp",
+    "Google": {
+      "BucketName": "..."
+    }
   }
 }
 ```
+
+`sharaq` also supports running on Google AppEngine (standard environment). For this to work, you will have to change the setup a bit. You will not need a `config.json` file, but you will have to setup your environment in app.yaml
+
+```yaml
+service: sharaq
+version: 1
+runtime: go
+api_version: go1
+handlers:
+  - url: /
+    script: _go_app
+env_variables:
+  SHARAQ_PRESETS: large=600x600,medium=400x400,small=200x200
+  SHARAQ_BACKEND_TYPE: gcp
+  SHARAQ_BACKEND_GCP_BUCKET_NAME: "bucket-name-of-your-choise"
+  SHARAQ_BACKEND_GCP_PREFIX: "resize (this is optional)"
+  SHARAQ_TOKENS: "foobarbaz (if you want to access the POST/DELETE endpoints via HTTP)"
+  SHARAQ_URLCACHE_TYPE: Memcached
+  SHARAQ_WHITELIST: "whitelisting your target is recommended"
+```
+
 ## File System Backend
 
 The FS backend stores all the images in a directory in the sharaq host. You probably don't want to use this except for testing and for debugging.
 
 ```json
 {
-  "FileSystem": {
-    "StorageRoot": "/path/to/storage-dir"
+  "Backend": {
+    "Type": "fs",
+    "FileSystem": {
+      "StorageRoot": "/path/to/storage-dir"
+    }
   }
 }
 ```

aws/backend.go

@@ -1,16 +1,19 @@
 package aws
 
 import (
-	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"sync"
 
+	"golang.org/x/net/context"
+	"golang.org/x/sync/errgroup"
+
 	"github.com/goamz/goamz/aws"
 	"github.com/goamz/goamz/s3"
 	"github.com/lestrrat/sharaq/internal/bbpool"
+	"github.com/lestrrat/sharaq/internal/errors"
+	"github.com/lestrrat/sharaq/internal/log"
 	"github.com/lestrrat/sharaq/internal/transformer"
 	"github.com/lestrrat/sharaq/internal/urlcache"
 	"github.com/lestrrat/sharaq/internal/util"
@@ -24,6 +27,13 @@ type S3Backend struct {
 	transformer *transformer.Transformer
 }
 
+type redirectContent string
+
+func (s redirectContent) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log.Debugf(util.RequestCtx(r), "Object %s exists. Redirecting to proper location", string(s))
+	w.Header().Add("Location", string(s))
+	w.WriteHeader(http.StatusMovedPermanently)
+}
 func NewBackend(c *Config, cache *urlcache.URLCache, trans *transformer.Transformer, presets map[string]string) (*S3Backend, error) {
 	auth := aws.Auth{
 		AccessKey: c.AccessKey,
@@ -40,107 +50,66 @@ func NewBackend(c *Config, cache *urlcache.URLCache, trans *transformer.Transfor
 	}, nil
 }
 
-func (s *S3Backend) Serve(w http.ResponseWriter, r *http.Request) {
-	u, err := util.GetTargetURL(r)
-	if err != nil {
-		log.Printf("Bad url: %s", err)
-		http.Error(w, "Bad url", 500)
-		return
-	}
-
-	preset, err := util.GetPresetFromRequest(r)
-	if err != nil {
-		log.Printf("Bad preset: %s", err)
-		http.Error(w, "Bad preset", 500)
-		return
-	}
-
-	cacheKey := urlcache.MakeCacheKey("s3", preset, u.String())
-	if cachedURL := s.cache.Lookup(cacheKey); cachedURL != "" {
-		log.Printf("Cached entry found for %s:%s -> %s", preset, u.String(), cachedURL)
-		w.Header().Add("Location", cachedURL)
-		w.WriteHeader(301)
-		return
+func (s *S3Backend) Get(ctx context.Context, u *url.URL, preset string) (http.Handler, error) {
+	cacheKey := urlcache.MakeCacheKey("aws", preset, u.String())
+	if cachedURL := s.cache.Lookup(ctx, cacheKey); cachedURL != "" {
+		log.Debugf(ctx, "Cached entry found for %s:%s -> %s", preset, u.String(), cachedURL)
+		return redirectContent(cachedURL), nil
 	}
 
 	// create the proper url
 	specificURL := "http://" + s.bucketName + ".s3.amazonaws.com/" + preset + u.Path
 
-	log.Printf("Making HEAD request to %s...", specificURL)
+	log.Debugf(ctx, "Making HEAD request to %s...", specificURL)
 	res, err := http.Head(specificURL)
 	if err != nil {
-		log.Printf("Failed to make HEAD request to %s: %s", specificURL, err)
-		goto FALLBACK
+		return nil, errors.TransformationRequiredError{}
 	}
 
-	log.Printf("HEAD request for %s returns %d", specificURL, res.StatusCode)
-	if res.StatusCode == 200 {
-		go s.cache.Set(cacheKey, specificURL)
-		log.Printf("HEAD request to %s was success. Redirecting to proper location", specificURL)
-		w.Header().Add("Location", specificURL)
-		w.WriteHeader(301)
-		return
+	log.Debugf(ctx, "HEAD request for %s returns %d", specificURL, res.StatusCode)
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.TransformationRequiredError{}
 	}
 
-	go func() {
-		if err := s.StoreTransformedContent(r.Context(), u); err != nil {
-			log.Printf("S3Backend: transformation failed: %s", err)
-		}
-	}()
-
-FALLBACK:
-	w.Header().Add("Location", u.String())
-	w.WriteHeader(302)
+	return redirectContent(specificURL), nil
 }
 
 func (s *S3Backend) StoreTransformedContent(ctx context.Context, u *url.URL) error {
-	log.Printf("S3Backend: transforming image at url %s", u)
+	log.Debugf(ctx, "S3Backend: transforming image at url %s", u)
 
 	// Transformation is completely done by the transformer, so just
 	// hand it over to it
-	wg := &sync.WaitGroup{}
-	errCh := make(chan error, len(s.presets))
-	for preset, rule := range s.presets {
-		wg.Add(1)
-		go func(wg *sync.WaitGroup, t *transformer.Transformer, preset string, rule string, errCh chan error) {
-			defer wg.Done()
+	var grp *errgroup.Group
+	grp, ctx = errgroup.WithContext(ctx)
 
+	for preset, rule := range s.presets {
+		t := s.transformer
+		preset := preset
+		rule := rule
+		grp.Go(func() error {
 			buf := bbpool.Get()
 			defer bbpool.Release(buf)
 
 			var res transformer.Result
 			res.Content = buf
 
-			if err := t.Transform(rule, u.String(), &res); err != nil {
-				errCh <- err
-				return
+			if err := t.Transform(ctx, rule, u.String(), &res); err != nil {
+				return errors.Wrap(err, `failed to transform image`)
 			}
 
 			// good, done. save it to S3
 			path := "/" + preset + u.Path
-			log.Printf("Sending PUT to S3 %s...", path)
-			err := s.bucket.PutReader(path, buf, res.Size, res.ContentType, s3.PublicRead, s3.Options{})
-			if err != nil {
-				errCh <- err
-				return
+			log.Debugf(ctx, "Sending PUT to S3 %s...", path)
+			if err := s.bucket.PutReader(path, buf, res.Size, res.ContentType, s3.PublicRead, s3.Options{}); err != nil {
+				return errors.Wrapf(err, `failed to write data to %s`, path)
 			}
-		}(wg, s.transformer, preset, rule, errCh)
-	}
-	wg.Wait()
-	close(errCh)
-
-	buf := bbpool.Get()
-	defer bbpool.Release(buf)
-
-	for err := range errCh {
-		fmt.Fprintf(buf, "Err: %s\n", err)
-	}
-
-	if buf.Len() > 0 {
-		return fmt.Errorf("error while transforming: %s", buf.String())
+			cacheKey := urlcache.MakeCacheKey("gcp", preset, u.String())
+			specificURL := "http://" + s.bucketName + ".s3.amazonaws.com/" + preset + u.Path
+			s.cache.Set(ctx, cacheKey, specificURL)
+			return nil
+		})
 	}
-
-	return nil
+	return grp.Wait()
 }
 
 func (s *S3Backend) Delete(ctx context.Context, u *url.URL) error {
@@ -151,15 +120,15 @@ func (s *S3Backend) Delete(ctx context.Context, u *url.URL) error {
 		go func(wg *sync.WaitGroup, preset string, errCh chan error) {
 			defer wg.Done()
 			path := "/" + preset + u.Path
-			log.Printf(" + DELETE S3 entry %s\n", path)
+			log.Debugf(ctx, " + DELETE S3 entry %s\n", path)
 			err := s.bucket.Del(path)
 			if err != nil {
 				errCh <- err
 			}
 
 			// fallthrough here regardless, because it's better to lose the
 			// cache than to accidentally have one linger
-			s.cache.Delete(urlcache.MakeCacheKey(preset, u.String()))
+			s.cache.Delete(context.Background(), urlcache.MakeCacheKey(preset, u.String()))
 		}(&wg, preset, errCh)
 	}