implemented cache strategies

cache/cache.go

@@ -5,12 +5,12 @@ import (
 )
 
 type Cache struct {
-	lruBackend *lru.ARCCache
+	lruBackend *lru.Cache
 }
 
 // NewCache creates a cache with max 100MB and max 10.000 Entries
 func NewCache(entrySize int) *Cache {
-	arc, err := lru.NewARC(entrySize)
+	arc, err := lru.New(entrySize)
 	if err != nil {
 		panic(err)
 	}

cache/cache_strategy.go

@@ -0,0 +1,162 @@
+package cache
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"github.com/pquerna/cachecontrol/cacheobject"
+	"github.com/tarent/lib-compose/logging"
+	"net/http"
+	"strings"
+)
+
+const (
+	// The request method was POST and an Expiration header was not supplied.
+	ReasonRequestMethodPOST = cacheobject.ReasonRequestMethodPOST
+
+	// The request method was PUT and PUTs are not cachable.
+	ReasonRequestMethodPUT = cacheobject.ReasonRequestMethodPUT
+
+	// The request method was DELETE and DELETEs are not cachable.
+	ReasonRequestMethodDELETE = cacheobject.ReasonRequestMethodDELETE
+
+	// The request method was CONNECT and CONNECTs are not cachable.
+	ReasonRequestMethodCONNECT = cacheobject.ReasonRequestMethodCONNECT
+
+	// The request method was OPTIONS and OPTIONS are not cachable.
+	ReasonRequestMethodOPTIONS = cacheobject.ReasonRequestMethodOPTIONS
+
+	// The request method was TRACE and TRACEs are not cachable.
+	ReasonRequestMethodTRACE = cacheobject.ReasonRequestMethodTRACE
+
+	// The request method was not recognized by cachecontrol, and should not be cached.
+	ReasonRequestMethodUnkown = cacheobject.ReasonRequestMethodUnkown
+
+	// The request included an Cache-Control: no-store header
+	ReasonRequestNoStore = cacheobject.ReasonRequestNoStore
+
+	// The request included an Authorization header without an explicit Public or Expiration time: http://tools.ietf.org/html/rfc7234#section-3.2
+	ReasonRequestAuthorizationHeader = cacheobject.ReasonRequestAuthorizationHeader
+
+	// The response included an Cache-Control: no-store header
+	ReasonResponseNoStore = cacheobject.ReasonResponseNoStore
+
+	// The response included an Cache-Control: private header and this is not a Private cache
+	ReasonResponsePrivate = cacheobject.ReasonResponsePrivate
+
+	// The response failed to meet at least one of the conditions specified in RFC 7234 section 3: http://tools.ietf.org/html/rfc7234#section-3
+	ReasonResponseUncachableByDefault = cacheobject.ReasonResponseUncachableByDefault
+)
+
+var DefaultIncludeHeaders = []string{"Authorization", "Accept-Encoding"}
+
+var DefaultCacheStrategy = NewCacheStrategyWithDefault()
+
+type CacheStrategy struct {
+	includeHeaders []string
+	includeCookies []string
+	ignoreReasons  []cacheobject.Reason
+}
+
+func NewCacheStrategyWithDefault() *CacheStrategy {
+	return &CacheStrategy{
+		includeHeaders: DefaultIncludeHeaders,
+		includeCookies: nil,
+		ignoreReasons:  nil,
+	}
+}
+
+func NewCacheStrategy(includeHeaders []string, includeCookies []string, ignoreReasons []cacheobject.Reason) *CacheStrategy {
+	return &CacheStrategy{
+		includeHeaders: includeHeaders,
+		includeCookies: includeCookies,
+		ignoreReasons:  ignoreReasons,
+	}
+}
+
+// Hash computes a hash value based in the url, the method and selected header and cookien attributes,
+func (tcs *CacheStrategy) Hash(method string, url string, requestHeader http.Header) string {
+	return tcs.HashWithParameters(method, url, requestHeader, tcs.includeHeaders, tcs.includeCookies)
+}
+
+// Hash computes a hash value based in the url, the method and selected header and cookien attributes,
+func (tcs *CacheStrategy) HashWithParameters(method string, url string, requestHeader http.Header, includeHeaders []string, includeCookies []string) string {
+	hasher := md5.New()
+
+	hasher.Write([]byte(method))
+	hasher.Write([]byte(url))
+
+	for _, h := range includeHeaders {
+		if requestHeader.Get(h) != "" {
+			hasher.Write([]byte(h))
+			hasher.Write([]byte(requestHeader.Get(h)))
+		}
+	}
+
+	for _, c := range includeCookies {
+		if value, found := readCookieValue(requestHeader, c); found {
+			hasher.Write([]byte(c))
+			hasher.Write([]byte(value))
+		}
+	}
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func (tcs *CacheStrategy) IsCachable(method string, url string, statusCode int, requestHeader http.Header, responseHeader http.Header) bool {
+	// TODO: it is expensive to create a request object only for passing to the cachecontrol library
+	req := &http.Request{Method: method, Header: requestHeader}
+	reasons, _, err := cacheobject.UsingRequestResponse(req, statusCode, responseHeader, true)
+	if err != nil {
+		logging.Logger.WithError(err).Warnf("error checking cachability fot %v %v: %v", method, url, err)
+		return false
+	}
+
+	for _, foundReason := range reasons {
+		if !tcs.isReasonIgnorable(foundReason) {
+			logging.Logger.WithField("notCachableReason", foundReason).Debugf("ressource not cachable %v %v: %v", method, url, foundReason)
+			return false
+		}
+	}
+	return true
+}
+
+func (tcs *CacheStrategy) isReasonIgnorable(reason cacheobject.Reason) bool {
+	for _, ignoreReason := range tcs.ignoreReasons {
+		if reason == ignoreReason {
+			return true
+		}
+	}
+	return false
+}
+
+// taken and adapted from net/http
+func readCookieValue(h http.Header, filterName string) (string, bool) {
+	lines, ok := h["Cookie"]
+	if !ok {
+		return "", false
+	}
+
+	for _, line := range lines {
+		parts := strings.Split(strings.TrimSpace(line), ";")
+		if len(parts) == 1 && parts[0] == "" {
+			continue
+		}
+		for i := 0; i < len(parts); i++ {
+			parts[i] = strings.TrimSpace(parts[i])
+			if len(parts[i]) == 0 {
+				continue
+			}
+			name, val := parts[i], ""
+			if j := strings.Index(name, "="); j >= 0 {
+				name, val = name[:j], name[j+1:]
+			}
+			if filterName == name {
+				if len(val) > 1 && val[0] == '"' && val[len(val)-1] == '"' {
+					val = val[1 : len(val)-1]
+				}
+				return val, true
+			}
+		}
+	}
+	return "", false
+}

composition/cache_loader.go

@@ -8,27 +8,35 @@ import (
 type CachingContentLoader struct {
 	httpContentLoader ContentLoader
 	fileContentLoader ContentLoader
-	cache             cache.Cache
+	cache             *cache.Cache
 }
 
 func NewCachingContentLoader() *CachingContentLoader {
 	return &CachingContentLoader{
 		httpContentLoader: NewHttpContentLoader(),
 		fileContentLoader: NewFileContentLoader(),
+		cache:             cache.NewCache(10000),
 	}
 }
 
 func (loader *CachingContentLoader) Load(fd *FetchDefinition) (Content, error) {
 	hash := fd.Hash()
 
 	if c, exist := loader.cache.Get(hash); exist {
+		println("found: " + fd.URL + " " + hash)
 		return c.(Content), nil
+	} else {
+		println("not found: " + fd.URL + " " + hash)
+
 	}
 
 	c, err := loader.load(fd)
 	if err == nil {
-		if fd.IsCachable(c.HttpHeader()) {
+		if fd.IsCachable(c.HttpStatusCode(), c.HttpHeader()) {
+			println("Set: " + fd.URL + " " + hash)
 			loader.cache.Set(hash, c.MemorySize(), c)
+		} else {
+			println("Not cachable: " + fd.URL + " " + hash)
 		}
 	}
 	return c, err

composition/content_fetcher.go

@@ -2,10 +2,8 @@ package composition
 
 import (
 	"errors"
-	"sync"
-
 	"github.com/tarent/lib-compose/logging"
-	"strings"
+	"sync"
 )
 
 // IsFetchable returns, whether the fetch definition refers to a fetchable resource
@@ -32,8 +30,7 @@ type ContentFetcher struct {
 		json  map[string]interface{}
 		mutex sync.Mutex
 	}
-	httpContentLoader ContentLoader
-	fileContentLoader ContentLoader
+	Loader ContentLoader
 }
 
 // NewContentFetcher creates a ContentFetcher with an HtmlContentParser as default.
@@ -42,8 +39,7 @@ type ContentFetcher struct {
 func NewContentFetcher(defaultMetaJSON map[string]interface{}) *ContentFetcher {
 	f := &ContentFetcher{}
 	f.r.results = make([]*FetchResult, 0, 0)
-	f.httpContentLoader = NewHttpContentLoader()
-	f.fileContentLoader = NewFileContentLoader()
+	f.Loader = NewHttpContentLoader()
 	f.meta.json = defaultMetaJSON
 	if f.meta.json == nil {
 		f.meta.json = make(map[string]interface{})
@@ -94,7 +90,7 @@ func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 		// want to override the original URL with expanded values
 		definitionCopy := *d
 		definitionCopy.URL = url
-		fetchResult.Content, fetchResult.Err = fetcher.fetch(&definitionCopy)
+		fetchResult.Content, fetchResult.Err = fetcher.Loader.Load(&definitionCopy)
 
 		if fetchResult.Err == nil {
 			fetcher.addMeta(fetchResult.Content.Meta())
@@ -112,13 +108,6 @@ func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 	}()
 }
 
-func (fetcher *ContentFetcher) fetch(fd *FetchDefinition) (Content, error) {
-	if strings.HasPrefix(fd.URL, FileURLPrefix) {
-		return fetcher.fileContentLoader.Load(fd)
-	}
-	return fetcher.httpContentLoader.Load(fd)
-}
-
 // isAlreadySheduled checks, if there is already a job for a FetchDefinition, or it is already fetched.
 // The method has to be called in a locked mutex block.
 func (fetcher *ContentFetcher) isAlreadySheduled(fetchDefinitionHash string) bool {

composition/content_fetcher_test.go

@@ -40,17 +40,6 @@ func Test_ContentFetcher_FetchDefinitionHash(t *testing.T) {
 			},
 			true,
 		},
-		{
-			&FetchDefinition{
-				URL:    "/foo",
-				Header: http.Header{"Some": {"header"}},
-			},
-			&FetchDefinition{
-				URL:    "/foo",
-				Header: http.Header{"Some": {"other header"}},
-			},
-			false,
-		},
 	}
 
 	for _, t := range tests {
@@ -73,7 +62,7 @@ func Test_ContentFetcher_FetchingWithDependency(t *testing.T) {
 	bazzFd := getFetchDefinitionMock(ctrl, loader, "/bazz", []*FetchDefinition{barFd}, time.Millisecond, map[string]interface{}{})
 
 	fetcher := NewContentFetcher(nil)
-	fetcher.httpContentLoader = loader
+	fetcher.Loader = loader
 
 	fetcher.AddFetchJob(fooFd)
 	fetcher.AddFetchJob(bazzFd)

composition/fetch_definition.go

@@ -1,8 +1,7 @@
 package composition
 
 import (
-	"crypto/md5"
-	"encoding/hex"
+	"github.com/tarent/lib-compose/cache"
 	"io"
 	"net/http"
 	"strings"
@@ -61,7 +60,7 @@ type FetchDefinition struct {
 	Body          io.Reader
 	RespProc      ResponseProcessor
 	ErrHandler    ErrorHandler
-	cacheStrategy CacheStrategy
+	CacheStrategy CacheStrategy
 	//ServeResponseHeaders bool
 	//IsPrimary            bool
 	//FallbackURL string
@@ -77,23 +76,25 @@ func NewFetchDefinitionWithErrorHandler(url string, errHandler ErrorHandler) *Fe
 		errHandler = NewDefaultErrorHandler()
 	}
 	return &FetchDefinition{
-		URL:        url,
-		Timeout:    DefaultTimeout,
-		Required:   true,
-		Method:     "GET",
-		ErrHandler: errHandler,
+		URL:           url,
+		Timeout:       DefaultTimeout,
+		Required:      true,
+		Method:        "GET",
+		ErrHandler:    errHandler,
+		CacheStrategy: cache.DefaultCacheStrategy,
 	}
 }
 
 // If a ResponseProcessor-Implementation is given it can be used to change the response before composition
 func NewFetchDefinitionWithResponseProcessor(url string, rp ResponseProcessor) *FetchDefinition {
 	return &FetchDefinition{
-		URL:        url,
-		Timeout:    DefaultTimeout,
-		Required:   true,
-		Method:     "GET",
-		RespProc:   rp,
-		ErrHandler: NewDefaultErrorHandler(),
+		URL:           url,
+		Timeout:       DefaultTimeout,
+		Required:      true,
+		Method:        "GET",
+		RespProc:      rp,
+		ErrHandler:    NewDefaultErrorHandler(),
+		CacheStrategy: cache.DefaultCacheStrategy,
 	}
 }
 
@@ -136,19 +137,15 @@ func NewFetchDefinitionWithResponseProcessorFromRequest(baseUrl string, r *http.
 // If two hashes of fetch resources are equal, they refer the same resource
 // and can e.g. be taken as replacement for each other. E.g. in case of caching.
 func (def *FetchDefinition) Hash() string {
-	if def.cacheStrategy != nil {
-		return def.cacheStrategy.Hash(def.Method, def.URL, def.Header)
+	if def.CacheStrategy != nil {
+		return def.CacheStrategy.Hash(def.Method, def.URL, def.Header)
 	}
-
-	hasher := md5.New()
-	hasher.Write([]byte(def.URL))
-	def.Header.Write(hasher)
-	return hex.EncodeToString(hasher.Sum(nil))
+	return def.URL
 }
 
-func (def *FetchDefinition) IsCachable(responseHeaders http.Header) bool {
-	if def.cacheStrategy != nil {
-		return def.cacheStrategy.IsCachable(def.Method, def.URL, def.Header, responseHeaders)
+func (def *FetchDefinition) IsCachable(responseStatus int, responseHeaders http.Header) bool {
+	if def.CacheStrategy != nil {
+		return def.CacheStrategy.IsCachable(def.Method, def.URL, responseStatus, def.Header, responseHeaders)
 	}
 	return false
 }

composition/interfaces.go

@@ -36,7 +36,7 @@ type FetchResultSupplier interface {
 
 type CacheStrategy interface {
 	Hash(method string, url string, requestHeader http.Header) string
-	IsCachable(method string, url string, requestHeader http.Header, responseHeader http.Header) bool
+	IsCachable(method string, url string, statusCode int, requestHeader http.Header, responseHeader http.Header) bool
 }
 
 // Vontent is the abstration over includable data.