From d7738973a7cc1d7c22de9b3aabb3c3b86a9e7f0b Mon Sep 17 00:00:00 2001 From: Valere JEANTET Date: Tue, 25 Jul 2017 23:06:57 +0200 Subject: [PATCH] core : webhook - delete processor's routes on processor stop, handle panic --- core/agent.go | 6 +- core/webhook.go | 88 ++-- processors/base.go | 5 + processors/processor.go | 1 + processors/webhook.go | 3 +- vendor/github.com/justinas/alice/LICENSE | 20 + vendor/github.com/justinas/alice/README.md | 94 +++++ vendor/github.com/justinas/alice/chain.go | 112 ++++++ vendor/github.com/vjeantet/go.enmime/LICENSE | 20 + .../github.com/vjeantet/go.enmime/README.md | 24 ++ .../github.com/vjeantet/go.enmime/base64.go | 43 ++ vendor/github.com/vjeantet/go.enmime/doc.go | 29 ++ .../github.com/vjeantet/go.enmime/header.go | 375 ++++++++++++++++++ vendor/github.com/vjeantet/go.enmime/mail.go | 152 +++++++ vendor/github.com/vjeantet/go.enmime/match.go | 107 +++++ vendor/github.com/vjeantet/go.enmime/part.go | 231 +++++++++++ vendor/golang.org/x/sync/LICENSE | 27 ++ vendor/golang.org/x/sync/PATENTS | 22 + vendor/golang.org/x/sync/syncmap/map.go | 372 +++++++++++++++++ vendor/vendor.json | 18 + 20 files changed, 1722 insertions(+), 27 deletions(-) create mode 100644 vendor/github.com/justinas/alice/LICENSE create mode 100644 vendor/github.com/justinas/alice/README.md create mode 100644 vendor/github.com/justinas/alice/chain.go create mode 100644 vendor/github.com/vjeantet/go.enmime/LICENSE create mode 100644 vendor/github.com/vjeantet/go.enmime/README.md create mode 100644 vendor/github.com/vjeantet/go.enmime/base64.go create mode 100644 vendor/github.com/vjeantet/go.enmime/doc.go create mode 100644 vendor/github.com/vjeantet/go.enmime/header.go create mode 100644 vendor/github.com/vjeantet/go.enmime/mail.go create mode 100644 vendor/github.com/vjeantet/go.enmime/match.go create mode 100644 vendor/github.com/vjeantet/go.enmime/part.go create mode 100644 vendor/golang.org/x/sync/LICENSE create mode 100644 vendor/golang.org/x/sync/PATENTS create mode 100644 vendor/golang.org/x/sync/syncmap/map.go diff --git a/core/agent.go b/core/agent.go index b8082835..cae27f20 100644 --- a/core/agent.go +++ b/core/agent.go @@ -66,7 +66,7 @@ func (a *agent) configure(conf *config.Agent) error { ctx.dataLocation = filepath.Join(dataLocation, conf.Type) ctx.configWorkingLocation = conf.Wd ctx.memory = myStore.Space(conf.Type) - ctx.webHook = newWebHook(conf.Label) + ctx.webHook = newWebHook(conf.PipelineName, conf.Label) Log().Debugf("data location : %s", ctx.dataLocation) if _, err := os.Stat(ctx.dataLocation); os.IsNotExist(err) { @@ -208,6 +208,10 @@ func (a *agent) stop() { myScheduler.Remove(a.Label) Log().Debugf("agent %d schedule job removed", a.ID) + // unregister processor's webhooks URLs + a.processor.B().WebHook.Unregister() + Log().Debugf("agent %d webhook routes unregistered", a.ID) + Log().Debugf("Processor '%s' stopping... - %d in pipe ", a.Label, len(a.packetChan)) close(a.packetChan) <-a.Done diff --git a/core/webhook.go b/core/webhook.go index 6611fa92..7df3f394 100644 --- a/core/webhook.go +++ b/core/webhook.go @@ -1,51 +1,89 @@ package core import ( + "fmt" "net/http" + "golang.org/x/sync/syncmap" + "github.com/gosimple/slug" + "github.com/justinas/alice" ) -var webHookMap = map[string]*webHook{} - type webHook struct { - mux *http.ServeMux - namespace string - uri string - Hooks map[string]*hook + pipelineLabel string + namespace string + Hooks []string } -type hook struct { - Url string - handler *func(http.ResponseWriter, *http.Request) +var webHookMap = syncmap.Map{} +var httpHookServerMux *http.ServeMux + +func newWebHook(pipelineLabel, nameSpace string) *webHook { + return &webHook{pipelineLabel: pipelineLabel, namespace: nameSpace, Hooks: []string{}} } -func newWebHook(nameSpace string) *webHook { - return &webHook{namespace: nameSpace, mux: httpHookServerMux, Hooks: map[string]*hook{}} +// Add a new route to a given http.HandlerFunc +func (w *webHook) Add(hookName string, hf http.HandlerFunc) { + hUrl := "/" + slug.Make(w.pipelineLabel) + "/" + slug.Make(w.namespace) + "/" + slug.Make(hookName) + w.Hooks = append(w.Hooks, hookName) + webHookMap.Store(hUrl, hf) + Log().Infof("Hook %s => %s", hookName, hUrl) } -// Add register a new route matcher linked to hh -func (w *webHook) Add(hookName string, hh func(http.ResponseWriter, *http.Request)) { - h := &hook{} - h.Url = slug.Make(w.namespace) + "/" + slug.Make(hookName) - h.handler = &hh - w.Hooks[hookName] = h - w.mux.HandleFunc("/"+h.Url, *h.handler) - Log().Infof("Hook %s => %s", hookName, "/"+h.Url) +// Delete a route +func (w *webHook) Delete(hookName string) { + hUrl := "/" + slug.Make(w.pipelineLabel) + "/" + slug.Make(w.namespace) + "/" + slug.Make(hookName) + webHookMap.Delete(hUrl) + Log().Debugf("WebHook unregisted [%s]", hUrl) } -func getAgentHooks(agentName string) (hooks map[string]*hook) { - if _, ok := webHookMap[agentName]; ok { - hooks = webHookMap[agentName].Hooks +// Delete all routes belonging to webHook +func (w *webHook) Unregister() { + for _, hookName := range w.Hooks { + w.Delete(hookName) } - return hooks } -var httpHookServerMux *http.ServeMux +func routerHandler(w http.ResponseWriter, r *http.Request) { + if hfi, ok := webHookMap.Load(r.URL.Path); ok { + Log().Debugf("Webhook found for %s", r.URL.Path) + hfi.(http.HandlerFunc)(w, r) + } else { + Log().Warnf("Webhook not found for %s", r.URL.Path) + w.WriteHeader(404) + fmt.Fprint(w, "Not Found !") + } +} func listenAndServeWebHook(addr string) { httpHookServerMux = http.NewServeMux() + commonHandlers := alice.New(loggingHandler, recoverHandler) + httpHookServerMux.Handle("/", commonHandlers.ThenFunc(routerHandler)) + go http.ListenAndServe(addr, httpHookServerMux) Log().Infof("Agents webHook listening on %s", addr) - go http.ListenAndServe(addr, httpHookServerMux) +} + +func loggingHandler(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + Log().Debugf("Webhook [%s] %s", r.Method, r.URL.Path) + } + return http.HandlerFunc(fn) +} + +func recoverHandler(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + Log().Errorf("Webhook panic [%s] %s : %+v", r.Method, r.URL.Path, err) + http.Error(w, http.StatusText(500), 500) + } + }() + + next.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) } diff --git a/processors/base.go b/processors/base.go index 002edf84..8d019b75 100644 --- a/processors/base.go +++ b/processors/base.go @@ -19,6 +19,11 @@ type Base struct { PipelineID int } +// B returns the Base Processor +func (b *Base) B() *Base { + return b +} + func (b *Base) Doc() *doc.Processor { return &doc.Processor{} } diff --git a/processors/processor.go b/processors/processor.go index 896cf327..cedfe7b7 100644 --- a/processors/processor.go +++ b/processors/processor.go @@ -3,6 +3,7 @@ package processors import "github.com/vjeantet/bitfan/processors/doc" type Processor interface { + B() *Base Configure(ProcessorContext, map[string]interface{}) error Start(IPacket) error Tick(IPacket) error diff --git a/processors/webhook.go b/processors/webhook.go index 7c1039f5..88ed4464 100644 --- a/processors/webhook.go +++ b/processors/webhook.go @@ -3,5 +3,6 @@ package processors import "net/http" type WebHook interface { - Add(string, func(http.ResponseWriter, *http.Request)) + Add(string, http.HandlerFunc) + Unregister() } diff --git a/vendor/github.com/justinas/alice/LICENSE b/vendor/github.com/justinas/alice/LICENSE new file mode 100644 index 00000000..0d0d352e --- /dev/null +++ b/vendor/github.com/justinas/alice/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Justinas Stankevicius + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/justinas/alice/README.md b/vendor/github.com/justinas/alice/README.md new file mode 100644 index 00000000..9877df70 --- /dev/null +++ b/vendor/github.com/justinas/alice/README.md @@ -0,0 +1,94 @@ +# Alice + +[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](http://godoc.org/github.com/justinas/alice) +[![Build Status](https://travis-ci.org/justinas/alice.svg?branch=master)](https://travis-ci.org/justinas/alice) +[![Coverage](http://gocover.io/_badge/github.com/justinas/alice)](http://gocover.io/github.com/justinas/alice) + +Alice provides a convenient way to chain +your HTTP middleware functions and the app handler. + +In short, it transforms + + Middleware1(Middleware2(Middleware3(App))) + +to + + alice.New(Middleware1, Middleware2, Middleware3).Then(App) + +### Why? + +None of the other middleware chaining solutions +behaves exactly like Alice. +Alice is as minimal as it gets: +in essence, it's just a for loop that does the wrapping for you. + +Check out [this blog post](http://justinas.org/alice-painless-middleware-chaining-for-go/) +for explanation how Alice is different from other chaining solutions. + +### Usage + +Your middleware constructors should have the form of + + func (http.Handler) http.Handler + +Some middleware provide this out of the box. +For ones that don't, it's trivial to write one yourself. + +```go +func myStripPrefix(h http.Handler) http.Handler { + return http.StripPrefix("/old", h) +} +``` + +This complete example shows the full power of Alice. + +```go +package main + +import ( + "net/http" + "time" + + "github.com/throttled/throttled" + "github.com/justinas/alice" + "github.com/justinas/nosurf" +) + +func timeoutHandler(h http.Handler) http.Handler { + return http.TimeoutHandler(h, 1*time.Second, "timed out") +} + +func myApp(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Hello world!")) +} + +func main() { + th := throttled.Interval(throttled.PerSec(10), 1, &throttled.VaryBy{Path: true}, 50) + myHandler := http.HandlerFunc(myApp) + + chain := alice.New(th.Throttle, timeoutHandler, nosurf.NewPure).Then(myHandler) + http.ListenAndServe(":8000", chain) +} +``` + +Here, the request will pass [throttled](https://github.com/PuerkitoBio/throttled) first, +then an http.TimeoutHandler we've set up, +then [nosurf](https://github.com/justinas/nosurf) +and will finally reach our handler. + +Note that Alice makes **no guarantees** for +how one or another piece of middleware will behave. +It executes all middleware sequentially so that if a +piece of middleware were to stop the chain, +the request will not reach the inner handlers. +This is intentional behavior. + +Alice works with Go 1.0 and higher, +but running tests requires at least Go 1.1. + +### Contributing + +0. Find an issue that bugs you / open a new one. +1. Discuss. +2. Branch off, commit, test. +3. Make a pull request / attach the commits to the issue. diff --git a/vendor/github.com/justinas/alice/chain.go b/vendor/github.com/justinas/alice/chain.go new file mode 100644 index 00000000..c75f9c9c --- /dev/null +++ b/vendor/github.com/justinas/alice/chain.go @@ -0,0 +1,112 @@ +// Package alice provides a convenient way to chain http handlers. +package alice + +import "net/http" + +// A constructor for a piece of middleware. +// Some middleware use this constructor out of the box, +// so in most cases you can just pass somepackage.New +type Constructor func(http.Handler) http.Handler + +// Chain acts as a list of http.Handler constructors. +// Chain is effectively immutable: +// once created, it will always hold +// the same set of constructors in the same order. +type Chain struct { + constructors []Constructor +} + +// New creates a new chain, +// memorizing the given list of middleware constructors. +// New serves no other function, +// constructors are only called upon a call to Then(). +func New(constructors ...Constructor) Chain { + return Chain{append(([]Constructor)(nil), constructors...)} +} + +// Then chains the middleware and returns the final http.Handler. +// New(m1, m2, m3).Then(h) +// is equivalent to: +// m1(m2(m3(h))) +// When the request comes in, it will be passed to m1, then m2, then m3 +// and finally, the given handler +// (assuming every middleware calls the following one). +// +// A chain can be safely reused by calling Then() several times. +// stdStack := alice.New(ratelimitHandler, csrfHandler) +// indexPipe = stdStack.Then(indexHandler) +// authPipe = stdStack.Then(authHandler) +// Note that constructors are called on every call to Then() +// and thus several instances of the same middleware will be created +// when a chain is reused in this way. +// For proper middleware, this should cause no problems. +// +// Then() treats nil as http.DefaultServeMux. +func (c Chain) Then(h http.Handler) http.Handler { + if h == nil { + h = http.DefaultServeMux + } + + for i := range c.constructors { + h = c.constructors[len(c.constructors)-1-i](h) + } + + return h +} + +// ThenFunc works identically to Then, but takes +// a HandlerFunc instead of a Handler. +// +// The following two statements are equivalent: +// c.Then(http.HandlerFunc(fn)) +// c.ThenFunc(fn) +// +// ThenFunc provides all the guarantees of Then. +func (c Chain) ThenFunc(fn http.HandlerFunc) http.Handler { + if fn == nil { + return c.Then(nil) + } + return c.Then(http.HandlerFunc(fn)) +} + +// Append extends a chain, adding the specified constructors +// as the last ones in the request flow. +// +// Append returns a new chain, leaving the original one untouched. +// +// stdChain := alice.New(m1, m2) +// extChain := stdChain.Append(m3, m4) +// // requests in stdChain go m1 -> m2 +// // requests in extChain go m1 -> m2 -> m3 -> m4 +func (c Chain) Append(constructors ...Constructor) Chain { + newCons := make([]Constructor, len(c.constructors)+len(constructors)) + copy(newCons, c.constructors) + copy(newCons[len(c.constructors):], constructors) + + return New(newCons...) +} + +// Extend extends a chain by adding the specified chain +// as the last one in the request flow. +// +// Extend returns a new chain, leaving the original one untouched. +// +// stdChain := alice.New(m1, m2) +// ext1Chain := alice.New(m3, m4) +// ext2Chain := stdChain.Extend(ext1Chain) +// // requests in stdChain go m1 -> m2 +// // requests in ext1Chain go m3 -> m4 +// // requests in ext2Chain go m1 -> m2 -> m3 -> m4 +// +// Another example: +// aHtmlAfterNosurf := alice.New(m2) +// aHtml := alice.New(m1, func(h http.Handler) http.Handler { +// csrf := nosurf.New(h) +// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail)) +// return csrf +// }).Extend(aHtmlAfterNosurf) +// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler +// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail +func (c Chain) Extend(chain Chain) Chain { + return c.Append(chain.constructors...) +} diff --git a/vendor/github.com/vjeantet/go.enmime/LICENSE b/vendor/github.com/vjeantet/go.enmime/LICENSE new file mode 100644 index 00000000..01d5c176 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2012 James Hillyerd, All Rights Reserved + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/vjeantet/go.enmime/README.md b/vendor/github.com/vjeantet/go.enmime/README.md new file mode 100644 index 00000000..7a179d17 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/README.md @@ -0,0 +1,24 @@ +enmime [![Build Status](https://travis-ci.org/jhillyerd/go.enmime.png?branch=master)](https://travis-ci.org/jhillyerd/go.enmime) [![GoDoc](https://godoc.org/github.com/jhillyerd/go.enmime?status.png)](https://godoc.org/github.com/jhillyerd/go.enmime) +====== + +enmime is a MIME parsing library for Go. It's built ontop of Go's included mime/multipart +support, but is geared towards parsing MIME encoded emails. + +It is being developed in tandem with the Inbucket email service. + +API documentation can be found here: +http://godoc.org/github.com/jhillyerd/go.enmime + +Development Status +------------------ +enmime is alpha quality: it works but has not been tested with a wide variety of source data, +and it's likely the API will evolve some before an official release. + +About +----- +enmime is written in [Google Go][1]. + +enmime is open source software released under the MIT License. The latest +version can be found at https://github.com/jhillyerd/go.enmime + +[1]: http://golang.org/ diff --git a/vendor/github.com/vjeantet/go.enmime/base64.go b/vendor/github.com/vjeantet/go.enmime/base64.go new file mode 100644 index 00000000..3ea3f5f9 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/base64.go @@ -0,0 +1,43 @@ +package enmime + +import ( + "io" +) + +// Base64Cleaner helps work around bugs in Go's built-in base64 decoder by stripping out +// whitespace that would cause Go to lose count of things and issue an "illegal base64 data at +// input byte..." error +type Base64Cleaner struct { + in io.Reader + buf [1024]byte + //count int64 +} + +// NewBase64Cleaner returns a Base64Cleaner object for the specified reader. Base64Cleaner +// implements the io.Reader interface. +func NewBase64Cleaner(r io.Reader) *Base64Cleaner { + return &Base64Cleaner{in: r} +} + +// Read method for io.Reader interface. +func (qp *Base64Cleaner) Read(p []byte) (n int, err error) { + // Size our slice to theirs + size := len(qp.buf) + if len(p) < size { + size = len(p) + } + buf := qp.buf[:size] + bn, err := qp.in.Read(buf) + for i := 0; i < bn; i++ { + switch buf[i] { + case ' ', '\t', '\r', '\n': + // Strip these + default: + p[n] = buf[i] + n++ + } + } + // Count may be useful if I need to pad to even quads + //qp.count += int64(n) + return n, err +} diff --git a/vendor/github.com/vjeantet/go.enmime/doc.go b/vendor/github.com/vjeantet/go.enmime/doc.go new file mode 100644 index 00000000..ff990f31 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/doc.go @@ -0,0 +1,29 @@ +/* + Package enmime implements a MIME parsing library for Go. It's built ontop + of Go's included mime/multipart support, but is geared towards parsing MIME + encoded emails. + + The basics: + + Calling ParseMIMEBody causes enmime to parse the body of the message object + into a tree of MIMEPart objects, each of which is aware of its content + type, filename and headers. If the part was encoded in quoted-printable or + base64, it is decoded before being stored in the MIMEPart object. + + ParseMIMEBody returns a MIMEBody struct. The struct contains both the + plain text and HTML portions of the email (if available). The root of the + tree, as well as slices of the email's inlines and attachments are + available in the struct. + + If you need to locate a particular MIMEPart, you can pass a custom + MIMEPartMatcher function into BreadthMatchFirst() or DepthMatchFirst() to + search the MIMEPart tree. BreadthMatchAll() and DepthMatchAll() will + collect all matching parts. + + Please note that enmime parses messages into memory, so it is not likely to + perform well with multi-gigabyte attachments. + + enmime is open source software released under the MIT License. The latest + version can be found at https://github.com/jhillyerd/go.enmime +*/ +package enmime diff --git a/vendor/github.com/vjeantet/go.enmime/header.go b/vendor/github.com/vjeantet/go.enmime/header.go new file mode 100644 index 00000000..25bbdfc0 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/header.go @@ -0,0 +1,375 @@ +package enmime + +import ( + "bytes" + "encoding/base64" + "fmt" + "strconv" + "strings" + + "github.com/axgle/mahonia" +) + +func debug(format string, args ...interface{}) { + if false { + fmt.Printf(format, args...) + fmt.Println() + } +} + +// Terminology from RFC 2047: +// encoded-word: the entire =?charset?encoding?encoded-text?= string +// charset: the character set portion of the encoded word +// encoding: the character encoding type used for the encoded-text +// encoded-text: the text we are decoding + +// State function modeled on Rob Pike's lexer talk (see source of +// Go's text/template/parser/lex.go) +type stateFn func(*headerDec) stateFn + +const eof = -1 + +// headerDec holds the state of the scanner and an output buffer +type headerDec struct { + input []byte // Input to decode + state stateFn // Next state + start int // Start of text we don't yet know what to do with + pos int // Current parsing position + charset string // Character set of current encoded word + encoding string // Encoding of current encoded word + outbuf bytes.Buffer +} + +// eof returns true if we've read the last rune +func (h *headerDec) eof() bool { + return h.pos >= len(h.input) +} + +// next returns the next rune in the input +func (h *headerDec) next() rune { + if h.eof() { + return eof + } + r := h.input[h.pos] + h.pos++ + return rune(r) +} + +// backup a single rune +func (h *headerDec) backup() { + if h.pos > 0 { + h.pos-- + } +} + +// peek at the next rune without consuming it +func (h *headerDec) peek() rune { + if h.eof() { + return eof + } + r := h.next() + h.backup() + return r +} + +// ignore will forget all input between start and pos +func (h *headerDec) ignore() { + h.start = h.pos +} + +// output will append all input from start to pos (inclusive) to outbuf +func (h *headerDec) output() { + if h.pos > h.start { + h.outbuf.Write(h.input[h.start:h.pos]) + h.start = h.pos + } +} + +// accept consumes the next rune if it's part of the valid set +func (h *headerDec) accept(valid string) bool { + if r := h.next(); r != eof { + if strings.IndexRune(valid, r) >= 0 { + return true + } + h.backup() + } + return false +} + +// Decode a MIME header per RFC 2047 +func decodeHeader(input string) string { + if !strings.Contains(input, "=?") { + // Don't scan if there is nothing to do here + return input + } + + h := &headerDec{ + input: []byte(input), + state: plainSpaceState, + } + + debug("Starting parse of: '%v'\n", input) + + for h.state != nil { + h.state = h.state(h) + } + + return h.outbuf.String() +} + +// State: Reset, output mangled encoded-word as plaintext +// +// There was a problem parsing an encoded-word, recover by outputting +// plain-text until next encoded word +func resetState(h *headerDec) stateFn { + debug("entering reset state with buf %q", h.outbuf.String()) + h.output() + return plainTextState +} + +// State: In plain space - we want to output this space, and it is legal to transition into +// an encoded word from here. +func plainSpaceState(h *headerDec) stateFn { + debug("entering plain space state with buf %q", h.outbuf.String()) + for !h.eof() { + switch { + case h.accept("="): + // Possible encoded word, dump out preceeding plaintext, w/o leading = + h.backup() + h.output() + h.next() + if h.accept("?") { + return charsetState + } + case h.accept(" \t\r\n("): + // '(' is the start of a MIME header "comment", so it is still legal to transition + // to an encoded word after it + default: + // Hit some plain text + return plainTextState + } + } + // Hitting EOF in plain space state means we are done + h.output() + return nil +} + +// State: In plain text - we want to output this text. It is not legal to transition into +// an encoded word from here! +func plainTextState(h *headerDec) stateFn { + debug("entering plain text state with buf %q", h.outbuf.String()) + for !h.eof() { + if h.accept(" \t\r\n(") { + // TODO Not sure if '(' belongs here, maybe require space first? + // Whitespace character + h.backup() + return plainSpaceState + } + h.next() + } + // Hitting EOF in plain text state means we are done + h.output() + return nil +} + +// State: In encoded-word charset name +func charsetState(h *headerDec) stateFn { + debug("entering charset state with buf %q", h.outbuf.String()) + myStart := h.pos + for r := h.next(); r != eof; r = h.next() { + // Parse character set + switch { + case isTokenChar(r): + // Part of charset name, keep going + case r == '?': + // End of charset name + h.charset = string(h.input[myStart : h.pos-1]) + debug("charset %q", h.charset) + return encodingState + default: + // Invalid character + return resetState + } + } + // Hit eof! + return resetState +} + +// State: In encoded-word encoding name +func encodingState(h *headerDec) stateFn { + debug("entering encoding state with buf %q", h.outbuf.String()) + myStart := h.pos + for r := h.next(); r != eof; r = h.next() { + // Parse encoding + switch { + case isTokenChar(r): + // Part of encoding name, keep going + case r == '?': + // End of encoding name + h.encoding = string(h.input[myStart : h.pos-1]) + debug("encoding %q", h.encoding) + return encTextState + default: + // Invalid character + debug("invalid character") + return resetState + } + } + // Hit eof! + debug("hit unexpected eof") + return resetState +} + +// State: In encoded-text +func encTextState(h *headerDec) stateFn { + debug("entering encText state with buf %q", h.outbuf.String()) + myStart := h.pos + for r := h.next(); r != eof; r = h.next() { + // Decode encoded-text + switch { + case r < 33: + // No controls or space allowed + debug("Encountered control character") + return resetState + case r > 126: + // No DEL or extended ascii allowed + debug("Encountered DEL or extended ascii") + return resetState + case r == '?': + if h.accept("=") { + text, err := convertText(h.charset, h.encoding, h.input[myStart:h.pos-2]) + if err == nil { + debug("Text converted to: %q", text) + h.outbuf.WriteString(text) + h.ignore() + // Entering post-word space + return spaceState + } else { + // Conversion failed + debug("Text conversion failed: %q", err) + return resetState + } + } else { + // Invalid termination + debug("Invalid termination") + return resetState + } + } + } + // Hit eof! + debug("Hit eof early") + return resetState +} + +// State: White space following an encoded-word +func spaceState(h *headerDec) stateFn { + debug("entering space state with buf %q", h.outbuf.String()) +Loop: + for { + // Eat space characters only between encoded words + switch { + case h.accept(" \t\r\n"): + debug("In space") + // Still in space + case h.accept("="): + debug("In =") + if h.accept("?") { + // Start of new encoded word. If the word is valid, we want to eat + // the whitespace. If not, h.start was set in transition to SPACE, + // and we will output the space. + return charsetState + } + default: + debug("In default") + break Loop + } + } + debug("In plain") + // We hit plain text, will need to output whitespace + h.output() + return plainTextState +} + +// Convert the encTextBytes to UTF-8 and return as a string +func convertText(charsetName string, encoding string, encTextBytes []byte) (string, error) { + // Setup mahonia to convert bytes to UTF-8 string + charset := mahonia.GetCharset(charsetName) + if charset == nil { + // Unknown charset + return "", fmt.Errorf("Unknown (to mahonia) charset: %q", charsetName) + } + decoder := charset.NewDecoder() + + // Unpack quoted-printable or base64 first + var textBytes []byte + var err error + switch strings.ToLower(encoding) { + case "b": + // Base64 encoded + textBytes, err = decodeBase64(encTextBytes) + case "q": + // Quoted printable encoded + textBytes, err = decodeQuotedPrintable(encTextBytes) + default: + err = fmt.Errorf("Invalid encoding: %v", encoding) + } + if err != nil { + return "", err + } + + return decoder.ConvertString(string(textBytes)), nil +} + +func decodeQuotedPrintable(input []byte) ([]byte, error) { + output := make([]byte, 0, len(input)) + for pos := 0; pos < len(input); pos++ { + switch ch := input[pos]; ch { + case '_': + output = append(output, ' ') + case '=': + if len(input) < pos+3 { + return nil, fmt.Errorf("Ran out of chars parsing: %v", input[pos:]) + } + x, err := strconv.ParseInt(string(input[pos+1:pos+3]), 16, 64) + if err != nil { + return nil, fmt.Errorf("Failed to convert: %v", input[pos:pos+3]) + } + output = append(output, byte(x)) + pos += 2 + default: + output = append(output, input[pos]) + } + } + return output, nil +} + +func decodeBase64(input []byte) ([]byte, error) { + output := make([]byte, len(input)) + n, err := base64.StdEncoding.Decode(output, input) + return output[:n], err +} + +// Is this an especial character per RFC 2047 +func isEspecialChar(ch rune) bool { + switch ch { + case '(', ')', '<', '>', '@', ',', ';', ':': + return true + case '"', '/', '[', ']', '?', '.', '=': + return true + } + return false +} + +// Is this a "token" character per RFC 2047 +func isTokenChar(ch rune) bool { + // No controls or space + if ch < 33 { + return false + } + // No DEL or extended ascii + if ch > 126 { + return false + } + // No especials + return !isEspecialChar(ch) +} diff --git a/vendor/github.com/vjeantet/go.enmime/mail.go b/vendor/github.com/vjeantet/go.enmime/mail.go new file mode 100644 index 00000000..d97e126f --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/mail.go @@ -0,0 +1,152 @@ +package enmime + +import ( + "fmt" + "mime" + "net/mail" + "strings" +) + +// MIMEBody is the outer wrapper for MIME messages. +type MIMEBody struct { + Text string // The plain text portion of the message + Html string // The HTML portion of the message + Root MIMEPart // The top-level MIMEPart + Attachments []MIMEPart // All parts having a Content-Disposition of attachment + Inlines []MIMEPart // All parts having a Content-Disposition of inline + OtherParts []MIMEPart // All parts not in Attachments and Inlines + header mail.Header // Header from original message +} + +// IsMultipartMessage returns true if the message has a recognized multipart Content-Type +// header. You don't need to check this before calling ParseMIMEBody, it can handle +// non-multipart messages. +func IsMultipartMessage(mailMsg *mail.Message) bool { + // Parse top-level multipart + ctype := mailMsg.Header.Get("Content-Type") + mediatype, _, err := mime.ParseMediaType(ctype) + if err != nil { + return false + } + switch mediatype { + case "multipart/alternative", + "multipart/mixed", + "multipart/related", + "multipart/signed": + return true + } + + return false +} + +// ParseMIMEBody parses the body of the message object into a tree of MIMEPart objects, +// each of which is aware of its content type, filename and headers. If the part was +// encoded in quoted-printable or base64, it is decoded before being stored in the +// MIMEPart object. +func ParseMIMEBody(mailMsg *mail.Message) (*MIMEBody, error) { + mimeMsg := &MIMEBody{header: mailMsg.Header} + + if !IsMultipartMessage(mailMsg) { + // Parse as text only + bodyBytes, err := decodeSection(mailMsg.Header.Get("Content-Transfer-Encoding"), + mailMsg.Body) + if err != nil { + return nil, fmt.Errorf("Error decoding text-only message: %v", err) + } + mimeMsg.Text = string(bodyBytes) + + // Check for HTML at top-level, eat errors quietly + ctype := mailMsg.Header.Get("Content-Type") + if ctype != "" { + if mediatype, _, err := mime.ParseMediaType(ctype); err == nil { + if mediatype == "text/html" { + mimeMsg.Html = mimeMsg.Text + } + } + } + } else { + // Parse top-level multipart + ctype := mailMsg.Header.Get("Content-Type") + mediatype, params, err := mime.ParseMediaType(ctype) + if err != nil { + return nil, fmt.Errorf("Unable to parse media type: %v", err) + } + if !strings.HasPrefix(mediatype, "multipart/") { + return nil, fmt.Errorf("Unknown mediatype: %v", mediatype) + } + boundary := params["boundary"] + if boundary == "" { + return nil, fmt.Errorf("Unable to locate boundary param in Content-Type header") + } + // Root Node of our tree + root := NewMIMEPart(nil, mediatype) + mimeMsg.Root = root + err = parseParts(root, mailMsg.Body, boundary) + if err != nil { + return nil, err + } + + // Locate text body + if mediatype == "multipart/altern" { + match := BreadthMatchFirst(root, func(p MIMEPart) bool { + return p.ContentType() == "text/plain" && p.Disposition() != "attachment" + }) + if match != nil { + mimeMsg.Text = string(match.Content()) + } + } else { + // multipart is of a mixed type + match := DepthMatchAll(root, func(p MIMEPart) bool { + return p.ContentType() == "text/plain" && p.Disposition() != "attachment" + }) + for i, m := range match { + if i > 0 { + mimeMsg.Text += "\n--\n" + } + mimeMsg.Text += string(m.Content()) + } + } + + // Locate HTML body + match := BreadthMatchFirst(root, func(p MIMEPart) bool { + return p.ContentType() == "text/html" && p.Disposition() != "attachment" + }) + if match != nil { + mimeMsg.Html = string(match.Content()) + } + + // Locate attachments + mimeMsg.Attachments = BreadthMatchAll(root, func(p MIMEPart) bool { + return p.Disposition() == "attachment" || p.ContentType() == "application/octet-stream" + }) + + // Locate inlines + mimeMsg.Inlines = BreadthMatchAll(root, func(p MIMEPart) bool { + return p.Disposition() == "inline" + }) + + // Locate others parts not handled in "Attachments" and "inlines" + mimeMsg.OtherParts = BreadthMatchAll(root, func(p MIMEPart) bool { + if strings.HasPrefix(p.ContentType(), "multipart/") { + return false + } + + if p.Disposition() != "" { + return false + } + + if p.ContentType() == "application/octet-stream" { + return false + } + + return p.ContentType() != "text/plain" && p.ContentType() != "text/html" + }) + } + + return mimeMsg, nil +} + +// Process the specified header for RFC 2047 encoded words and return the result +func (m *MIMEBody) GetHeader(name string) string { + return decodeHeader(m.header.Get(name)) +} diff --git a/vendor/github.com/vjeantet/go.enmime/match.go b/vendor/github.com/vjeantet/go.enmime/match.go new file mode 100644 index 00000000..bc572f0f --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/match.go @@ -0,0 +1,107 @@ +package enmime + +import ( + "container/list" +) + +// MIMEPartMatcher is a function type that you must implement to search for MIMEParts using +// the BreadthMatch* functions. Implementators should inspect the provided MIMEPart and +// return true if it matches your criteria. +type MIMEPartMatcher func(part MIMEPart) bool + +// BreadthMatchFirst performs a breadth first search of the MIMEPart tree and returns the +// first part that causes the given matcher to return true +func BreadthMatchFirst(p MIMEPart, matcher MIMEPartMatcher) MIMEPart { + q := list.New() + q.PushBack(p) + + // Push children onto queue and attempt to match in that order + for q.Len() > 0 { + e := q.Front() + p := e.Value.(MIMEPart) + if matcher(p) { + return p + } + q.Remove(e) + c := p.FirstChild() + for c != nil { + q.PushBack(c) + c = c.NextSibling() + } + } + + return nil +} + +// BreadthMatchAll performs a breadth first search of the MIMEPart tree and returns all parts +// that cause the given matcher to return true +func BreadthMatchAll(p MIMEPart, matcher MIMEPartMatcher) []MIMEPart { + q := list.New() + q.PushBack(p) + + matches := make([]MIMEPart, 0, 10) + + // Push children onto queue and attempt to match in that order + for q.Len() > 0 { + e := q.Front() + p := e.Value.(MIMEPart) + if matcher(p) { + matches = append(matches, p) + } + q.Remove(e) + c := p.FirstChild() + for c != nil { + q.PushBack(c) + c = c.NextSibling() + } + } + + return matches +} + +// DepthMatchFirst performs a depth first search of the MIMEPart tree and returns the +// first part that causes the given matcher to return true +func DepthMatchFirst(p MIMEPart, matcher MIMEPartMatcher) MIMEPart { + root := p + for { + if matcher(p) { + return p + } + c := p.FirstChild() + if c != nil { + p = c + } else { + for p.NextSibling() == nil { + if p == root { + return nil + } + p = p.Parent() + } + p = p.NextSibling() + } + } +} + +// DepthMatchAll performs a depth first search of the MIMEPart tree and returns all parts +// that causes the given matcher to return true +func DepthMatchAll(p MIMEPart, matcher MIMEPartMatcher) []MIMEPart { + root := p + matches := make([]MIMEPart, 0, 10) + for { + if matcher(p) { + matches = append(matches, p) + } + c := p.FirstChild() + if c != nil { + p = c + } else { + for p.NextSibling() == nil { + if p == root { + return matches + } + p = p.Parent() + } + p = p.NextSibling() + } + } +} diff --git a/vendor/github.com/vjeantet/go.enmime/part.go b/vendor/github.com/vjeantet/go.enmime/part.go new file mode 100644 index 00000000..8a91f970 --- /dev/null +++ b/vendor/github.com/vjeantet/go.enmime/part.go @@ -0,0 +1,231 @@ +package enmime + +import ( + "bufio" + "bytes" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "net/textproto" + "strings" + + "github.com/sloonz/go-qprintable" +) + +// MIMEPart is the primary interface enmine clients will use. Each MIMEPart represents +// a node in the MIME multipart tree. The Content-Type, Disposition and File Name are +// parsed out of the header for easier access. +// +// TODO Content should probably be a reader so that it does not need to be stored in +// memory. +type MIMEPart interface { + Parent() MIMEPart // Parent of this part (can be nil) + FirstChild() MIMEPart // First (top most) child of this part + NextSibling() MIMEPart // Next sibling of this part + Header() textproto.MIMEHeader // Header as parsed by textproto package + ContentType() string // Content-Type header without parameters + Disposition() string // Content-Disposition header without parameters + FileName() string // File Name from disposition or type header + Content() []byte // Decoded content of this part (can be empty) +} + +// memMIMEPart is an in-memory implementation of the MIMEPart interface. It will likely +// choke on huge attachments. +type memMIMEPart struct { + parent MIMEPart + firstChild MIMEPart + nextSibling MIMEPart + header textproto.MIMEHeader + contentType string + disposition string + fileName string + content []byte +} + +// NewMIMEPart creates a new memMIMEPart object. It does not update the parents FirstChild +// attribute. +func NewMIMEPart(parent MIMEPart, contentType string) *memMIMEPart { + return &memMIMEPart{parent: parent, contentType: contentType} +} + +// Parent of this part (can be nil) +func (p *memMIMEPart) Parent() MIMEPart { + return p.parent +} + +// First (top most) child of this part +func (p *memMIMEPart) FirstChild() MIMEPart { + return p.firstChild +} + +// Next sibling of this part +func (p *memMIMEPart) NextSibling() MIMEPart { + return p.nextSibling +} + +// Header as parsed by textproto package +func (p *memMIMEPart) Header() textproto.MIMEHeader { + return p.header +} + +// Content-Type header without parameters +func (p *memMIMEPart) ContentType() string { + return p.contentType +} + +// Content-Disposition header without parameters +func (p *memMIMEPart) Disposition() string { + return p.disposition +} + +// File Name from disposition or type header +func (p *memMIMEPart) FileName() string { + return p.fileName +} + +// Decoded content of this part (can be empty) +func (p *memMIMEPart) Content() []byte { + return p.content +} + +// ParseMIME reads a MIME document from the provided reader and parses it into +// tree of MIMEPart objects. +func ParseMIME(reader *bufio.Reader) (MIMEPart, error) { + tr := textproto.NewReader(reader) + header, err := tr.ReadMIMEHeader() + if err != nil { + return nil, err + } + mediatype, params, err := mime.ParseMediaType(header.Get("Content-Type")) + if err != nil { + return nil, err + } + root := &memMIMEPart{header: header, contentType: mediatype} + + if strings.HasPrefix(mediatype, "multipart/") { + boundary := params["boundary"] + err = parseParts(root, reader, boundary) + if err != nil { + return nil, err + } + } else { + // Content is text or data, decode it + content, err := decodeSection(header.Get("Content-Transfer-Encoding"), reader) + if err != nil { + return nil, err + } + root.content = content + } + + return root, nil +} + +// parseParts recursively parses a mime multipart document. +func parseParts(parent *memMIMEPart, reader io.Reader, boundary string) error { + var prevSibling *memMIMEPart + + // Loop over MIME parts + mr := multipart.NewReader(reader, boundary) + for { + // mrp is golang's built in mime-part + mrp, err := mr.NextPart() + if err != nil { + if err == io.EOF { + // This is a clean end-of-message signal + break + } + return err + } + if len(mrp.Header) == 0 { + // Empty header probably means the part didn't using the correct trailing "--" + // syntax to close its boundary. We will let this slide if this this the + // last MIME part. + if _, err := mr.NextPart(); err != nil { + if err == io.EOF || strings.HasSuffix(err.Error(), "EOF") { + // This is what we were hoping for + break + } else { + return fmt.Errorf("Error at boundary %v: %v", boundary, err) + } + } + + return fmt.Errorf("Empty header at boundary %v", boundary) + } + ctype := mrp.Header.Get("Content-Type") + if ctype == "" { + return fmt.Errorf("Missing Content-Type at boundary %v", boundary) + } + mediatype, mparams, err := mime.ParseMediaType(ctype) + if err != nil { + return err + } + + // Insert ourselves into tree, p is enmime's mime-part + p := NewMIMEPart(parent, mediatype) + p.header = mrp.Header + if prevSibling != nil { + prevSibling.nextSibling = p + } else { + parent.firstChild = p + } + prevSibling = p + + // Figure out our disposition, filename + disposition, dparams, err := mime.ParseMediaType(mrp.Header.Get("Content-Disposition")) + if err == nil { + // Disposition is optional + p.disposition = disposition + p.fileName = dparams["filename"] + } + if p.fileName == "" && mparams["name"] != "" { + p.fileName = mparams["name"] + } + if p.fileName == "" && mparams["file"] != "" { + p.fileName = mparams["file"] + } + + boundary := mparams["boundary"] + if boundary != "" { + // Content is another multipart + err = parseParts(p, mrp, boundary) + if err != nil { + return err + } + } else { + // Content is text or data, decode it + data, err := decodeSection(mrp.Header.Get("Content-Transfer-Encoding"), mrp) + if err != nil { + return err + } + p.content = data + } + } + + return nil +} + +// decodeSection attempts to decode the data from reader using the algorithm listed in +// the Content-Transfer-Encoding header, returning the raw data if it does not known +// the encoding type. +func decodeSection(encoding string, reader io.Reader) ([]byte, error) { + // Default is to just read input into bytes + decoder := reader + + switch strings.ToLower(encoding) { + case "quoted-printable": + decoder = qprintable.NewDecoder(qprintable.WindowsTextEncoding, reader) + case "base64": + cleaner := NewBase64Cleaner(reader) + decoder = base64.NewDecoder(base64.StdEncoding, cleaner) + } + + // Read bytes into buffer + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(decoder) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 00000000..6a66aea5 --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 00000000..73309904 --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/syncmap/map.go b/vendor/golang.org/x/sync/syncmap/map.go new file mode 100644 index 00000000..80e15847 --- /dev/null +++ b/vendor/golang.org/x/sync/syncmap/map.go @@ -0,0 +1,372 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syncmap provides a concurrent map implementation. +// It is a prototype for a proposed addition to the sync package +// in the standard library. +// (https://golang.org/issue/18177) +package syncmap + +import ( + "sync" + "sync/atomic" + "unsafe" +) + +// Map is a concurrent map with amortized-constant-time loads, stores, and deletes. +// It is safe for multiple goroutines to call a Map's methods concurrently. +// +// The zero Map is valid and empty. +// +// A Map must not be copied after first use. +type Map struct { + mu sync.Mutex + + // read contains the portion of the map's contents that are safe for + // concurrent access (with or without mu held). + // + // The read field itself is always safe to load, but must only be stored with + // mu held. + // + // Entries stored in read may be updated concurrently without mu, but updating + // a previously-expunged entry requires that the entry be copied to the dirty + // map and unexpunged with mu held. + read atomic.Value // readOnly + + // dirty contains the portion of the map's contents that require mu to be + // held. To ensure that the dirty map can be promoted to the read map quickly, + // it also includes all of the non-expunged entries in the read map. + // + // Expunged entries are not stored in the dirty map. An expunged entry in the + // clean map must be unexpunged and added to the dirty map before a new value + // can be stored to it. + // + // If the dirty map is nil, the next write to the map will initialize it by + // making a shallow copy of the clean map, omitting stale entries. + dirty map[interface{}]*entry + + // misses counts the number of loads since the read map was last updated that + // needed to lock mu to determine whether the key was present. + // + // Once enough misses have occurred to cover the cost of copying the dirty + // map, the dirty map will be promoted to the read map (in the unamended + // state) and the next store to the map will make a new dirty copy. + misses int +} + +// readOnly is an immutable struct stored atomically in the Map.read field. +type readOnly struct { + m map[interface{}]*entry + amended bool // true if the dirty map contains some key not in m. +} + +// expunged is an arbitrary pointer that marks entries which have been deleted +// from the dirty map. +var expunged = unsafe.Pointer(new(interface{})) + +// An entry is a slot in the map corresponding to a particular key. +type entry struct { + // p points to the interface{} value stored for the entry. + // + // If p == nil, the entry has been deleted and m.dirty == nil. + // + // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry + // is missing from m.dirty. + // + // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty + // != nil, in m.dirty[key]. + // + // An entry can be deleted by atomic replacement with nil: when m.dirty is + // next created, it will atomically replace nil with expunged and leave + // m.dirty[key] unset. + // + // An entry's associated value can be updated by atomic replacement, provided + // p != expunged. If p == expunged, an entry's associated value can be updated + // only after first setting m.dirty[key] = e so that lookups using the dirty + // map find the entry. + p unsafe.Pointer // *interface{} +} + +func newEntry(i interface{}) *entry { + return &entry{p: unsafe.Pointer(&i)} +} + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *Map) Load(key interface{}) (value interface{}, ok bool) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + // Avoid reporting a spurious miss if m.dirty got promoted while we were + // blocked on m.mu. (If further loads of the same key will not miss, it's + // not worth copying the dirty map for this key.) + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + e, ok = m.dirty[key] + // Regardless of whether the entry was present, record a miss: this key + // will take the slow path until the dirty map is promoted to the read + // map. + m.missLocked() + } + m.mu.Unlock() + } + if !ok { + return nil, false + } + return e.load() +} + +func (e *entry) load() (value interface{}, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return nil, false + } + return *(*interface{})(p), true +} + +// Store sets the value for a key. +func (m *Map) Store(key, value interface{}) { + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok && e.tryStore(&value) { + return + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + // The entry was previously expunged, which implies that there is a + // non-nil dirty map and this entry is not in it. + m.dirty[key] = e + } + e.storeLocked(&value) + } else if e, ok := m.dirty[key]; ok { + e.storeLocked(&value) + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + } + m.mu.Unlock() +} + +// tryStore stores a value if the entry has not been expunged. +// +// If the entry is expunged, tryStore returns false and leaves the entry +// unchanged. +func (e *entry) tryStore(i *interface{}) bool { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + for { + if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { + return true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + } +} + +// unexpungeLocked ensures that the entry is not marked as expunged. +// +// If the entry was previously expunged, it must be added to the dirty map +// before m.mu is unlocked. +func (e *entry) unexpungeLocked() (wasExpunged bool) { + return atomic.CompareAndSwapPointer(&e.p, expunged, nil) +} + +// storeLocked unconditionally stores a value to the entry. +// +// The entry must be known not to be expunged. +func (e *entry) storeLocked(i *interface{}) { + atomic.StorePointer(&e.p, unsafe.Pointer(i)) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *Map) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) { + // Avoid locking if it's a clean hit. + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + actual, loaded, ok := e.tryLoadOrStore(value) + if ok { + return actual, loaded + } + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + m.dirty[key] = e + } + actual, loaded, _ = e.tryLoadOrStore(value) + } else if e, ok := m.dirty[key]; ok { + actual, loaded, _ = e.tryLoadOrStore(value) + m.missLocked() + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + actual, loaded = value, false + } + m.mu.Unlock() + + return actual, loaded +} + +// tryLoadOrStore atomically loads or stores a value if the entry is not +// expunged. +// +// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and +// returns with ok==false. +func (e *entry) tryLoadOrStore(i interface{}) (actual interface{}, loaded, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + + // Copy the interface after the first load to make this method more amenable + // to escape analysis: if we hit the "load" path or the entry is expunged, we + // shouldn't bother heap-allocating. + ic := i + for { + if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) { + return i, false, true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + } +} + +// Delete deletes the value for a key. +func (m *Map) Delete(key interface{}) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + delete(m.dirty, key) + } + m.mu.Unlock() + } + if ok { + e.delete() + } +} + +func (e *entry) delete() (hadValue bool) { + for { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return false + } + if atomic.CompareAndSwapPointer(&e.p, p, nil) { + return true + } + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// Range does not necessarily correspond to any consistent snapshot of the Map's +// contents: no key will be visited more than once, but if the value for any key +// is stored or deleted concurrently, Range may reflect any mapping for that key +// from any point during the Range call. +// +// Range may be O(N) with the number of elements in the map even if f returns +// false after a constant number of calls. +func (m *Map) Range(f func(key, value interface{}) bool) { + // We need to be able to iterate over all of the keys that were already + // present at the start of the call to Range. + // If read.amended is false, then read.m satisfies that property without + // requiring us to hold m.mu for a long time. + read, _ := m.read.Load().(readOnly) + if read.amended { + // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) + // (assuming the caller does not break out early), so a call to Range + // amortizes an entire copy of the map: we can promote the dirty copy + // immediately! + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if read.amended { + read = readOnly{m: m.dirty} + m.read.Store(read) + m.dirty = nil + m.misses = 0 + } + m.mu.Unlock() + } + + for k, e := range read.m { + v, ok := e.load() + if !ok { + continue + } + if !f(k, v) { + break + } + } +} + +func (m *Map) missLocked() { + m.misses++ + if m.misses < len(m.dirty) { + return + } + m.read.Store(readOnly{m: m.dirty}) + m.dirty = nil + m.misses = 0 +} + +func (m *Map) dirtyLocked() { + if m.dirty != nil { + return + } + + read, _ := m.read.Load().(readOnly) + m.dirty = make(map[interface{}]*entry, len(read.m)) + for k, e := range read.m { + if !e.tryExpungeLocked() { + m.dirty[k] = e + } + } +} + +func (e *entry) tryExpungeLocked() (isExpunged bool) { + p := atomic.LoadPointer(&e.p) + for p == nil { + if atomic.CompareAndSwapPointer(&e.p, nil, expunged) { + return true + } + p = atomic.LoadPointer(&e.p) + } + return p == expunged +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 7cc27734..3e146838 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -338,6 +338,12 @@ "revision": "ff71fe7a7d5279df4b964b31f7ee4adf117277f6", "revisionTime": "2015-07-17T04:28:40Z" }, + { + "checksumSHA1": "Lc5CWD5nzSttJNateoWlPgFE3Vw=", + "path": "github.com/justinas/alice", + "revision": "052b8b6c18edb9db317af86806d8e00ebaa94160", + "revisionTime": "2016-05-12T13:42:31Z" + }, { "checksumSHA1": "QVALLtQ7n6pt+YJiFrnng1O86PI=", "path": "github.com/k0kubun/pp", @@ -596,6 +602,12 @@ "revision": "7344c549c57e8da70a9579b1ad8b0b95232efa3d", "revisionTime": "2016-05-14T07:49:06Z" }, + { + "checksumSHA1": "yfYAIowXr2aRtR/LawlJiA/Ghtg=", + "path": "github.com/vjeantet/go.enmime", + "revision": "278de7067204c8910b01e8bce043ed8d530de0dd", + "revisionTime": "2016-05-16T09:47:39Z" + }, { "checksumSHA1": "yGLJPcNKlm3yPLp4d8x1E5iMiVI=", "path": "github.com/vjeantet/grok", @@ -650,6 +662,12 @@ "revision": "f2499483f923065a842d38eb4c7f1927e6fc6e6d", "revisionTime": "2017-01-14T04:22:49Z" }, + { + "checksumSHA1": "4TEYFKrAUuwBMqExjQBsnf/CgjQ=", + "path": "golang.org/x/sync/syncmap", + "revision": "f52d1811a62927559de87708c8913c1650ce4f26", + "revisionTime": "2017-05-17T20:25:26Z" + }, { "checksumSHA1": "aVgPDgwY3/t4J/JOw9H3FVMHqh0=", "path": "golang.org/x/sys/unix",