From 3d0c3ae7ab4a078922cb6150008b310b4b06ee91 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 1 May 2017 02:40:06 +0000 Subject: [PATCH 01/67] minor enhancements to API and Makefile --- Makefile | 2 +- api/v1/cmd/example-scheduler/app/app.go | 2 +- api/v1/lib/builders.go | 16 ++++++++++++++++ api/v1/lib/offers.go | 13 +++++++++++++ api/v1/lib/scheduler/calls/calls.go | 8 ++++++++ api/v1/lib/scheduler/events/events.go | 10 ++++++++++ 6 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 api/v1/lib/offers.go diff --git a/Makefile b/Makefile index 66629486..8bc5998f 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ $(COVERAGE_TARGETS): .PHONY: vet vet: - go $@ $(PACKAGES) + go $@ $(PACKAGES) $(BINARIES) .PHONY: codecs codecs: protobufs ffjson diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 1ecb3f28..ab4a4ee4 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -53,7 +53,7 @@ func buildControllerConfig(state *internalState, shutdown <-chan struct{}) contr frameworkIDStore = NewInMemoryIDStore() controlContext = &controller.ContextAdapter{ DoneFunc: state.isDone, - FrameworkIDFunc: func() string { return frameworkIDStore.Get() }, + FrameworkIDFunc: frameworkIDStore.Get, ErrorFunc: func(err error) { if err != nil { if err != io.EOF { diff --git a/api/v1/lib/builders.go b/api/v1/lib/builders.go index 01642b31..209f6e26 100644 --- a/api/v1/lib/builders.go +++ b/api/v1/lib/builders.go @@ -7,6 +7,22 @@ type ( RangeBuilder struct{ Ranges } ) +func CPUs(value float64) ResourceBuilder { + return BuildResource().Name("cpus").Scalar(value) +} + +func Memory(value float64) ResourceBuilder { + return BuildResource().Name("mem").Scalar(value) +} + +func Disk(value float64) ResourceBuilder { + return BuildResource().Name("disk").Scalar(value) +} + +func GPUs(value uint) ResourceBuilder { + return BuildResource().Name("gpus").Scalar(float64(value)) +} + func BuildRanges() RangeBuilder { return RangeBuilder{Ranges: Ranges(nil)} } diff --git a/api/v1/lib/offers.go b/api/v1/lib/offers.go new file mode 100644 index 00000000..398ba11c --- /dev/null +++ b/api/v1/lib/offers.go @@ -0,0 +1,13 @@ +package mesos + +// Offers is a convenience type wrapper for a collection of mesos Offer messages +type Offers []Offer + +// IDs extracts the ID field from the given list of offers +func (offers Offers) IDs() []OfferID { + ids := make([]OfferID, len(offers)) + for i := range offers { + ids[i] = offers[i].ID + } + return ids +} diff --git a/api/v1/lib/scheduler/calls/calls.go b/api/v1/lib/scheduler/calls/calls.go index 3a41b8b7..fda2d0a0 100644 --- a/api/v1/lib/scheduler/calls/calls.go +++ b/api/v1/lib/scheduler/calls/calls.go @@ -36,6 +36,14 @@ func RefuseSecondsWithJitter(r *rand.Rand, d time.Duration) scheduler.CallOpt { }) } +// RefuseSeconds returns a calls.Filters option that sets RefuseSeconds to the given duration +func RefuseSeconds(d time.Duration) scheduler.CallOpt { + asFloat := float64(d) + return Filters(func(f *mesos.Filters) { + f.RefuseSeconds = &asFloat + }) +} + // Framework sets a scheduler.Call's FrameworkID func Framework(id string) scheduler.CallOpt { return func(c *scheduler.Call) { diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index ed482801..d0ec2375 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -133,6 +133,13 @@ func DefaultHandler(eh Handler) Option { } } +type AckError struct { + Ack *scheduler.Call + Cause error +} + +func (err *AckError) Error() string { return err.Cause.Error() } + // AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every // UPDATE event that's received. func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { @@ -148,6 +155,9 @@ func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { uuid, ) err = calls.CallNoData(callerGetter(), ack) + if err != nil { + err = &AckError{ack, err} + } } return }) From 5832f888b5ea9dbfcc75dbca3cc63e9ccd1a989c Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 1 May 2017 02:40:20 +0000 Subject: [PATCH 02/67] fix RegistrationTokens bug --- api/v1/lib/extras/scheduler/controller/controller.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index 540c1662..c820c7e5 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -45,8 +45,8 @@ type ( Handler events.Handler // RegistrationTokens (optional) limits the rate at which a framework (re)registers with Mesos. - // The returned chan should either be non-blocking (nil/closed), or should yield a struct{} in - // order to allow the framework registration process to continue. May be nil. + // The chan should either be non-blocking, or should yield a struct{} in order to allow the + // framework registration process to continue. May be nil. RegistrationTokens <-chan struct{} } @@ -78,7 +78,9 @@ func (_ *controllerImpl) Run(config Config) (lastErr error) { if config.Framework.GetFailoverTimeout() > 0 && frameworkID != "" { subscribe.With(calls.SubscribeTo(frameworkID)) } - <-config.RegistrationTokens + if config.RegistrationTokens != nil { + <-config.RegistrationTokens + } resp, err := config.Caller.Call(subscribe) lastErr = processSubscription(config, resp, err) config.Context.Error(lastErr) From 3328e6eb7bc907df37477e793607f216129a056c Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 1 May 2017 02:44:01 +0000 Subject: [PATCH 03/67] example-scheduler: use new CPUs and Memory resource builders --- api/v1/cmd/example-scheduler/app/state.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index fa484271..df696e91 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -87,8 +87,8 @@ func prepareExecutorInfo( func buildWantsTaskResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.taskCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.taskMemory).Resource, + *mesos.CPUs(config.taskCPU).Resource, + *mesos.Memory(config.taskMemory).Resource, ) log.Println("wants-task-resources = " + r.String()) return @@ -96,8 +96,8 @@ func buildWantsTaskResources(config Config) (r mesos.Resources) { func buildWantsExecutorResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.execCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.execMemory).Resource, + *mesos.CPUs(config.execCPU).Resource, + *mesos.Memory(config.execMemory).Resource, ) log.Println("wants-executor-resources = " + r.String()) return From 9798f5a1654a3ed5c1b9305fec9f2d9926d30b78 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 4 May 2017 03:08:02 +0000 Subject: [PATCH 04/67] httpcli: better defaults and cleaner integration with encoding --- api/v1/cmd/example-scheduler/app/state.go | 7 ----- api/v1/lib/encoding/codec.go | 3 ++ api/v1/lib/httpcli/http.go | 36 ++++++++++++++--------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index df696e91..d951ee91 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -117,13 +117,6 @@ func buildHTTPSched(cfg Config, creds credentials) calls.Caller { httpcli.Do(httpcli.With( authConfigOpt, httpcli.Timeout(cfg.timeout), - httpcli.Transport(func(t *http.Transport) { - // all calls should be ack'd by the server within this interval. - // TODO(jdef) it probably doesn't make sense if this value is larger - // than cfg.timeout. - t.ResponseHeaderTimeout = 15 * time.Second - t.MaxIdleConnsPerHost = 2 // don't depend on go's default - }), )), ) if cfg.compression { diff --git a/api/v1/lib/encoding/codec.go b/api/v1/lib/encoding/codec.go index 1d4255b3..557b7aef 100644 --- a/api/v1/lib/encoding/codec.go +++ b/api/v1/lib/encoding/codec.go @@ -50,6 +50,9 @@ type Codec struct { // String implements the fmt.Stringer interface. func (c *Codec) String() string { return c.Name } +func (c *Codec) RequestContentType() string { return c.MediaTypes[0] } +func (c *Codec) ResponseContentType() string { return c.MediaTypes[1] } + type ( // Marshaler composes the supported marshaling formats. Marshaler interface { diff --git a/api/v1/lib/httpcli/http.go b/api/v1/lib/httpcli/http.go index ecc8bc17..0258eb47 100644 --- a/api/v1/lib/httpcli/http.go +++ b/api/v1/lib/httpcli/http.go @@ -25,13 +25,8 @@ type ProtocolError string // Error implements error interface func (pe ProtocolError) Error() string { return string(pe) } -var defaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) - const ( debug = false // TODO(jdef) kill me at some point - - indexRequestContentType = 0 // index into Client.codec.MediaTypes for request content type - indexResponseContentType = 1 // index into Client.codec.MediaTypes for expected response content type ) // DoFunc sends an HTTP request and returns an HTTP response. @@ -77,15 +72,31 @@ type Client struct { handleResponse ResponseHandler } +var ( + DefaultCodec = &encoding.ProtobufCodec + DefaultHeaders = http.Header{} + + // DefaultConfigOpt represents the default client config options. + DefaultConfigOpt = []ConfigOpt{ + Transport(func(t *http.Transport) { + // all calls should be ack'd by the server within this interval. + t.ResponseHeaderTimeout = 15 * time.Second + t.MaxIdleConnsPerHost = 2 // don't depend on go's default + }), + } + + DefaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) +) + // New returns a new Client with the given Opts applied. // Callers are expected to configure the URL, Do, and Codec options prior to // invoking Do. func New(opts ...Opt) *Client { c := &Client{ - codec: &encoding.ProtobufCodec, + codec: DefaultCodec, do: With(DefaultConfigOpt...), - header: http.Header{}, - errorMapper: defaultErrorMapper, + header: DefaultHeaders, + errorMapper: DefaultErrorMapper, } c.buildRequest = c.BuildRequest c.handleResponse = c.HandleResponse @@ -154,8 +165,8 @@ func (c *Client) BuildRequest(m encoding.Marshaler, opt ...RequestOpt) (*http.Re return helper. withOptions(c.requestOpts, opt). withHeaders(c.header). - withHeader("Content-Type", c.codec.MediaTypes[indexRequestContentType]). - withHeader("Accept", c.codec.MediaTypes[indexResponseContentType]). + withHeader("Content-Type", c.codec.RequestContentType()). + withHeader("Accept", c.codec.ResponseContentType()). Request, nil } @@ -183,7 +194,7 @@ func (c *Client) HandleResponse(res *http.Response, err error) (mesos.Response, log.Println("request OK, decoding response") } ct := res.Header.Get("Content-Type") - if ct != c.codec.MediaTypes[indexResponseContentType] { + if ct != c.codec.ResponseContentType() { res.Body.Close() return nil, ProtocolError(fmt.Sprintf("unexpected content type: %q", ct)) } @@ -313,9 +324,6 @@ type Config struct { type ConfigOpt func(*Config) -// DefaultConfigOpt represents the default client config options. -var DefaultConfigOpt []ConfigOpt - // With returns a DoFunc that executes HTTP round-trips. // The default implementation provides reasonable defaults for timeouts: // keep-alive, connection, request/response read/write, and TLS handshake. From b32d741e6a9b6395928b569e06d194dd7ee0e880 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 4 May 2017 03:49:13 +0000 Subject: [PATCH 05/67] latch: new package from refactored test code --- api/v1/lib/extras/latch/latch.go | 33 ++++++++++++++++++++++ api/v1/lib/httpcli/httpsched/state_test.go | 17 ++--------- 2 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 api/v1/lib/extras/latch/latch.go diff --git a/api/v1/lib/extras/latch/latch.go b/api/v1/lib/extras/latch/latch.go new file mode 100644 index 00000000..df7850fa --- /dev/null +++ b/api/v1/lib/extras/latch/latch.go @@ -0,0 +1,33 @@ +package latch + +import "sync" + +// Latch is Closed by default and should be Reset() in order to be useful. +type L struct { + sync.Once + line chan struct{} +} + +func (l *L) Done() <-chan struct{} { return l.line } + +func (l *L) Close() { + l.Do(func() { + defer func() { _ = recover() }() // swallow any panics here + close(l.line) + }) +} + +func (l *L) Closed() (result bool) { + select { + case <-l.line: + result = true + default: + } + return +} + +// Reset clears the state of the latch, not safe to execute concurrently with other L methods. +func (l *L) Reset() *L { + l.line, l.Once = make(chan struct{}), sync.Once{} + return l +} diff --git a/api/v1/lib/httpcli/httpsched/state_test.go b/api/v1/lib/httpcli/httpsched/state_test.go index 9210fe35..92a3e1fe 100644 --- a/api/v1/lib/httpcli/httpsched/state_test.go +++ b/api/v1/lib/httpcli/httpsched/state_test.go @@ -5,29 +5,16 @@ import ( "testing" "github.com/mesos/mesos-go/api/v1/lib/encoding" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) -type latch struct{ line chan struct{} } - -func newLatch() *latch { return &latch{make(chan struct{})} } -func (l *latch) Reset() { l.line = make(chan struct{}) } -func (l *latch) Close() { close(l.line) } -func (l *latch) Closed() (result bool) { - select { - case <-l.line: - result = true - default: - } - return -} - func TestDisconnectionDecoder(t *testing.T) { // invoke disconnect upon decoder errors expected := errors.New("unmarshaler error") decoder := encoding.DecoderFunc(func(_ encoding.Unmarshaler) error { return expected }) - latch := newLatch() + latch := new(latch.L).Reset() d := disconnectionDecoder(decoder, latch.Close) err := d.Decode(nil) From 327017249b97af2bee0108a71453a29c5ef84a4a Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 4 May 2017 04:01:01 +0000 Subject: [PATCH 06/67] store: new package refactored from example-scheduler --- api/v1/cmd/example-scheduler/app/app.go | 5 ++- api/v1/cmd/example-scheduler/app/store.go | 43 -------------------- api/v1/lib/extras/store/singleton.go | 48 +++++++++++++++++++++++ 3 files changed, 51 insertions(+), 45 deletions(-) delete mode 100644 api/v1/cmd/example-scheduler/app/store.go create mode 100644 api/v1/lib/extras/store/singleton.go diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index ab4a4ee4..7fa79999 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -12,6 +12,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/backoff" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" @@ -50,7 +51,7 @@ func Run(cfg Config) error { func buildControllerConfig(state *internalState, shutdown <-chan struct{}) controller.Config { var ( - frameworkIDStore = NewInMemoryIDStore() + frameworkIDStore = store.NewInMemorySingleton() controlContext = &controller.ContextAdapter{ DoneFunc: state.isDone, FrameworkIDFunc: frameworkIDStore.Get, @@ -90,7 +91,7 @@ func buildControllerConfig(state *internalState, shutdown <-chan struct{}) contr } // buildEventHandler generates and returns a handler to process events received from the subscription. -func buildEventHandler(state *internalState, frameworkIDStore IDStore) events.Handler { +func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { // TODO(jdef) would be nice to merge this ack handler with the status update handler below; need to // figure out appropriate error propagation among chained handlers. ack := events.AcknowledgeUpdates(func() calls.Caller { return state.cli }) diff --git a/api/v1/cmd/example-scheduler/app/store.go b/api/v1/cmd/example-scheduler/app/store.go deleted file mode 100644 index bc7fa7a2..00000000 --- a/api/v1/cmd/example-scheduler/app/store.go +++ /dev/null @@ -1,43 +0,0 @@ -package app - -import "sync/atomic" - -// IDStore is a thread-safe abstraction to load and store a stringified ID. -type IDStore interface { - Get() string - Set(string) -} - -type IDStoreAdapter struct { - GetFunc func() string - SetFunc func(string) -} - -func (a IDStoreAdapter) Get() string { - if a.GetFunc != nil { - return a.GetFunc() - } - return "" -} - -func (a IDStoreAdapter) Set(s string) { - if a.SetFunc != nil { - a.SetFunc(s) - } -} - -func NewInMemoryIDStore() IDStore { - var frameworkID atomic.Value - return &IDStoreAdapter{ - GetFunc: func() string { - x := frameworkID.Load() - if x == nil { - return "" - } - return x.(string) - }, - SetFunc: func(s string) { - frameworkID.Store(s) - }, - } -} diff --git a/api/v1/lib/extras/store/singleton.go b/api/v1/lib/extras/store/singleton.go new file mode 100644 index 00000000..540ad857 --- /dev/null +++ b/api/v1/lib/extras/store/singleton.go @@ -0,0 +1,48 @@ +package store + +import "sync/atomic" + +type ( + Getter interface { + Get() string + } + + GetFunc func() string + + Setter interface { + Set(string) error + } + + SetFunc func(string) error + + // Singleton is a thread-safe abstraction to load and store a string + Singleton interface { + Getter + Setter + } + + SingletonAdapter struct { + GetFunc + SetFunc + } +) + +func (f GetFunc) Get() string { return f() } +func (f SetFunc) Set(s string) error { return f(s) } + +func NewInMemorySingleton() Singleton { + var value atomic.Value + return &SingletonAdapter{ + func() string { + x := value.Load() + if x == nil { + return "" + } + return x.(string) + }, + func(s string) error { + value.Store(s) + return nil + }, + } +} From 3a7148e0c69faa50c12d3220f8f142aae88dbfd8 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 4 May 2017 04:08:32 +0000 Subject: [PATCH 07/67] offers: expand API and move into extras --- api/v1/lib/extras/offers/filters.go | 61 ++++++++ api/v1/lib/extras/offers/offers.go | 224 ++++++++++++++++++++++++++++ api/v1/lib/offers.go | 13 -- 3 files changed, 285 insertions(+), 13 deletions(-) create mode 100644 api/v1/lib/extras/offers/filters.go create mode 100644 api/v1/lib/extras/offers/offers.go delete mode 100644 api/v1/lib/offers.go diff --git a/api/v1/lib/extras/offers/filters.go b/api/v1/lib/extras/offers/filters.go new file mode 100644 index 00000000..080a23c3 --- /dev/null +++ b/api/v1/lib/extras/offers/filters.go @@ -0,0 +1,61 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Filter accepts or rejects a mesos Offer + Filter interface { + Accept(*Offer) bool + } + + // FilterFunc returns true if the given Offer passes the filter + FilterFunc func(*Offer) bool +) + +// Accept implements Filter for FilterFunc +func (f FilterFunc) Accept(o *Offer) bool { return f(o) } + +func nilFilter(_ *Offer) bool { return true } + +func not(f Filter) Filter { + return FilterFunc(func(offer *Offer) bool { return !f.Accept(offer) }) +} + +// ByHostname returns a Filter that accepts offers with a matching Hostname +func ByHostname(hostname string) Filter { + if hostname == "" { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return o.Hostname == hostname + }) +} + +// ByAttributes returns a Filter that accepts offers with an attribute set accepted by +// the provided Attribute filter func. +func ByAttributes(f func(attr []Attribute) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Attributes) + }) +} + +func ByExecutors(f func(exec []ExecutorID) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.ExecutorIDs) + }) +} + +func ByUnavailability(f func(u *Unavailability) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Unavailability) + }) +} diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/offers/offers.go new file mode 100644 index 00000000..a8615af3 --- /dev/null +++ b/api/v1/lib/extras/offers/offers.go @@ -0,0 +1,224 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Slice is a convenience type wrapper for a slice of mesos Offer messages + Slice []Offer + + // Index is a convenience type wrapper for a dictionary of Offer messages + Index map[interface{}]*Offer + + // KeyFunc generates a key used for indexing offers + KeyFunc func(*Offer) interface{} +) + +// IDs extracts the ID field from a Slice of offers +func (offers Slice) IDs() []OfferID { + ids := make([]OfferID, len(offers)) + for i := range offers { + ids[i] = offers[i].ID + } + return ids +} + +// IDs extracts the ID field from a Index of offers +func (offers Index) IDs() []OfferID { + ids := make([]OfferID, 0, len(offers)) + for _, offer := range offers { + ids = append(ids, offer.GetID()) + } + return ids +} + +// Find returns the first Offer that passes the given filter function. +func (offers Slice) Find(filter Filter) *Offer { + for i := range offers { + offer := &offers[i] + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Find returns the first Offer that passes the given filter function. +func (offers Index) Find(filter Filter) *Offer { + for _, offer := range offers { + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Filter returns the subset of the Slice that matches the given filter. +func (offers Slice) Filter(filter Filter) (result Slice) { + if sz := len(result); sz > 0 { + result = make(Slice, 0, sz) + for i := range offers { + if filter.Accept(&offers[i]) { + result = append(result, offers[i]) + } + } + } + return +} + +// Filter returns the subset of the Index that matches the given filter. +func (offers Index) Filter(filter Filter) (result Index) { + if sz := len(result); sz > 0 { + result = make(Index, sz) + for id, offer := range offers { + if filter.Accept(offer) { + result[id] = offer + } + } + } + return +} + +// FilterNot returns the subset of the Slice that does not match the given filter. +func (offers Slice) FilterNot(filter Filter) Slice { return offers.Filter(not(filter)) } + +// FilterNot returns the subset of the Index that does not match the given filter. +func (offers Index) FilterNot(filter Filter) Index { return offers.Filter(not(filter)) } + +// ContainsResources returns a filter function that returns true if the Resources of an Offer +// contain the wanted Resources. +func ContainsResources(wanted Resources) Filter { + return FilterFunc(func(o *Offer) bool { + return Resources(o.Resources).Flatten().ContainsAll(wanted) + }) +} + +// DefaultKeyFunc indexes offers by their OfferID. +var DefaultKeyFunc = KeyFunc(func(o *Offer) interface{} { return o.GetID() }) + +// NewIndex returns a new Index constructed from the list of mesos offers. +// If the KeyFunc is nil then offers are indexed by DefaultKeyFunc. +// The values of the returned Index are pointers to (not copies of) the offers of the slice receiver. +func NewIndex(slice []Offer, kf KeyFunc) Index { + if slice == nil { + return nil + } + if kf == nil { + kf = DefaultKeyFunc + } + index := make(Index, len(slice)) + for i := range slice { + offer := &slice[i] + index[kf(offer)] = offer + } + return index +} + +// ToSlice returns a Slice from the offers in the Index. +// The returned slice will contain shallow copies of the offers from the Index. +func (s Index) ToSlice() (slice Slice) { + if sz := len(s); sz > 0 { + slice = make(Slice, 0, sz) + for _, offer := range s { + slice = append(slice, *offer) + } + } + return +} + +type ( + Reducer interface { + Reduce(_, _ *Offer) *Offer + } + + ReduceFunc func(_, _ *Offer) *Offer +) + +func (f ReduceFunc) Reduce(a, b *Offer) *Offer { return f(a, b) } + +var _ = Reducer(ReduceFunc(func(_, _ *Offer) *Offer { return nil })) // sanity check + +func (slice Slice) Reduce(def Offer, r Reducer) (result Offer) { + result = def + if r != nil { + acc := &result + for i := range slice { + acc = r.Reduce(&result, &slice[i]) + } + if acc == nil { + result = Offer{} + } else { + result = *acc + } + } + return +} + +func (index Index) Reduce(def *Offer, r Reducer) (result *Offer) { + result = def + if r != nil { + for i := range index { + result = r.Reduce(result, index[i]) + } + } + return +} + +func (slice Slice) GroupBy(kf KeyFunc) map[interface{}]Slice { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(slice) == 0 { + return nil + } + result := make(map[interface{}]Slice) + for i := range slice { + groupKey := kf(&slice[i]) + result[groupKey] = append(result[groupKey], slice[i]) + } + return result +} + +func (index Index) GroupBy(kf KeyFunc) map[interface{}]Slice { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(index) == 0 { + return nil + } + result := make(map[interface{}]Slice) + for _, offer := range index { + groupKey := kf(offer) + result[groupKey] = append(result[groupKey], *offer) + } + return result +} + +func (index Index) Partition(f Filter) (accepted, rejected Index) { + if f == nil { + return index, nil + } + if len(index) > 0 { + accepted, rejected = make(Index), make(Index) + for id, offer := range index { + if f.Accept(offer) { + accepted[id] = offer + } else { + rejected[id] = offer + } + } + } + return +} + +func (index Index) Reindex(kf KeyFunc) Index { + sz := len(index) + if kf == nil || sz == 0 { + return index + } + result := make(Index, sz) + for _, offer := range index { + key := kf(offer) + result[key] = offer + } + return result +} diff --git a/api/v1/lib/offers.go b/api/v1/lib/offers.go deleted file mode 100644 index 398ba11c..00000000 --- a/api/v1/lib/offers.go +++ /dev/null @@ -1,13 +0,0 @@ -package mesos - -// Offers is a convenience type wrapper for a collection of mesos Offer messages -type Offers []Offer - -// IDs extracts the ID field from the given list of offers -func (offers Offers) IDs() []OfferID { - ids := make([]OfferID, len(offers)) - for i := range offers { - ids[i] = offers[i].ID - } - return ids -} From f4e99be126463d4e8e80952ba8745bbbc9001960 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 4 May 2017 04:27:03 +0000 Subject: [PATCH 08/67] example: minimal scheduler implementation --- api/v1/cmd/msh/msh.go | 222 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 api/v1/cmd/msh/msh.go diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go new file mode 100644 index 00000000..82ab62e7 --- /dev/null +++ b/api/v1/cmd/msh/msh.go @@ -0,0 +1,222 @@ +// msh is a minimal mesos v1 scheduler; it executes a shell command on a mesos agent. +package main + +// Usage: msh {...command line args...} +// +// For example: +// msh -master 10.2.0.5:5050 -- ls -laF /tmp +// +// TODO: -gpu=1 to enable GPU_RESOURCES caps and request 1 gpu +// + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" + "github.com/mesos/mesos-go/api/v1/lib/extras/offers" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/httpcli" + "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +const ( + RFC3339a = "20060102T150405Z0700" +) + +var ( + FrameworkName = "msh" + TaskName = "msh" + MesosMaster = "127.0.0.1:5050" + User = "root" + Role = mesos.RoleName("*") + CPUs = float64(0.010) + Memory = float64(64) + + frameworkIDStore store.Singleton + shouldDecline bool + refuseSeconds = calls.RefuseSeconds(8 * time.Hour) + stop func() + exitCode int + wantsResources mesos.Resources + taskPrototype mesos.TaskInfo +) + +func init() { + flag.StringVar(&FrameworkName, "framework_name", FrameworkName, "Name of the framework") + flag.StringVar(&TaskName, "task_name", TaskName, "Name of the msh task") + flag.StringVar(&MesosMaster, "master", MesosMaster, "IP:port of the mesos master") + flag.StringVar(&User, "user", User, "OS user that owns the launched task") + flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command") + flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command") + + frameworkIDStore = store.NewInMemorySingleton() +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) < 1 { // msh by itself prints usage + flag.Usage() + os.Exit(1) + } + + wantsResources = mesos.Resources{ + *mesos.CPUs(CPUs).Resource, + *mesos.Memory(Memory).Resource, + } + taskPrototype = mesos.TaskInfo{ + Name: TaskName, + Command: &mesos.CommandInfo{ + Value: proto.String(args[0]), + Shell: proto.Bool(false), + }, + } + if len(args) > 1 { + taskPrototype.Command.Arguments = args[1:] + } + err := controller.New().Run(buildControllerConfig(User)) + if err != nil { + log.Fatal(err) + } + os.Exit(exitCode) +} + +func buildControllerConfig(user string) controller.Config { + var ( + done = new(latch.L).Reset() + caller = calls.Decorators{ + calls.SubscribedCaller(frameworkIDStore.Get), + }.Apply(buildClient()) + ) + stop = done.Close + return controller.Config{ + Context: &controller.ContextAdapter{ + DoneFunc: done.Closed, + FrameworkIDFunc: frameworkIDStore.Get, + ErrorFunc: func(err error) { + defer stop() + if err != nil { + // don't overwrite an existing error code + if exitCode == 0 { + exitCode = 10 + } + if err != io.EOF { + log.Printf("%#v", err) + } + return + } + log.Println("disconnected") + }, + }, + Framework: &mesos.FrameworkInfo{User: user, Name: FrameworkName, Role: (*string)(&Role)}, + Caller: caller, + Handler: buildEventHandler(caller), + } +} + +func buildClient() calls.Caller { + return httpsched.NewCaller(httpcli.New( + httpcli.Endpoint(fmt.Sprintf("http://%s/api/v1/scheduler", MesosMaster)), + )) +} + +func buildEventHandler(caller calls.Caller) events.Handler { + ack := events.AcknowledgeUpdates(func() calls.Caller { return caller }) + return events.NewMux( + events.DefaultHandler(events.HandlerFunc(controller.DefaultHandler)), + events.MapFuncs(map[scheduler.Event_Type]events.HandlerFunc{ + scheduler.Event_OFFERS: func(e *scheduler.Event) error { + return resourceOffers(caller, e.GetOffers().GetOffers()) + }, + scheduler.Event_UPDATE: func(e *scheduler.Event) error { + err := ack.HandleEvent(e) + if err != nil { + err = fmt.Errorf("failed to ack status update for task: %#v", err) + } + statusUpdate(e.GetUpdate().GetStatus()) + return err + }, + scheduler.Event_SUBSCRIBED: func(e *scheduler.Event) error { + log.Println("received a SUBSCRIBED event") + fid := e.GetSubscribed().GetFrameworkID().GetValue() + if fid == "" { + // sanity check, should **never** happen + return fmt.Errorf("mesos gave us an empty frameworkID") + } + if current := frameworkIDStore.Get(); current != fid { + err := frameworkIDStore.Set(fid) + if err != nil { + return err + } + log.Println("FrameworkID", fid) + } + return nil + }, + }), + ) +} + +func resourceOffers(caller calls.Caller, off []mesos.Offer) error { + if shouldDecline { + return calls.CallNoData(caller, calls.Suppress()) + } + var ( + index = offers.NewIndex(off, nil) + match = index.Find(offers.ContainsResources(wantsResources)) + ) + if match != nil { + task := taskPrototype + task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)} + task.AgentID = match.AgentID + task.Resources = mesos.Resources(match.Resources).Find(wantsResources.Flatten(Role.Assign())) + + if err := calls.CallNoData(caller, calls.Accept( + calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), + )); err != nil { + return err + } + + shouldDecline = true // safeguard and suppress future offers + if err := calls.CallNoData(caller, calls.Suppress()); err != nil { + return err + } + } else { + // insufficient offers + log.Println("rejected insufficient offers") + } + // decline all but the possible match + delete(index, match.GetID()) + return calls.CallNoData(caller, calls.Decline(index.IDs()...).With(refuseSeconds)) +} + +func statusUpdate(s mesos.TaskStatus) { + switch st := s.GetState(); st { + case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING: + log.Println("status update", st) + if st != mesos.TASK_FINISHED { + return + } + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + log.Println("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") + exitCode = 3 + default: + log.Println("unexpected task state, aborting", st) + exitCode = 4 + } + stop() +} From 23d5fae4f826cd71a30da087b636af4915915763 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 5 May 2017 02:24:22 +0000 Subject: [PATCH 09/67] vendor: whole tree for gogo/protobuf in order to be able to rebuild protobufs --- api/v1/vendor/vendor.json | 703 +------------------------------------- 1 file changed, 4 insertions(+), 699 deletions(-) diff --git a/api/v1/vendor/vendor.json b/api/v1/vendor/vendor.json index b32651e1..9e8df969 100644 --- a/api/v1/vendor/vendor.json +++ b/api/v1/vendor/vendor.json @@ -3,706 +3,11 @@ "ignore": "test", "package": [ { - "checksumSHA1": "RIVAETtE3FvCtjbSOh0REf6JgQk=", - "path": "github.com/gogo/protobuf/codec", + "checksumSHA1": "2I4udA/dza74M1bOgQuEPz103gw=", + "path": "github.com/gogo/protobuf", "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "tEjFocOVKUr8IK6jkvegQlfqjOw=", - "path": "github.com/gogo/protobuf/gogoproto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CLfzGRKKVJkecd9WCEtnKI2NPtI=", - "path": "github.com/gogo/protobuf/io", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Ol9P36yJqSe2nMDjKyYmz7i3TRs=", - "path": "github.com/gogo/protobuf/jsonpb", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "paUMgsbzE5tXgsiZ+RTRcEI68I4=", - "path": "github.com/gogo/protobuf/jsonpb/jsonpb_test_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cTbYeqRl+/qhD1JZvZmT5OAjaXA=", - "path": "github.com/gogo/protobuf/plugin/defaultcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Pjc29+wvgMEWJkw9PqVwTWd5R/c=", - "path": "github.com/gogo/protobuf/plugin/description", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NP3xQexLANLm++9RCktWr1ovOY0=", - "path": "github.com/gogo/protobuf/plugin/embedcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yzLXQWgFsBkFkEHScmPkcwdN2iY=", - "path": "github.com/gogo/protobuf/plugin/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Tx1FOos8BwPTDNp1LhAoG8Co6ew=", - "path": "github.com/gogo/protobuf/plugin/equal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xS3aAjAbrgPgnNkTUlTOuAmxXQ8=", - "path": "github.com/gogo/protobuf/plugin/face", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ugvdtinMoMF6MdKu4lN+D+xb0gM=", - "path": "github.com/gogo/protobuf/plugin/gostring", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7VOG8kB8XBzoNkErOWyR3QdpF5I=", - "path": "github.com/gogo/protobuf/plugin/grpc", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "GK139O7VRP+DHzVOG7PIBrzEpYc=", - "path": "github.com/gogo/protobuf/plugin/marshalto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/dztWeNfEY/M2fBB2hjO4vC2idE=", - "path": "github.com/gogo/protobuf/plugin/oneofcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//9AgNP7EaQ42IGI5gp5GBlqq+8=", - "path": "github.com/gogo/protobuf/plugin/populate", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3doyBfoqf4eU3Rkn4nl6MK7I5QM=", - "path": "github.com/gogo/protobuf/plugin/size", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "6C5JEnHpRtsBQ+nhNnDuZtw35Gk=", - "path": "github.com/gogo/protobuf/plugin/stringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5/vombKpWe8yH5q2RH3wZPId0no=", - "path": "github.com/gogo/protobuf/plugin/testgen", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5fuoaG6BK9wOKAyp+AdIowdeIx8=", - "path": "github.com/gogo/protobuf/plugin/union", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zDRKoIeCInauKHmQkTuixhhAjmA=", - "path": "github.com/gogo/protobuf/plugin/unmarshal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hyS3vR96CWcq5guGQXKUWVrlvGA=", - "path": "github.com/gogo/protobuf/proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "WYKtRaKlF90mjEVLg8QuIFEuKH0=", - "path": "github.com/gogo/protobuf/proto/proto3_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "buFKlCd6bVy4ynsgaFZBC/NhhJM=", - "path": "github.com/gogo/protobuf/proto/testdata", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "VkdsPnt/LryLUDkSNuVVIqwq4Fc=", - "path": "github.com/gogo/protobuf/protoc-gen-combo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lYD+Czpx21KQfnMj/VJbD2ll1nI=", - "path": "github.com/gogo/protobuf/protoc-gen-gofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RNSh5A+eMLvpb36czti5+wfm4wA=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "u0+Vvbf2f8FWldJOohemISYPLYo=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/descriptor", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NQbUjvqwqPFFVa8H4ntwuViQtLE=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/generator", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DmPXpIbl/qpcje0w4JAJW4QjEDM=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/plugin", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "X0kogVRUqIRdipIVRbcTwZmODac=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "n6qb61BdGIyL1Sr6+k2L9WvbaQY=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofaster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "igZu+7E154ueL7+91KAoUH3Kb4c=", - "path": "github.com/gogo/protobuf/protoc-gen-gogoslick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "J1bPjNLs1CCZv3pnl2QMYiD9f7Q=", - "path": "github.com/gogo/protobuf/protoc-min-version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6cMbpJj41MpihS5eP4SIliRBK4=", - "path": "github.com/gogo/protobuf/sortkeys", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lbHS2lM7RHQE3xseWie466dKY2I=", - "path": "github.com/gogo/protobuf/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "SxY4AAxSzboRLBc38Yyn1s4okI0=", - "path": "github.com/gogo/protobuf/test/casttype", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CfGm603zYf9bqWAC0Z3eBdrzuGg=", - "path": "github.com/gogo/protobuf/test/casttype/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "FmoK2j/5ncq8T+eHCMPxSeqnzFY=", - "path": "github.com/gogo/protobuf/test/casttype/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "p4kmpuTD71Z3dvhbICrF2xbGosU=", - "path": "github.com/gogo/protobuf/test/casttype/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0c98JonLeFNSc3qdVyoVcfCriN4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mTi59HJocNgdkNGkYRLz2pn8+WE=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "TYBTBsMDW3Fsfs3s0E8CRNcxEu4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sIpq5aF4bvnqOa/4YK2gWQNcOoc=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4XzzMi5+/zaIuy2EWEe0ufCuQtI=", - "path": "github.com/gogo/protobuf/test/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "aq5aEfH8+HPgqKAo3U9EpMjZZUo=", - "path": "github.com/gogo/protobuf/test/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "OCUBqnfbWfraPTB8oW+8/cFiXKw=", - "path": "github.com/gogo/protobuf/test/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RqtUd2ZnGrwwf13KjYQwOK16F3M=", - "path": "github.com/gogo/protobuf/test/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "1d0nW4JvpT3ALIkT5f0y8T5M+xk=", - "path": "github.com/gogo/protobuf/test/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "PCBUEHiaKfjZZfo77m4W32qOeB4=", - "path": "github.com/gogo/protobuf/test/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Q6fiuAzwlM4kVK3PMF9vWnvmays=", - "path": "github.com/gogo/protobuf/test/custom", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2qwk23WTep6Z6DAq2hCMf9OsT/I=", - "path": "github.com/gogo/protobuf/test/custom-dash-type", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "h0LHbcpzexyjfSPu/LbrRM1Zg8c=", - "path": "github.com/gogo/protobuf/test/custombytesnonstruct", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "E2/sHgeCMMpdjoEBb4SY7G8p4Uk=", - "path": "github.com/gogo/protobuf/test/dashfilename", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9yHuzxDG9ZjtDFoJLEomVUFIyz0=", - "path": "github.com/gogo/protobuf/test/defaultconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "S+k0F1XhrDFd+igafb5FLceyUr0=", - "path": "github.com/gogo/protobuf/test/embedconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "75vwg1za0UP2O3YLwPR8HD14SOU=", - "path": "github.com/gogo/protobuf/test/empty-issue70", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ncjf6Xzc4vOfR9YidS6Cc9QTLtw=", - "path": "github.com/gogo/protobuf/test/enumprefix", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fsQ9GD7yVIWT2VC2qI6wHYIdISM=", - "path": "github.com/gogo/protobuf/test/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "UIBxh/5EDEL5DIvVoH4ISmHBDys=", - "path": "github.com/gogo/protobuf/test/example", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YiSMdmPIdyFvXvT/tlc86DjwJLI=", - "path": "github.com/gogo/protobuf/test/fuzztests", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yp7uRKY5gyfh6vSXnVjXIzGAXE4=", - "path": "github.com/gogo/protobuf/test/group", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DeBkH1l+Gub2I4LgrWrnWbLVjpE=", - "path": "github.com/gogo/protobuf/test/importdedup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AVJIC2MdvEh9UfbN40073Z/d0f8=", - "path": "github.com/gogo/protobuf/test/importdedup/subpkg", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0AWSyTxYd97FyMn7gZ95Ej/3m4s=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/t6kWiQ82zTH5KSMafg/uQ16ask=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72/index", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9NEWM8d2HXtilq+/QR+rLmJhOkI=", - "path": "github.com/gogo/protobuf/test/issue34", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "EUSuYKzjDRdtYYSF7r30Xs7FgDg=", - "path": "github.com/gogo/protobuf/test/issue42order", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7fjvhomoxX3rt5gJZZBWp5sKkhs=", - "path": "github.com/gogo/protobuf/test/issue8", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4ZPo2LIaVv3lJZEQYwPLhlvBktc=", - "path": "github.com/gogo/protobuf/test/mapsproto2", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cNiwjpj2RmyJNjiqvJoFQUPsRqM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "vR6wpg/XpCULa9KTONFHXrFLmIQ=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mqF9ePL2nMR0iyJWBwpZl7VSjqA=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "psHlD68ijJjBBHqN/MgG4MGkC3c=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hCfdSModEttRBnDgDBy2Zx0e/uc=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cIszinXlkReEzP4DLVqhFr80loI=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AqEcvNXhMBPX9y4C1JdU5ci6djM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//KmhDh/C/fHRA3oGNMSGYALpuU=", - "path": "github.com/gogo/protobuf/test/mixbench", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0fgRfF8zijkFLXG6OcmgfXv+070=", - "path": "github.com/gogo/protobuf/test/moredefaults", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ahMVz714gNBvYBmO2I+T2EJIeIc=", - "path": "github.com/gogo/protobuf/test/oneof", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/vpBUlSaib5le0urbk9j8ySiRD0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4TGArepiMDOGxLQUnyIFWkCV860=", - "path": "github.com/gogo/protobuf/test/oneof/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sbzC8eZJZ5VWaGwzRxSTPIPQPME=", - "path": "github.com/gogo/protobuf/test/oneof/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YgoWSjBZxO1FAkc1tXUndt4IXS0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "O8WR3D8SHq0gFxa7+rf0MdelAoA=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ebUJvjKLbg5R9F7nTUuNT5s661w=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "HWM1axY96SEoc1bY2edTjlld9c4=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YBnrab1VG14LcgYyVLM2uFswhgc=", - "path": "github.com/gogo/protobuf/test/oneof3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2MOPs9vr5XgDUTtjm8E67tJnXyo=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xfBBM0eq7rW41faUfZh75z+Xj3I=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "T7NdX8Dsv3mfYDXic/YVo1urNSQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zb2LHnn4ql6QQwgVpE1JkZIxcHE=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6bMJP3iB2kyb8e6p3GdoRCBtCk=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "QdexN5eBoxSSga9kn8uw3la1jJA=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fDFnUhPE51bxH9yhMjWpEPu/IQQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "XVtCxPLxfWUo33u2xSRP1hzgd/0=", - "path": "github.com/gogo/protobuf/test/packed", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ZYwqz+QwhSsp/1YKpn1ixA4ahCE=", - "path": "github.com/gogo/protobuf/test/required", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "bCqTYwvGJl6k/7nSxSmMjtIs4VE=", - "path": "github.com/gogo/protobuf/test/sizeunderscore", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9ZhQFwpC3uupLOELXbdLJ2Ss040=", - "path": "github.com/gogo/protobuf/test/tags", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eM0kVXUbz6sinSfxARQe/cTk15k=", - "path": "github.com/gogo/protobuf/test/theproto3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eBxOWTGmgxhWG1U1wcSjuOFl4sU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "BhPGpjdC9375YDejUFVlsfzolXg=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "MdGf5p9TOSeoMj0vGhFHqktACNk=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oSeY/ryVt9ZenUM4xC4ocDGB6Oc=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ivhKH76IrT1EHyWzImjedlTPdCo=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oXfCUeKjTKLTqjmJlDUaz0HZesU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2QEqlqrIIY9Y1On/qVoMu6ABj80=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "IG6H7z0wDBAqcsF7sojaOWTWNI4=", - "path": "github.com/gogo/protobuf/test/unmarshalmerge", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Y5w3d7x5quR19o+iT/HhphY89/I=", - "path": "github.com/gogo/protobuf/test/unrecognized", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "8CjEqnI9sTWfSeGX98CNrrilguw=", - "path": "github.com/gogo/protobuf/test/unrecognizedgroup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3KeNOHQHQb4ZjB3XpigiYUDqLWM=", - "path": "github.com/gogo/protobuf/vanity", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6dohUIo6Qf0ixPXzxH7LExfrDk=", - "path": "github.com/gogo/protobuf/vanity/command", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0/Iq33SFI/UbS8LJeTyOadMxgPE=", - "path": "github.com/gogo/protobuf/vanity/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "jEwyZC+hiPl2avUlrgEDhtCEiT4=", - "path": "github.com/gogo/protobuf/vanity/test/fast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "F3rR7HTm2Wi4jlPedckmvNT7TI8=", - "path": "github.com/gogo/protobuf/vanity/test/faster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "C9/L7Jpst+Unink8JiZuXLHJgv4=", - "path": "github.com/gogo/protobuf/vanity/test/slick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "KLFcaMb5M2MQ7Sl3gLXl70wgKIg=", - "path": "github.com/gogo/protobuf/version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" + "revisionTime": "2015-09-16T11:01:26Z", + "tree": true }, { "checksumSHA1": "ANHyMfOc1XnqDnlxNipBdpFE1qw=", From 78ceea4e8ffd502d2b79c9b6b66748290d280f14 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 5 May 2017 20:15:11 +0000 Subject: [PATCH 10/67] fix: correctly convert to Seconds --- api/v1/lib/scheduler/calls/calls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1/lib/scheduler/calls/calls.go b/api/v1/lib/scheduler/calls/calls.go index fda2d0a0..42fdbfc0 100644 --- a/api/v1/lib/scheduler/calls/calls.go +++ b/api/v1/lib/scheduler/calls/calls.go @@ -38,7 +38,7 @@ func RefuseSecondsWithJitter(r *rand.Rand, d time.Duration) scheduler.CallOpt { // RefuseSeconds returns a calls.Filters option that sets RefuseSeconds to the given duration func RefuseSeconds(d time.Duration) scheduler.CallOpt { - asFloat := float64(d) + asFloat := d.Seconds() return Filters(func(f *mesos.Filters) { f.RefuseSeconds = &asFloat }) From 6242813f67c05cf323c9790effa011aaceb9aef0 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 5 May 2017 20:19:25 +0000 Subject: [PATCH 11/67] latch: API cleanup; examples: use new latch API; msh: better default for refuseSeconds --- api/v1/cmd/example-scheduler/app/app.go | 8 ++++---- api/v1/cmd/example-scheduler/app/state.go | 22 +++------------------- api/v1/cmd/msh/msh.go | 4 ++-- api/v1/lib/extras/latch/latch.go | 10 ++++++++++ 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 7fa79999..028fc124 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -53,7 +53,7 @@ func buildControllerConfig(state *internalState, shutdown <-chan struct{}) contr var ( frameworkIDStore = store.NewInMemorySingleton() controlContext = &controller.ContextAdapter{ - DoneFunc: state.isDone, + DoneFunc: state.done.Closed, FrameworkIDFunc: frameworkIDStore.Get, ErrorFunc: func(err error) { if err != nil { @@ -61,7 +61,7 @@ func buildControllerConfig(state *internalState, shutdown <-chan struct{}) contr log.Println(err) } if _, ok := err.(StateError); ok { - state.markDone() + state.done.Close() } return } @@ -257,7 +257,7 @@ func statusUpdate(state *internalState, s mesos.TaskStatus) { if state.tasksFinished == state.totalTasks { log.Println("mission accomplished, terminating") - state.markDone() + state.done.Close() } else { tryReviveOffers(state) } @@ -268,7 +268,7 @@ func statusUpdate(state *internalState, s mesos.TaskStatus) { " with reason " + s.GetReason().String() + " from source " + s.GetSource().String() + " with message '" + s.GetMessage() + "'") - state.markDone() + state.done.Close() } } diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index d951ee91..c4dd89e1 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -8,12 +8,12 @@ import ( "math/rand" "net/http" "os" - "sync" "time" proto "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" @@ -209,26 +209,11 @@ func newInternalState(cfg Config) (*internalState, error) { metricsAPI: metricsAPI, cli: buildHTTPSched(cfg, creds), random: rand.New(rand.NewSource(time.Now().Unix())), - done: make(chan struct{}), + done: latch.New(), } return state, nil } -func (state *internalState) markDone() { - state.doneOnce.Do(func() { - close(state.done) - }) -} - -func (state *internalState) isDone() bool { - select { - case <-state.done: - return true - default: - return false - } -} - type internalState struct { tasksLaunched int tasksFinished int @@ -242,7 +227,6 @@ type internalState struct { reviveTokens <-chan struct{} metricsAPI *metricsAPI err error - done chan struct{} - doneOnce sync.Once + done latch.Interface random *rand.Rand } diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 82ab62e7..a8386413 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -45,7 +45,7 @@ var ( frameworkIDStore store.Singleton shouldDecline bool - refuseSeconds = calls.RefuseSeconds(8 * time.Hour) + refuseSeconds = calls.RefuseSeconds(5 * time.Second) stop func() exitCode int wantsResources mesos.Resources @@ -94,7 +94,7 @@ func main() { func buildControllerConfig(user string) controller.Config { var ( - done = new(latch.L).Reset() + done = latch.New() caller = calls.Decorators{ calls.SubscribedCaller(frameworkIDStore.Get), }.Apply(buildClient()) diff --git a/api/v1/lib/extras/latch/latch.go b/api/v1/lib/extras/latch/latch.go index df7850fa..0cb395f3 100644 --- a/api/v1/lib/extras/latch/latch.go +++ b/api/v1/lib/extras/latch/latch.go @@ -2,12 +2,22 @@ package latch import "sync" +type Interface interface { + Done() <-chan struct{} + Close() + Closed() bool +} + // Latch is Closed by default and should be Reset() in order to be useful. type L struct { sync.Once line chan struct{} } +func New() Interface { + return new(L).Reset() +} + func (l *L) Done() <-chan struct{} { return l.line } func (l *L) Close() { From d861e34539a2fce99204765a4f8d8aebf28319fe Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 5 May 2017 20:36:43 +0000 Subject: [PATCH 12/67] latch: cleanup go-vet errors, document API --- api/v1/lib/extras/latch/latch.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/api/v1/lib/extras/latch/latch.go b/api/v1/lib/extras/latch/latch.go index 0cb395f3..35f4d80b 100644 --- a/api/v1/lib/extras/latch/latch.go +++ b/api/v1/lib/extras/latch/latch.go @@ -1,30 +1,35 @@ package latch -import "sync" +import "sync/atomic" +// Interface funcs are safe to invoke concurrently. type Interface interface { + // Done returns a chan that blocks until Close is called. It never returns data. Done() <-chan struct{} + // Close closes the latch; all future calls to Closed return true. Safe to invoke multiple times. Close() + // Closed returns false while the latch is "open" and true after it has been closed via Close. Closed() bool } -// Latch is Closed by default and should be Reset() in order to be useful. type L struct { - sync.Once - line chan struct{} + value int32 + line chan struct{} } +// New returns a new "open" latch such that Closed returns false until Close is invoked. func New() Interface { return new(L).Reset() } func (l *L) Done() <-chan struct{} { return l.line } +// Close may panic for an uninitialized L func (l *L) Close() { - l.Do(func() { - defer func() { _ = recover() }() // swallow any panics here + if atomic.AddInt32(&l.value, 1) == 1 { close(l.line) - }) + } + <-l.line // concurrent calls to Close block until the latch is actually closed } func (l *L) Closed() (result bool) { @@ -38,6 +43,6 @@ func (l *L) Closed() (result bool) { // Reset clears the state of the latch, not safe to execute concurrently with other L methods. func (l *L) Reset() *L { - l.line, l.Once = make(chan struct{}), sync.Once{} + l.line, l.value = make(chan struct{}), 0 return l } From 469572fc128c9aa9a737db44d76db65862f07759 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 5 May 2017 20:57:29 +0000 Subject: [PATCH 13/67] offers: add Partition for Slice, rewrite GroupBy for Index --- api/v1/lib/extras/offers/offers.go | 35 ++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/offers/offers.go index a8615af3..c9fa1bd7 100644 --- a/api/v1/lib/extras/offers/offers.go +++ b/api/v1/lib/extras/offers/offers.go @@ -178,17 +178,22 @@ func (slice Slice) GroupBy(kf KeyFunc) map[interface{}]Slice { return result } -func (index Index) GroupBy(kf KeyFunc) map[interface{}]Slice { +func (index Index) GroupBy(kf KeyFunc) map[interface{}]Index { if kf == nil { panic("keyFunc must not be nil") } if len(index) == 0 { return nil } - result := make(map[interface{}]Slice) - for _, offer := range index { + result := make(map[interface{}]Index) + for i, offer := range index { groupKey := kf(offer) - result[groupKey] = append(result[groupKey], *offer) + group, ok := result[groupKey] + if !ok { + group = make(Index) + result[groupKey] = group + } + group[i] = offer } return result } @@ -210,6 +215,28 @@ func (index Index) Partition(f Filter) (accepted, rejected Index) { return } +func (s Slice) Partition(f Filter) (accepted, rejected []int) { + if f == nil { + accepted = make([]int, len(s)) + for i := range s { + accepted[i] = i + } + return + } + if sz := len(s); sz > 0 { + accepted, rejected = make([]int, 0, sz/2), make([]int, 0, sz/2) + for i := range s { + offer := &s[i] + if f.Accept(offer) { + accepted = append(accepted, i) + } else { + rejected = append(rejected, i) + } + } + } + return +} + func (index Index) Reindex(kf KeyFunc) Index { sz := len(index) if kf == nil || sz == 0 { From 83c17773d48f3f5f7162f515ec152a20f6ffe954 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 14 May 2017 02:51:46 +0000 Subject: [PATCH 14/67] offers: clarify docs for Find --- api/v1/lib/extras/offers/offers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/offers/offers.go index c9fa1bd7..012127fe 100644 --- a/api/v1/lib/extras/offers/offers.go +++ b/api/v1/lib/extras/offers/offers.go @@ -31,7 +31,8 @@ func (offers Index) IDs() []OfferID { return ids } -// Find returns the first Offer that passes the given filter function. +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. func (offers Slice) Find(filter Filter) *Offer { for i := range offers { offer := &offers[i] @@ -42,7 +43,8 @@ func (offers Slice) Find(filter Filter) *Offer { return nil } -// Find returns the first Offer that passes the given filter function. +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. func (offers Index) Find(filter Filter) *Offer { for _, offer := range offers { if filter.Accept(offer) { From a3af6217aca2a2915a4ec3fac2de54273339f343 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 14 May 2017 02:59:59 +0000 Subject: [PATCH 15/67] latch: unit test for Interface --- api/v1/lib/extras/latch/latch_test.go | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 api/v1/lib/extras/latch/latch_test.go diff --git a/api/v1/lib/extras/latch/latch_test.go b/api/v1/lib/extras/latch/latch_test.go new file mode 100644 index 00000000..caab35a5 --- /dev/null +++ b/api/v1/lib/extras/latch/latch_test.go @@ -0,0 +1,33 @@ +package latch_test + +import ( + "testing" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/latch" +) + +func TestInterface(t *testing.T) { + l := New() + if l == nil { + t.Fatalf("expected a valid latch, not nil") + } + if l.Closed() { + t.Fatalf("expected new latch to be non-closed") + } + select { + case <-l.Done(): + t.Fatalf("Done chan unexpectedly closed for a new latch") + default: + } + for i := 0; i < 2; i++ { + l.Close() // multiple calls to close should not panic + } + if !l.Closed() { + t.Fatalf("expected closed latch") + } + select { + case <-l.Done(): + default: + t.Fatalf("Done chan unexpectedly non-closed for a closed latch") + } +} From b2c963ac854af30b5f81b1127792a541a0e66207 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 14 May 2017 03:19:26 +0000 Subject: [PATCH 16/67] ResourceBuilder API: pass pointers to builder, embed a Resource struct not a pointer --- api/v1/cmd/example-scheduler/app/state.go | 8 +++--- api/v1/cmd/msh/msh.go | 4 +-- api/v1/lib/builders.go | 34 +++++++++++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index c4dd89e1..cc310f9a 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -87,8 +87,8 @@ func prepareExecutorInfo( func buildWantsTaskResources(config Config) (r mesos.Resources) { r.Add( - *mesos.CPUs(config.taskCPU).Resource, - *mesos.Memory(config.taskMemory).Resource, + mesos.CPUs(config.taskCPU).Resource, + mesos.Memory(config.taskMemory).Resource, ) log.Println("wants-task-resources = " + r.String()) return @@ -96,8 +96,8 @@ func buildWantsTaskResources(config Config) (r mesos.Resources) { func buildWantsExecutorResources(config Config) (r mesos.Resources) { r.Add( - *mesos.CPUs(config.execCPU).Resource, - *mesos.Memory(config.execMemory).Resource, + mesos.CPUs(config.execCPU).Resource, + mesos.Memory(config.execMemory).Resource, ) log.Println("wants-executor-resources = " + r.String()) return diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index a8386413..8a69ffbe 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -72,8 +72,8 @@ func main() { } wantsResources = mesos.Resources{ - *mesos.CPUs(CPUs).Resource, - *mesos.Memory(Memory).Resource, + mesos.CPUs(CPUs).Resource, + mesos.Memory(Memory).Resource, } taskPrototype = mesos.TaskInfo{ Name: TaskName, diff --git a/api/v1/lib/builders.go b/api/v1/lib/builders.go index 209f6e26..2d5ba6ab 100644 --- a/api/v1/lib/builders.go +++ b/api/v1/lib/builders.go @@ -2,65 +2,65 @@ package mesos type ( // ResourceBuilder simplifies construction of Resource objects - ResourceBuilder struct{ *Resource } + ResourceBuilder struct{ Resource } // RangeBuilder simplifies construction of Range objects RangeBuilder struct{ Ranges } ) -func CPUs(value float64) ResourceBuilder { +func CPUs(value float64) *ResourceBuilder { return BuildResource().Name("cpus").Scalar(value) } -func Memory(value float64) ResourceBuilder { +func Memory(value float64) *ResourceBuilder { return BuildResource().Name("mem").Scalar(value) } -func Disk(value float64) ResourceBuilder { +func Disk(value float64) *ResourceBuilder { return BuildResource().Name("disk").Scalar(value) } -func GPUs(value uint) ResourceBuilder { +func GPUs(value uint) *ResourceBuilder { return BuildResource().Name("gpus").Scalar(float64(value)) } -func BuildRanges() RangeBuilder { - return RangeBuilder{Ranges: Ranges(nil)} +func BuildRanges() *RangeBuilder { + return &RangeBuilder{Ranges: Ranges(nil)} } // Span is a functional option for Ranges, defines the begin and end points of a // continuous span within a range -func (rb RangeBuilder) Span(bp, ep uint64) RangeBuilder { +func (rb *RangeBuilder) Span(bp, ep uint64) *RangeBuilder { rb.Ranges = append(rb.Ranges, Value_Range{Begin: bp, End: ep}) return rb } -func BuildResource() ResourceBuilder { - return ResourceBuilder{&Resource{}} +func BuildResource() *ResourceBuilder { + return &ResourceBuilder{} } -func (rb ResourceBuilder) Name(name string) ResourceBuilder { +func (rb *ResourceBuilder) Name(name string) *ResourceBuilder { rb.Resource.Name = name return rb } -func (rb ResourceBuilder) Role(role string) ResourceBuilder { +func (rb *ResourceBuilder) Role(role string) *ResourceBuilder { rb.Resource.Role = &role return rb } -func (rb ResourceBuilder) Scalar(x float64) ResourceBuilder { +func (rb *ResourceBuilder) Scalar(x float64) *ResourceBuilder { rb.Resource.Type = SCALAR.Enum() rb.Resource.Scalar = &Value_Scalar{Value: x} return rb } -func (rb ResourceBuilder) Set(x ...string) ResourceBuilder { +func (rb *ResourceBuilder) Set(x ...string) *ResourceBuilder { rb.Resource.Type = SET.Enum() rb.Resource.Set = &Value_Set{Item: x} return rb } -func (rb ResourceBuilder) Ranges(rs Ranges) ResourceBuilder { +func (rb *ResourceBuilder) Ranges(rs Ranges) *ResourceBuilder { rb.Resource.Type = RANGES.Enum() rb.Resource.Ranges = rb.Resource.Ranges.Add(&Value_Ranges{Range: rs}) return rb } -func (rb ResourceBuilder) Disk(persistenceID, containerPath string) ResourceBuilder { +func (rb *ResourceBuilder) Disk(persistenceID, containerPath string) *ResourceBuilder { rb.Resource.Disk = &Resource_DiskInfo{} if containerPath != "" { rb.Resource.Disk.Volume = &Volume{ContainerPath: containerPath} @@ -70,7 +70,7 @@ func (rb ResourceBuilder) Disk(persistenceID, containerPath string) ResourceBuil } return rb } -func (rb ResourceBuilder) Revocable() ResourceBuilder { +func (rb *ResourceBuilder) Revocable() *ResourceBuilder { rb.Resource.Revocable = &Resource_RevocableInfo{} return rb } From cdcea0b3e02a80611fe49b21f4d7ba7d0d7d70f0 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 14 May 2017 03:19:51 +0000 Subject: [PATCH 17/67] offers: move ContainsResources --- api/v1/lib/extras/offers/filters.go | 8 ++++++++ api/v1/lib/extras/offers/offers.go | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1/lib/extras/offers/filters.go b/api/v1/lib/extras/offers/filters.go index 080a23c3..9a872665 100644 --- a/api/v1/lib/extras/offers/filters.go +++ b/api/v1/lib/extras/offers/filters.go @@ -59,3 +59,11 @@ func ByUnavailability(f func(u *Unavailability) bool) Filter { return f(o.Unavailability) }) } + +// ContainsResources returns a filter function that returns true if the Resources of an Offer +// contain the wanted Resources. +func ContainsResources(wanted Resources) Filter { + return FilterFunc(func(o *Offer) bool { + return Resources(o.Resources).Flatten().ContainsAll(wanted) + }) +} diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/offers/offers.go index 012127fe..75ccdfef 100644 --- a/api/v1/lib/extras/offers/offers.go +++ b/api/v1/lib/extras/offers/offers.go @@ -86,14 +86,6 @@ func (offers Slice) FilterNot(filter Filter) Slice { return offers.Filter(not(fi // FilterNot returns the subset of the Index that does not match the given filter. func (offers Index) FilterNot(filter Filter) Index { return offers.Filter(not(filter)) } -// ContainsResources returns a filter function that returns true if the Resources of an Offer -// contain the wanted Resources. -func ContainsResources(wanted Resources) Filter { - return FilterFunc(func(o *Offer) bool { - return Resources(o.Resources).Flatten().ContainsAll(wanted) - }) -} - // DefaultKeyFunc indexes offers by their OfferID. var DefaultKeyFunc = KeyFunc(func(o *Offer) interface{} { return o.GetID() }) From a91a6c6e2d5f785c9fcf755081a0c6f42d881755 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 14 May 2017 03:26:49 +0000 Subject: [PATCH 18/67] more clearly doc ACKing behavior --- api/v1/lib/scheduler/events/events.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index d0ec2375..2ed65d44 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -141,13 +141,14 @@ type AckError struct { func (err *AckError) Error() string { return err.Cause.Error() } // AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every -// UPDATE event that's received. +// UPDATE event that's received (that requests an ACK). func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { return WhenFunc(scheduler.Event_UPDATE, func(e *scheduler.Event) (err error) { var ( s = e.GetUpdate().GetStatus() uuid = s.GetUUID() ) + // only ACK non-empty UUID's, as per mesos scheduler spec if len(uuid) > 0 { ack := calls.Acknowledge( s.GetAgentID().GetValue(), From 979e6edbbfafdc6b7a7cb9dbba873d2b6b2a88db Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 17:29:46 +0000 Subject: [PATCH 19/67] initial support for rule generator and scheduler event rules --- Makefile | 4 + api/v1/lib/extras/rules/rules.go | 412 ++++++++++++++++++ .../eventrules/eventrules_generated.go | 252 +++++++++++ .../eventrules/eventrules_generated_test.go | 71 +++ api/v1/lib/extras/scheduler/eventrules/gen.go | 3 + .../extras/scheduler/eventrules/handlers.go | 61 +++ 6 files changed, 803 insertions(+) create mode 100644 api/v1/lib/extras/rules/rules.go create mode 100644 api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go create mode 100644 api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go create mode 100644 api/v1/lib/extras/scheduler/eventrules/gen.go create mode 100644 api/v1/lib/extras/scheduler/eventrules/handlers.go diff --git a/Makefile b/Makefile index 8bc5998f..d2142b4f 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,10 @@ sync: (cd ${API_VENDOR}; govendor sync) (cd ${CMD_VENDOR}; govendor sync) +.PHONY: generate +generate: + go generate ./api/v1/lib/extras/scheduler/eventrules + GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) UID ?= $(shell id -u $$USER) diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go new file mode 100644 index 00000000..00587709 --- /dev/null +++ b/api/v1/lib/extras/rules/rules.go @@ -0,0 +1,412 @@ +// +build ignore + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + "text/template" +) + +type ( + config struct { + Package string + Imports []string + EventType string + } +) + +func (c *config) String() string { + if c == nil { + return "" + } + return fmt.Sprintf("%#v", ([]string)(c.Imports)) +} + +func (c *config) Set(s string) error { + c.Imports = append(c.Imports, s) + return nil +} + +func main() { + var ( + c = config{ + Package: os.Getenv("GOPACKAGE"), + EventType: "Event", + } + defaultOutput = "foo.go" + ) + if c.Package != "" { + defaultOutput = c.Package + "_generated.go" + } + + output := defaultOutput + + flag.StringVar(&c.Package, "package", c.Package, "destination package") + flag.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") + flag.StringVar(&output, "output", output, "path of the to-be-generated file") + flag.Var(&c, "import", "packages to import") + flag.Parse() + + if c.Package == "" { + c.Package = "foo" + } + if c.EventType == "" { + c.EventType = "Event" + } + if output == "" { + output = defaultOutput + } + + testOutput := output + "_test" + if strings.HasSuffix(output, ".go") { + testOutput = output[:len(output)-3] + "_test.go" + } + + // main template + f, err := os.Create(output) + if err != nil { + log.Fatal(err) + } + defer f.Close() + rulesTemplate.Execute(f, c) + + // unit test template + f, err = os.Create(testOutput) + if err != nil { + log.Fatal(err) + } + testTemplate.Execute(f, c) +} + +var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "fmt" + "sync" +{{range .Imports}} + {{ printf "%q" . }} +{{ end -}} +) + +type ( + // Rule executes a filter, rule, or decorator function; if the returned event is nil then + // no additional Rule func should be processed for the event. + // Rule implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + Rule func({{.EventType}}, error, Chain) ({{.EventType}}, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func({{.EventType}}, error) ({{.EventType}}, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +// chainIdentity is a Chain that returns the arguments as its results. +var chainIdentity = func(e {{.EventType}}, err error) ({{.EventType}}, error) { + return e, err +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if len(rs) == 0 { + return ch(e, err) // noop + } + // we know there's at least 1 rule in the initial list; start with it and let the chain + // handle the iteration. + return ch(rs[0](e, err, NewChain(rs))) +} + +// Rule adapts Rules to the Rule interface, for convenient call chaining. +func (rs Rules) Rule() Rule { return rs.Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + return b + } + if b == nil { + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + var result ErrorList + for _, err := range es { + if err != nil { + if multi, ok := err.(ErrorList); ok { + // flatten nested error lists + if len(multi) > 0 { + result = append(result, multi...) + } + } else { + result = append(result, err) + } + } + } + return result.Err() +} + +// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + var once sync.Once + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + once.Do(func() { + e, err = r(e, err, ch) + }) + return e, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. +// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + select { + case <-p: + // do something + return r(e, err, ch) + default: + // drop + return ch(nil, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this is a noop. +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if forward() { + return r(e, err, ch) + } + // else, drop + return ch(nil, err) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving +// rule. +func (r Rule) DropOnError() Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if err != nil || e == nil { + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing +// for any of the following cases: +// - there are no more rules to process +// - the event has been zero'ed out (nil) +// Any nil rules in the list are processed as skipped (noop's). +func NewChain(rs Rules) Chain { + sz := len(rs) + if sz == 0 { + return chainIdentity + } + var ( + i = 0 + chain Chain + ) + chain = Chain(func(x {{.EventType}}, y error) ({{.EventType}}, error) { + i++ + if i >= sz || x == nil { + // we're at the end, or DROP was issued (x==nil) + return x, y + } else if rs[i] != nil { + return rs[i](x, y, chain) + } else { + return chain(x, y) + } + + }) + return chain +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if e != nil && err == nil { + // bypass remainder of chain + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} +`)) + +var testTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "errors" + "reflect" + "testing" +{{range .Imports}} + {{ printf "%q" . }} +{{ end -}} +) + +func counter(i *int) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + *i++ + return ch(e, err) + } +} + +func returnError(re error) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return ch(e, Error2(err, re)) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + }{ + {nil, nil, nil}, + {a, nil, a}, + {nil, b, b}, + {a, b, ErrorList{a, b}}, + } { + result := Error2(tc.a, tc.b) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + if !IsErrorList(result) && result == tc.wants { + continue + } + if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { + continue + } + } + t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } +} +`)) diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go new file mode 100644 index 00000000..bb4dd636 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -0,0 +1,252 @@ +package eventrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + // Rule executes a filter, rule, or decorator function; if the returned event is nil then + // no additional Rule func should be processed for the event. + // Rule implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + Rule func(*scheduler.Event, error, Chain) (*scheduler.Event, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(*scheduler.Event, error) (*scheduler.Event, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +// chainIdentity is a Chain that returns the arguments as its results. +var chainIdentity = func(e *scheduler.Event, err error) (*scheduler.Event, error) { + return e, err +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if len(rs) == 0 { + return ch(e, err) // noop + } + // we know there's at least 1 rule in the initial list; start with it and let the chain + // handle the iteration. + return ch(rs[0](e, err, NewChain(rs))) +} + +// Rule adapts Rules to the Rule interface, for convenient call chaining. +func (rs Rules) Rule() Rule { return rs.Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + return b + } + if b == nil { + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + var result ErrorList + for _, err := range es { + if err != nil { + if multi, ok := err.(ErrorList); ok { + // flatten nested error lists + if len(multi) > 0 { + result = append(result, multi...) + } + } else { + result = append(result, err) + } + } + } + return result.Err() +} + +// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + var once sync.Once + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + once.Do(func() { + e, err = r(e, err, ch) + }) + return e, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. +// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + select { + case <-p: + // do something + return r(e, err, ch) + default: + // drop + return ch(nil, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this is a noop. +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if forward() { + return r(e, err, ch) + } + // else, drop + return ch(nil, err) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving +// rule. +func (r Rule) DropOnError() Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if err != nil || e == nil { + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing +// for any of the following cases: +// - there are no more rules to process +// - the event has been zero'ed out (nil) +// Any nil rules in the list are processed as skipped (noop's). +func NewChain(rs Rules) Chain { + sz := len(rs) + if sz == 0 { + return chainIdentity + } + var ( + i = 0 + chain Chain + ) + chain = Chain(func(x *scheduler.Event, y error) (*scheduler.Event, error) { + i++ + if i >= sz || x == nil { + // we're at the end, or DROP was issued (x==nil) + return x, y + } else if rs[i] != nil { + return rs[i](x, y, chain) + } else { + return chain(x, y) + } + + }) + return chain +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if e != nil && err == nil { + // bypass remainder of chain + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go new file mode 100644 index 00000000..f5a6f1de --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -0,0 +1,71 @@ +package eventrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func counter(i *int) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + *i++ + return ch(e, err) + } +} + +func returnError(re error) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return ch(e, Error2(err, re)) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + }{ + {nil, nil, nil}, + {a, nil, a}, + {nil, b, b}, + {a, b, ErrorList{a, b}}, + } { + result := Error2(tc.a, tc.b) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + if !IsErrorList(result) && result == tc.wants { + continue + } + if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { + continue + } + } + t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } +} diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go new file mode 100644 index 00000000..44a4854c --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -0,0 +1,3 @@ +package eventrules + +//go:generate go run ../../rules/rules.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers.go new file mode 100644 index 00000000..16a08b8d --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/handlers.go @@ -0,0 +1,61 @@ +package eventrules + +import ( + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +// Handler generates a rule that executes the given handler. +func Handle(h events.Handler) Rule { + if h == nil { + return nil + } + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + newErr := h.HandleEvent(e) + return chain(e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h events.HandlerFunc) Rule { + return Handle(events.Handler(h)) +} + +// Handler returns a rule that invokes the given Handler +func (r Rule) Handle(h events.Handler) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h events.HandlerFunc) Rule { + return r.Handle(events.Handler(h)) +} + +// HandleEvent implements events.Handler for Rule +func (r Rule) HandleEvent(e *scheduler.Event) (err error) { + if r == nil { + return nil + } + _, err = r(e, nil, chainIdentity) + return +} + +// HandleEvent implements events.Handler for Rules +func (rs Rules) HandleEvent(e *scheduler.Event) error { + return rs.Rule().HandleEvent(e) +} + +/* +// Apply returns the result of a singleton rule set (the receiver) applied to the given event handler. +func (r Rule) Apply(h events.Handler) events.HandlerFunc { + if r == nil { + return h.HandleEvent + } + return r.Handle(h).HandleEvent +} + +// ApplyF is the functional equivalent of Apply +func (r Rule) ApplyF(h events.HandlerFunc) events.HandlerFunc { + return r.Apply(events.Handler(h)) +} +*/ From 1c9595abeb9baece9d3e7dee6e77160c717ba83e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 17:31:47 +0000 Subject: [PATCH 20/67] convenience rules for scheduler components --- .../lib/extras/scheduler/controller/rules.go | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 api/v1/lib/extras/scheduler/controller/rules.go diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go new file mode 100644 index 00000000..a4ff8f86 --- /dev/null +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -0,0 +1,125 @@ +package controller + +import ( + "fmt" + "log" + "time" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +// ErrEvent errors are generated by LiftErrors upon receiving an ERROR event from Mesos. +type ErrEvent string + +func (e ErrEvent) Error() string { + return string(e) +} + +// LiftErrors extract the error message from a scheduler error event and returns it as an ErrEvent +// so that downstream rules/handlers may continue processing. +func LiftErrors() Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) + } + if e.GetType() == scheduler.Event_ERROR { + // it's recommended that we abort and re-try subscribing; returning an + // error here will cause the event loop to terminate and the connection + // will be reset. + return chain(e, ErrEvent(e.GetError().GetMessage())) + } + return chain(e, nil) + } +} + +// StateError is returned when the system encounters an unresolvable state transition error and +// should likely exit. +type StateError string + +func (err StateError) Error() string { return string(err) } + +func TrackSubscription(frameworkIDStore store.Singleton, failoverTimeout time.Duration) Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) + } + if e.GetType() == scheduler.Event_SUBSCRIBED { + var ( + storedFrameworkID, err = frameworkIDStore.Get() + frameworkID = e.GetSubscribed().GetFrameworkID().GetValue() + ) + if err != nil && err != store.ErrNotFound { + return chain(e, err) + } + // order of `if` statements are important: tread carefully w/ respect to future changes + if frameworkID == "" { + // sanity check, should **never** happen + return chain(e, StateError("mesos sent an empty frameworkID?!")) + } + if storedFrameworkID != "" && storedFrameworkID != frameworkID && failoverTimeout > 0 { + return chain(e, StateError(fmt.Sprintf( + "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", failoverTimeout))) + } + if storedFrameworkID != frameworkID { + frameworkIDStore.Set(frameworkID) + } + } + return chain(e, nil) + } +} + +// AckStatusUpdates sends an acknowledgement of a task status update back to mesos and drops the event if +// sending the ack fails. If successful, the specified err param (if any) is forwarded. Acknowledgements +// are only attempted for task status updates tagged with a UUID. +func AckStatusUpdates(caller calls.Caller) Rule { + return AckStatusUpdatesF(func() calls.Caller { return caller }) +} + +// AckStatusUpdatesF is a functional adapter for AckStatusUpdates, useful for cases where the caller may +// change over time. +func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + // aggressively attempt to ack updates: even if there's pre-existing error state attempt + // to acknowledge all status updates. + origErr := err + if e.GetType() == scheduler.Event_UPDATE { + var ( + s = e.GetUpdate().GetStatus() + uuid = s.GetUUID() + ) + // only ACK non-empty UUID's, as per mesos scheduler spec + if len(uuid) > 0 { + ack := calls.Acknowledge( + s.GetAgentID().GetValue(), + s.TaskID.Value, + uuid, + ) + err = calls.CallNoData(callerLookup(), ack) + if err != nil { + err = &events.AckError{Ack: ack, Cause: err} + return nil, Error2(origErr, err) // drop + } + } + } + return chain(e, origErr) + } +} + +var ( + // EventLabel is, by default, logged as the first argument by EventLogger + EventLabel = "event" + // EventLogger is the logger used by the LogEvents rule generator + EventLogger = func(e *scheduler.Event) { log.Println(EventLabel, e) } +) + +// LogEvents returns a rule that logs scheduler events to the EventLogger +func LogEvents() Rule { + return Rule(func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + EventLogger(e) + return chain(e, err) + }) +} From 2e20eb553dd4ee7489a172624387254f42ef84e8 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 17:37:06 +0000 Subject: [PATCH 21/67] rename event predicates APIs. refactor handler sets as first class types --- api/v1/lib/scheduler/events.go | 4 +- api/v1/lib/scheduler/events/events.go | 91 +++++++++++++++-------- api/v1/lib/scheduler/events/predicates.go | 3 +- 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/api/v1/lib/scheduler/events.go b/api/v1/lib/scheduler/events.go index c998f003..26bcea9e 100644 --- a/api/v1/lib/scheduler/events.go +++ b/api/v1/lib/scheduler/events.go @@ -14,5 +14,5 @@ func (ep EventPredicate) Apply(e *Event) (result bool) { return } -// Happens implements scheduler/events.Happens -func (t Event_Type) Happens() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } +// Predicate implements scheduler/events.Predicate +func (t Event_Type) Predicate() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index 2ed65d44..7366280c 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -1,6 +1,8 @@ package events import ( + "sync" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" ) @@ -15,11 +17,14 @@ type ( // HandlerFunc is a functional adaptation of the Handler interface HandlerFunc func(*scheduler.Event) error + HandlerSet map[scheduler.Event_Type]Handler + HandlerFuncSet map[scheduler.Event_Type]HandlerFunc + // Mux maps event types to Handlers (only one Handler for each type). A "default" // Handler implementation may be provided to handle cases in which there is no // registered Handler for specific event type. Mux struct { - handlers map[scheduler.Event_Type]Handler + handlers HandlerSet defaultHandler Handler } @@ -30,18 +35,20 @@ type ( // Handlers aggregates Handler things Handlers []Handler - Happens interface { - Happens() scheduler.EventPredicate + Predicate interface { + Predicate() scheduler.EventPredicate } ) // HandleEvent implements Handler for HandlerFunc func (f HandlerFunc) HandleEvent(e *scheduler.Event) error { return f(e) } +func NoopHandler() HandlerFunc { return func(_ *scheduler.Event) error { return nil } } + // NewMux generates and returns a new, empty Mux instance. func NewMux(opts ...Option) *Mux { m := &Mux{ - handlers: make(map[scheduler.Event_Type]Handler), + handlers: make(HandlerSet), } m.With(opts...) return m @@ -62,15 +69,15 @@ func (m *Mux) With(opts ...Option) Option { } // HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *scheduler.Event) (err error) { - h, found := m.handlers[e.GetType()] - if !found { - h = m.defaultHandler +func (m *Mux) HandleEvent(e *scheduler.Event) error { + ok, err := m.handlers.tryHandleEvent(e) + if ok { + return err } - if h != nil { - err = h.HandleEvent(e) + if m.defaultHandler != nil { + return m.defaultHandler.HandleEvent(e) } - return + return nil } // Handle returns an option that configures a Handler to handle a specific event type. @@ -88,8 +95,22 @@ func Handle(et scheduler.Event_Type, eh Handler) Option { } } +// HandleEvent implements Handler for HandlerSet +func (hs HandlerSet) HandleEvent(e *scheduler.Event) (err error) { + _, err = hs.tryHandleEvent(e) + return +} + +// tryHandleEvent returns true if the event was handled by a member of the HandlerSet +func (hs HandlerSet) tryHandleEvent(e *scheduler.Event) (bool, error) { + if h := hs[e.GetType()]; h != nil { + return true, h.HandleEvent(e) + } + return false, nil +} + // Map returns an Option that configures multiple Handler objects. -func Map(handlers map[scheduler.Event_Type]Handler) (option Option) { +func (handlers HandlerSet) ToOption() (option Option) { option = func(m *Mux) Option { type history struct { et scheduler.Event_Type @@ -114,13 +135,18 @@ func Map(handlers map[scheduler.Event_Type]Handler) (option Option) { return } -// MapFuncs is the functional adaptation of Map -func MapFuncs(handlers map[scheduler.Event_Type]HandlerFunc) (option Option) { - h := make(map[scheduler.Event_Type]Handler, len(handlers)) +// HandlerSet converts a HandlerFuncSet +func (handlers HandlerFuncSet) HandlerSet() HandlerSet { + h := make(HandlerSet, len(handlers)) for k, v := range handlers { h[k] = v } - return Map(h) + return h +} + +// ToOption converts a HandlerFuncSet +func (hs HandlerFuncSet) ToOption() (option Option) { + return hs.HandlerSet().ToOption() } // DefaultHandler returns an option that configures the default handler that's invoked @@ -133,6 +159,7 @@ func DefaultHandler(eh Handler) Option { } } +// AckError wraps a caller-generated error and tracks the call that failed. type AckError struct { Ack *scheduler.Call Cause error @@ -142,7 +169,7 @@ func (err *AckError) Error() string { return err.Cause.Error() } // AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every // UPDATE event that's received (that requests an ACK). -func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { +func AcknowledgeUpdates(callerLookup func() calls.Caller) Handler { return WhenFunc(scheduler.Event_UPDATE, func(e *scheduler.Event) (err error) { var ( s = e.GetUpdate().GetStatus() @@ -155,7 +182,7 @@ func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { s.TaskID.Value, uuid, ) - err = calls.CallNoData(callerGetter(), ack) + err = calls.CallNoData(callerLookup(), ack) if err != nil { err = &AckError{ack, err} } @@ -164,33 +191,39 @@ func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { }) } +// When +// Deprecated in favor of Rules. func Once(h Handler) Handler { - called := false + var once sync.Once return HandlerFunc(func(e *scheduler.Event) (err error) { - if !called { - called = true + once.Do(func() { err = h.HandleEvent(e) - } + }) return }) } +// When +// Deprecated in favor of Rules. func OnceFunc(h HandlerFunc) Handler { return Once(h) } -func When(p Happens, h Handler) Handler { +// When +// Deprecated in favor of Rules. +func When(p Predicate, h Handler) Handler { return HandlerFunc(func(e *scheduler.Event) (err error) { - if p.Happens().Apply(e) { + if p.Predicate().Apply(e) { err = h.HandleEvent(e) } return }) } -func WhenFunc(p Happens, h HandlerFunc) Handler { return When(p, h) } - -var _ = Handler(Handlers{}) // Handlers implements Handler +// WhenFunc +// Deprecated in favor of Rules. +func WhenFunc(p Predicate, h HandlerFunc) Handler { return When(p, h) } -// HandleEvent implements Handler for Handlers +// HandleEvent implements Handler for Handlers. +// Deprecated in favor of Rules. func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { for _, h := range hs { if h != nil { @@ -199,5 +232,5 @@ func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { } } } - return err + return } diff --git a/api/v1/lib/scheduler/events/predicates.go b/api/v1/lib/scheduler/events/predicates.go index fb5be74d..6aefb64a 100644 --- a/api/v1/lib/scheduler/events/predicates.go +++ b/api/v1/lib/scheduler/events/predicates.go @@ -6,6 +6,7 @@ import ( type PredicateBool func() bool -func (b PredicateBool) Happens() scheduler.EventPredicate { +// Predicate implements scheduler.events.Predicate +func (b PredicateBool) Predicate() scheduler.EventPredicate { return func(_ *scheduler.Event) bool { return b() } } From f1f3f49210425c291facd362c3f66b2ddb47ff61 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 17:38:38 +0000 Subject: [PATCH 22/67] overhaul existing examples and scheduler controller infra --- api/v1/cmd/example-scheduler/app/app.go | 201 ++++++++--------- api/v1/cmd/example-scheduler/app/config.go | 4 +- api/v1/cmd/example-scheduler/app/state.go | 1 - api/v1/cmd/msh/msh.go | 207 +++++++++--------- .../extras/scheduler/controller/controller.go | 188 ++++++++-------- api/v1/lib/extras/store/singleton.go | 125 ++++++++++- 6 files changed, 405 insertions(+), 321 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 028fc124..d21953f3 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -2,7 +2,6 @@ package app import ( "errors" - "fmt" "io" "log" "strconv" @@ -12,6 +11,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/backoff" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" @@ -42,108 +42,84 @@ func Run(cfg Config) error { // TODO(jdef) how to track/handle timeout errors that occur for SUBSCRIBE calls? we should // probably tolerate X number of subsequent subscribe failures before bailing. we'll need // to track the lastCallAttempted along with subsequentSubscribeTimeouts. - err = controller.New().Run(buildControllerConfig(state, shutdown)) - if state.err != nil { - err = state.err - } - return err -} -func buildControllerConfig(state *internalState, shutdown <-chan struct{}) controller.Config { - var ( - frameworkIDStore = store.NewInMemorySingleton() - controlContext = &controller.ContextAdapter{ - DoneFunc: state.done.Closed, - FrameworkIDFunc: frameworkIDStore.Get, - ErrorFunc: func(err error) { - if err != nil { - if err != io.EOF { - log.Println(err) - } - if _, ok := err.(StateError); ok { - state.done.Close() - } - return - } - log.Println("disconnected") - }, - } - ) + frameworkIDStore := store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) state.cli = calls.Decorators{ callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}), - // automatically set the frameworkID for all outgoing calls - calls.SubscribedCaller(frameworkIDStore.Get), + calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), // automatically set the frameworkID for all outgoing calls }.Apply(state.cli) - return controller.Config{ - Context: controlContext, - Framework: buildFrameworkInfo(state.config), - Caller: state.cli, - RegistrationTokens: backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), - - Handler: events.Decorators{ + err = controller.Run( + buildFrameworkInfo(state.config), + state.cli, + controller.WithDone(state.done.Closed), + controller.WithEventHandler( + buildEventHandler(state, frameworkIDStore), eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), events.Decorator(logAllEvents).If(state.config.verbose), - }.Apply(buildEventHandler(state, frameworkIDStore)), + ), + controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithRegistrationTokens( + backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), + ), + controller.WithSubscriptionTerminated(func(err error) { + if err != nil { + if err != io.EOF { + log.Println(err) + } + if _, ok := err.(StateError); ok { + state.done.Close() + } + return + } + log.Println("disconnected") + }), + ) + if state.err != nil { + err = state.err } + return err } // buildEventHandler generates and returns a handler to process events received from the subscription. func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { - // TODO(jdef) would be nice to merge this ack handler with the status update handler below; need to - // figure out appropriate error propagation among chained handlers. - ack := events.AcknowledgeUpdates(func() calls.Caller { return state.cli }) - return events.NewMux( - events.DefaultHandler(events.HandlerFunc(controller.DefaultHandler)), - events.MapFuncs(map[scheduler.Event_Type]events.HandlerFunc{ - scheduler.Event_FAILURE: func(e *scheduler.Event) error { - log.Println("received a FAILURE event") - f := e.GetFailure() - failure(f.ExecutorID, f.AgentID, f.Status) - return nil - }, - scheduler.Event_OFFERS: func(e *scheduler.Event) error { + logger := controller.LogEvents() + return controller.LiftErrors().Handle(events.HandlerSet{ + scheduler.Event_FAILURE: logger.HandleF(func(e *scheduler.Event) error { + f := e.GetFailure() + failure(f.ExecutorID, f.AgentID, f.Status) + return nil + }), + scheduler.Event_OFFERS: trackOffersReceived(state).AndThen().HandleF( + func(e *scheduler.Event) error { if state.config.verbose { log.Println("received an OFFERS event") } - offers := e.GetOffers().GetOffers() - state.metricsAPI.offersReceived.Int(len(offers)) - resourceOffers(state, offers) - return nil - }, - scheduler.Event_UPDATE: func(e *scheduler.Event) error { - if err := ack.HandleEvent(e); err != nil { - log.Printf("failed to ack status update for task: %+v", err) - // TODO(jdef) we don't return the error because that would cause the subscription - // to terminate; is that the right thing to do? - } - statusUpdate(state, e.GetUpdate().GetStatus()) - return nil - }, - scheduler.Event_SUBSCRIBED: func(e *scheduler.Event) error { - log.Println("received a SUBSCRIBED event") - frameworkID := e.GetSubscribed().GetFrameworkID().GetValue() - // order of `if` statements are important: tread carefully w/ respect to future changes - if frameworkID == "" { - // sanity check, should **never** happen - return StateError("mesos gave us an empty frameworkID") - } - if state.frameworkID != "" && state.frameworkID != frameworkID && state.config.checkpoint { - return StateError(fmt.Sprintf( - "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", - state.config.failoverTimeout)) - } - if state.frameworkID != frameworkID { - state.frameworkID = frameworkID - frameworkIDStore.Set(frameworkID) - log.Println("FrameworkID", frameworkID) - } - return nil - }, - }), - ) + return resourceOffers(state, e.GetOffers().GetOffers()) + }), + scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), + scheduler.Event_SUBSCRIBED: eventrules.Rules{ + logger, + controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout), + }, + }) +} + +func trackOffersReceived(state *internalState) eventrules.Rule { + return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + if err == nil { + state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers())) + } + return chain(e, nil) + + } } func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { @@ -163,7 +139,7 @@ func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { } } -func resourceOffers(state *internalState, offers []mesos.Offer) { +func resourceOffers(state *internalState, offers []mesos.Offer) error { callOption := calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) tasksLaunchedThisCycle := 0 offersDeclined := 0 @@ -239,36 +215,41 @@ func resourceOffers(state *internalState, offers []mesos.Offer) { if state.config.summaryMetrics { state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) } + return nil } -func statusUpdate(state *internalState, s mesos.TaskStatus) { - if state.config.verbose { - msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() - if m := s.GetMessage(); m != "" { - msg += " with message '" + m + "'" +func statusUpdate(state *internalState) events.HandlerFunc { + return func(e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() + if state.config.verbose { + msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() + if m := s.GetMessage(); m != "" { + msg += " with message '" + m + "'" + } + log.Println(msg) } - log.Println(msg) - } - switch st := s.GetState(); st { - case mesos.TASK_FINISHED: - state.tasksFinished++ - state.metricsAPI.tasksFinished() + switch st := s.GetState(); st { + case mesos.TASK_FINISHED: + state.tasksFinished++ + state.metricsAPI.tasksFinished() - if state.tasksFinished == state.totalTasks { - log.Println("mission accomplished, terminating") + if state.tasksFinished == state.totalTasks { + log.Println("mission accomplished, terminating") + state.done.Close() + } else { + tryReviveOffers(state) + } + + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + state.err = errors.New("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") state.done.Close() - } else { - tryReviveOffers(state) } - - case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: - state.err = errors.New("Exiting because task " + s.GetTaskID().Value + - " is in an unexpected state " + st.String() + - " with reason " + s.GetReason().String() + - " from source " + s.GetSource().String() + - " with message '" + s.GetMessage() + "'") - state.done.Close() + return nil } } diff --git a/api/v1/cmd/example-scheduler/app/config.go b/api/v1/cmd/example-scheduler/app/config.go index ecd7cbb9..9727dff4 100644 --- a/api/v1/cmd/example-scheduler/app/config.go +++ b/api/v1/cmd/example-scheduler/app/config.go @@ -49,8 +49,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.Var(&cfg.codec, "codec", "Codec to encode/decode scheduler API communications [protobuf, json]") fs.StringVar(&cfg.url, "url", cfg.url, "Mesos scheduler API URL") fs.DurationVar(&cfg.timeout, "timeout", cfg.timeout, "Mesos scheduler API connection timeout") - fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout") - fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable framework checkpointing") + fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout (recover from scheduler failure)") + fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable agent checkpointing for framework tasks (recover from agent failure)") fs.StringVar(&cfg.principal, "principal", cfg.principal, "Framework principal with which to authenticate") fs.StringVar(&cfg.hostname, "hostname", cfg.hostname, "Framework hostname that is advertised to the master") fs.Var(&cfg.labels, "label", "Framework label, may be specified multiple times") diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index cc310f9a..de768e88 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -218,7 +218,6 @@ type internalState struct { tasksLaunched int tasksFinished int totalTasks int - frameworkID string role string executor *mesos.ExecutorInfo cli calls.Caller diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 8a69ffbe..f8856701 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -22,6 +22,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/extras/offers" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" @@ -43,13 +44,11 @@ var ( CPUs = float64(0.010) Memory = float64(64) - frameworkIDStore store.Singleton - shouldDecline bool - refuseSeconds = calls.RefuseSeconds(5 * time.Second) - stop func() - exitCode int - wantsResources mesos.Resources - taskPrototype mesos.TaskInfo + frameworkIDStore store.Singleton + declineAndSuppress bool + refuseSeconds = calls.RefuseSeconds(5 * time.Second) + wantsResources mesos.Resources + taskPrototype mesos.TaskInfo ) func init() { @@ -60,7 +59,12 @@ func init() { flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command") flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command") - frameworkIDStore = store.NewInMemorySingleton() + frameworkIDStore = store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) } func main() { @@ -85,44 +89,40 @@ func main() { if len(args) > 1 { taskPrototype.Command.Arguments = args[1:] } - err := controller.New().Run(buildControllerConfig(User)) - if err != nil { - log.Fatal(err) + if err := run(); err != nil { + if exitErr, ok := err.(ExitError); ok { + if code := int(exitErr); code != 0 { + log.Println(exitErr) + os.Exit(code) + } + // else, code=0 indicates success, exit normally + } else { + panic(fmt.Sprintf("%#v", err)) + } } - os.Exit(exitCode) } -func buildControllerConfig(user string) controller.Config { +func run() error { var ( done = latch.New() caller = calls.Decorators{ - calls.SubscribedCaller(frameworkIDStore.Get), + calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), }.Apply(buildClient()) ) - stop = done.Close - return controller.Config{ - Context: &controller.ContextAdapter{ - DoneFunc: done.Closed, - FrameworkIDFunc: frameworkIDStore.Get, - ErrorFunc: func(err error) { - defer stop() - if err != nil { - // don't overwrite an existing error code - if exitCode == 0 { - exitCode = 10 - } - if err != io.EOF { - log.Printf("%#v", err) - } - return - } + + return controller.Run( + &mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)}, + caller, + controller.WithDone(done.Closed), + controller.WithEventHandler(buildEventHandler(caller)), + controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithSubscriptionTerminated(func(err error) { + defer done.Close() + if err == io.EOF { log.Println("disconnected") - }, - }, - Framework: &mesos.FrameworkInfo{User: user, Name: FrameworkName, Role: (*string)(&Role)}, - Caller: caller, - Handler: buildEventHandler(caller), - } + } + }), + ) } func buildClient() calls.Caller { @@ -132,80 +132,77 @@ func buildClient() calls.Caller { } func buildEventHandler(caller calls.Caller) events.Handler { - ack := events.AcknowledgeUpdates(func() calls.Caller { return caller }) - return events.NewMux( - events.DefaultHandler(events.HandlerFunc(controller.DefaultHandler)), - events.MapFuncs(map[scheduler.Event_Type]events.HandlerFunc{ - scheduler.Event_OFFERS: func(e *scheduler.Event) error { - return resourceOffers(caller, e.GetOffers().GetOffers()) - }, - scheduler.Event_UPDATE: func(e *scheduler.Event) error { - err := ack.HandleEvent(e) - if err != nil { - err = fmt.Errorf("failed to ack status update for task: %#v", err) - } - statusUpdate(e.GetUpdate().GetStatus()) - return err - }, - scheduler.Event_SUBSCRIBED: func(e *scheduler.Event) error { - log.Println("received a SUBSCRIBED event") - fid := e.GetSubscribed().GetFrameworkID().GetValue() - if fid == "" { - // sanity check, should **never** happen - return fmt.Errorf("mesos gave us an empty frameworkID") - } - if current := frameworkIDStore.Get(); current != fid { - err := frameworkIDStore.Set(fid) - if err != nil { - return err - } - log.Println("FrameworkID", fid) - } - return nil - }, - }), - ) + logger := controller.LogEvents() + return controller.LiftErrors().Handle(events.HandlerSet{ + scheduler.Event_FAILURE: logger, + scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(frameworkIDStore, 0)}, + scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), + scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), + }) } -func resourceOffers(caller calls.Caller, off []mesos.Offer) error { - if shouldDecline { - return calls.CallNoData(caller, calls.Suppress()) - } - var ( - index = offers.NewIndex(off, nil) - match = index.Find(offers.ContainsResources(wantsResources)) - ) - if match != nil { - task := taskPrototype - task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)} - task.AgentID = match.AgentID - task.Resources = mesos.Resources(match.Resources).Find(wantsResources.Flatten(Role.Assign())) - - if err := calls.CallNoData(caller, calls.Accept( - calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), - )); err != nil { - return err +func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { + return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) } + if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress { + return chain(e, err) + } + off := offers.Slice(e.GetOffers().GetOffers()) + err = calls.CallNoData(caller, calls.Decline(off.IDs()...).With(refuseSeconds)) + if err == nil { + // we shouldn't have received offers, maybe the prior suppress call failed? + err = calls.CallNoData(caller, calls.Suppress()) + } + return nil, err // drop + } +} - shouldDecline = true // safeguard and suppress future offers - if err := calls.CallNoData(caller, calls.Suppress()); err != nil { - return err +func resourceOffers(caller calls.Caller) events.HandlerFunc { + return func(e *scheduler.Event) (err error) { + var ( + off = e.GetOffers().GetOffers() + index = offers.NewIndex(off, nil) + match = index.Find(offers.ContainsResources(wantsResources)) + ) + if match != nil { + task := taskPrototype + task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)} + task.AgentID = match.AgentID + task.Resources = mesos.Resources(match.Resources).Find(wantsResources.Flatten(Role.Assign())) + + err = calls.CallNoData(caller, calls.Accept( + calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), + )) + if err != nil { + return + } + + declineAndSuppress = true + } else { + log.Println("rejected insufficient offers") + } + // decline all but the possible match + delete(index, match.GetID()) + err = calls.CallNoData(caller, calls.Decline(index.IDs()...).With(refuseSeconds)) + if err != nil { + return + } + if declineAndSuppress { + err = calls.CallNoData(caller, calls.Suppress()) } - } else { - // insufficient offers - log.Println("rejected insufficient offers") + return } - // decline all but the possible match - delete(index, match.GetID()) - return calls.CallNoData(caller, calls.Decline(index.IDs()...).With(refuseSeconds)) } -func statusUpdate(s mesos.TaskStatus) { +func statusUpdate(e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() switch st := s.GetState(); st { case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING: - log.Println("status update", st) + log.Printf("status update from agent %q: %v", s.GetAgentID().GetValue(), st) if st != mesos.TASK_FINISHED { - return + return nil } case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: log.Println("Exiting because task " + s.GetTaskID().Value + @@ -213,10 +210,14 @@ func statusUpdate(s mesos.TaskStatus) { " with reason " + s.GetReason().String() + " from source " + s.GetSource().String() + " with message '" + s.GetMessage() + "'") - exitCode = 3 + return ExitError(3) default: log.Println("unexpected task state, aborting", st) - exitCode = 4 + return ExitError(4) } - stop() + return ExitError(0) // kind of ugly, but better than os.Exit(0) } + +type ExitError int + +func (e ExitError) Error() string { return fmt.Sprintf("exit code %d", int(e)) } diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index c820c7e5..03a732e5 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -9,81 +9,111 @@ import ( ) type ( - Context interface { - // Done returns true when the controller should exit - Done() bool + // Option modifies a Config, returns an Option that acts as an "undo" + Option func(*Config) Option - // FrameworkID returns the current Mesos-assigned framework ID. Frameworks are expected to - // track this ID (that comes from Mesos, in a SUBSCRIBED event). - FrameworkID() string - - // Error is an error handler that is invoked at the end of every subscription cycle; the given - // error may be nil (if no errors occurred). - Error(error) + // Config is a controller configuration. Public fields are REQUIRED. Optional properties are + // configured by applying Option funcs. + Config struct { + doneFunc func() bool + frameworkIDFunc func() string + handler events.Handler + registrationTokens <-chan struct{} + subscriptionTerminated func(error) } +) - ContextAdapter struct { - // FrameworkIDFunc is optional; nil tells the controller to always register as a new framework - // for each subscription attempt. - FrameworkIDFunc func() string - - // Done is optional; nil equates to a func that always returns false - DoneFunc func() bool - - // ErrorFunc is optional; if nil then errors are swallowed - ErrorFunc func(error) +// WithEventHandler sets the consumer of scheduler events. The controller's internal event processing +// loop is aborted if a Handler returns a non-nil error, after which the controller may attempt +// to re-register (subscribe) with Mesos. +func WithEventHandler(handler events.Handler, ds ...events.Decorator) Option { + return func(c *Config) Option { + old := c.handler + c.handler = events.Decorators(ds).Apply(handler) + return WithEventHandler(old) } +} - Config struct { - Context Context // Context is required - Framework *mesos.FrameworkInfo // FrameworkInfo is required - Caller calls.Caller // Caller is required - - // Handler (optional) processes scheduler events. The controller's internal event processing - // loop is aborted if a Handler returns a non-nil error, after which the controller may attempt - // to re-register (subscribe) with Mesos. - Handler events.Handler - - // RegistrationTokens (optional) limits the rate at which a framework (re)registers with Mesos. - // The chan should either be non-blocking, or should yield a struct{} in order to allow the - // framework registration process to continue. May be nil. - RegistrationTokens <-chan struct{} +// WithFrameworkID sets a fetcher for the current Mesos-assigned framework ID. Frameworks are expected to +// track this ID (that comes from Mesos, in a SUBSCRIBED event). +// frameworkIDFunc is optional; nil tells the controller to always register as a new framework +// for each subscription attempt. +func WithFrameworkID(frameworkIDFunc func() string) Option { + return func(c *Config) Option { + old := c.frameworkIDFunc + c.frameworkIDFunc = frameworkIDFunc + return WithFrameworkID(old) } +} - Controller interface { - // Run executes the controller using the given Config - Run(Config) error +// WithDone sets a fetcher func that returns true when the controller should exit. +// doneFunc is optional; nil equates to a func that always returns false. +func WithDone(doneFunc func() bool) Option { + return func(c *Config) Option { + old := c.doneFunc + c.doneFunc = doneFunc + return WithDone(old) } +} - // ControllerFunc is a functional adaptation of a Controller - ControllerFunc func(Config) error - - controllerImpl int -) +// WithSubscriptionTerminated sets a handler that is invoked at the end of every subscription cycle; the +// given error may be nil if no error occurred. subscriptionTerminated is optional; if nil then errors are +// swallowed. +func WithSubscriptionTerminated(handler func(error)) Option { + return func(c *Config) Option { + old := c.subscriptionTerminated + c.subscriptionTerminated = handler + return WithSubscriptionTerminated(old) + } +} -// Run implements Controller for ControllerFunc -func (cf ControllerFunc) Run(config Config) error { return cf(config) } +// WithRegistrationTokens limits the rate at which a framework (re)registers with Mesos. +// The chan should either be non-blocking, or should yield a struct{} in order to allow the +// framework registration process to continue. May be nil. +func WithRegistrationTokens(registrationTokens <-chan struct{}) Option { + return func(c *Config) Option { + old := c.registrationTokens + c.registrationTokens = registrationTokens + return WithRegistrationTokens(old) + } +} -func New() Controller { - return new(controllerImpl) +func (c *Config) tryFrameworkID() (result string) { + if c.frameworkIDFunc != nil { + result = c.frameworkIDFunc() + } + return } +func (c *Config) tryDone() (result bool) { return c.doneFunc != nil && c.doneFunc() } + // Run executes a control loop that registers a framework with Mesos and processes the scheduler events -// that flow through the subscription. Upon disconnection, if the given Context reports !Done() then the -// controller will attempt to re-register the framework and continue processing events. -func (_ *controllerImpl) Run(config Config) (lastErr error) { - subscribe := calls.Subscribe(config.Framework) - for !config.Context.Done() { - frameworkID := config.Context.FrameworkID() - if config.Framework.GetFailoverTimeout() > 0 && frameworkID != "" { +// that flow through the subscription. Upon disconnection, if the current configuration reports "not done" +// then the controller will attempt to re-register the framework and continue processing events. +func Run(framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) (lastErr error) { + var config Config + for _, opt := range options { + if opt != nil { + opt(&config) + } + } + if config.handler == nil { + config.handler = DefaultHandler + } + subscribe := calls.Subscribe(framework) + for !config.tryDone() { + frameworkID := config.tryFrameworkID() + if framework.GetFailoverTimeout() > 0 && frameworkID != "" { subscribe.With(calls.SubscribeTo(frameworkID)) } - if config.RegistrationTokens != nil { - <-config.RegistrationTokens + if config.registrationTokens != nil { + <-config.registrationTokens } - resp, err := config.Caller.Call(subscribe) + resp, err := caller.Call(subscribe) lastErr = processSubscription(config, resp, err) - config.Context.Error(lastErr) + if config.subscriptionTerminated != nil { + config.subscriptionTerminated(lastErr) + } } return } @@ -101,50 +131,14 @@ func processSubscription(config Config, resp mesos.Response, err error) error { // eventLoop returns the framework ID received by mesos (if any); callers should check for a // framework ID regardless of whether error != nil. func eventLoop(config Config, eventDecoder encoding.Decoder) (err error) { - h := config.Handler - if h == nil { - h = events.HandlerFunc(DefaultHandler) - } - for err == nil && !config.Context.Done() { + for err == nil && !config.tryDone() { var e scheduler.Event if err = eventDecoder.Decode(&e); err == nil { - err = h.HandleEvent(&e) + err = config.handler.HandleEvent(&e) } } return err } -var _ = Context(&ContextAdapter{}) // ContextAdapter implements Context - -func (ca *ContextAdapter) Done() bool { - return ca.DoneFunc != nil && ca.DoneFunc() -} -func (ca *ContextAdapter) FrameworkID() (id string) { - if ca.FrameworkIDFunc != nil { - id = ca.FrameworkIDFunc() - } - return -} -func (ca *ContextAdapter) Error(err error) { - if ca.ErrorFunc != nil { - ca.ErrorFunc(err) - } -} - -// ErrEvent errors are generated by the DefaultHandler upon receiving an ERROR event from Mesos. -type ErrEvent string - -func (e ErrEvent) Error() string { - return string(e) -} - -// DefaultHandler provides the minimum implementation required for correct controller behavior. -func DefaultHandler(e *scheduler.Event) (err error) { - if e.GetType() == scheduler.Event_ERROR { - // it's recommended that we abort and re-try subscribing; returning an - // error here will cause the event loop to terminate and the connection - // will be reset. - err = ErrEvent(e.GetError().GetMessage()) - } - return -} +// DefaultHandler defaults to events.NoopHandler +var DefaultHandler = events.NoopHandler() diff --git a/api/v1/lib/extras/store/singleton.go b/api/v1/lib/extras/store/singleton.go index 540ad857..fab5bc3b 100644 --- a/api/v1/lib/extras/store/singleton.go +++ b/api/v1/lib/extras/store/singleton.go @@ -1,13 +1,16 @@ package store -import "sync/atomic" +import ( + "errors" + "sync/atomic" +) type ( Getter interface { - Get() string + Get() (string, error) } - GetFunc func() string + GetFunc func() (string, error) Setter interface { Set(string) error @@ -25,20 +28,26 @@ type ( GetFunc SetFunc } + + SingletonDecorator interface { + Decorate(Singleton) Singleton + } ) -func (f GetFunc) Get() string { return f() } -func (f SetFunc) Set(s string) error { return f(s) } +func (f GetFunc) Get() (string, error) { return f() } +func (f SetFunc) Set(s string) error { return f(s) } + +var ErrNotFound = errors.New("value not found in store") func NewInMemorySingleton() Singleton { var value atomic.Value return &SingletonAdapter{ - func() string { + func() (string, error) { x := value.Load() if x == nil { - return "" + return "", ErrNotFound } - return x.(string) + return x.(string), nil }, func(s string) error { value.Store(s) @@ -46,3 +55,103 @@ func NewInMemorySingleton() Singleton { }, } } + +type ( + GetFuncDecorator func(Getter, string, error) (string, error) + SetFuncDecorator func(Setter, string, error) error +) + +func DoSet() SetFuncDecorator { + return func(s Setter, v string, _ error) error { + return s.Set(v) + } +} + +func (f SetFuncDecorator) AndThen(f2 SetFuncDecorator) SetFuncDecorator { + return func(s Setter, v string, err error) error { + err = f(s, v, err) + if err != nil { + return err + } + return f2(s, v, nil) + } +} + +func (f SetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + s.Get, + SetFunc(func(v string) error { + return f(s, v, nil) + }), + } +} + +func DoGet() GetFuncDecorator { + return func(s Getter, _ string, _ error) (string, error) { + return s.Get() + } +} + +func (f GetFuncDecorator) AndThen(f2 GetFuncDecorator) GetFuncDecorator { + return func(s Getter, v string, err error) (string, error) { + v, err = f(s, v, err) + if err != nil { + return v, err + } + return f2(s, v, nil) + } +} + +func (f GetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + GetFunc(func() (string, error) { + return f(s, "", nil) + }), + s.Set, + } +} + +func DecorateSingleton(s Singleton, ds ...SingletonDecorator) Singleton { + for _, d := range ds { + if d != nil { + s = d.Decorate(s) + } + } + return s +} + +// GetOrPanic curries the result of a Getter invocation: the returned func only ever returns the string component when +// the error component of the underlying Get() call is nil. If Get() generates an error then the curried func panics. +func GetOrPanic(g Getter) func() string { + return func() string { + v, err := g.Get() + if err != nil { + panic(err) + } + return v + } +} + +func GetIgnoreErrors(g Getter) func() string { + return func() string { + v, _ := g.Get() + return v + } +} + +// SetOrPanic curries the result of a Setter invocation: the returned func only ever returns normally when the error +// component of the underlying Set() call is nil. If Set() generates an error then the curried func panics. +func SetOrPanic(s Setter) func(v string) { + return func(v string) { + err := s.Set(v) + if err != nil { + panic(err) + } + } +} From 4d845931b4a0bd5709519dbc6744436f28fe8aba Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 21:17:49 +0000 Subject: [PATCH 23/67] move builders.go to api/v1/lib/extras/resources --- api/v1/cmd/example-scheduler/app/state.go | 9 +-- api/v1/cmd/msh/msh.go | 5 +- api/v1/lib/builders.go | 76 --------------------- api/v1/lib/extras/resources/builders.go | 80 +++++++++++++++++++++++ 4 files changed, 88 insertions(+), 82 deletions(-) delete mode 100644 api/v1/lib/builders.go create mode 100644 api/v1/lib/extras/resources/builders.go diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index de768e88..96c05edc 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -14,6 +14,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" "github.com/mesos/mesos-go/api/v1/lib/extras/latch" + "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" @@ -87,8 +88,8 @@ func prepareExecutorInfo( func buildWantsTaskResources(config Config) (r mesos.Resources) { r.Add( - mesos.CPUs(config.taskCPU).Resource, - mesos.Memory(config.taskMemory).Resource, + resources.CPUs(config.taskCPU).Resource, + resources.Memory(config.taskMemory).Resource, ) log.Println("wants-task-resources = " + r.String()) return @@ -96,8 +97,8 @@ func buildWantsTaskResources(config Config) (r mesos.Resources) { func buildWantsExecutorResources(config Config) (r mesos.Resources) { r.Add( - mesos.CPUs(config.execCPU).Resource, - mesos.Memory(config.execMemory).Resource, + resources.CPUs(config.execCPU).Resource, + resources.Memory(config.execMemory).Resource, ) log.Println("wants-executor-resources = " + r.String()) return diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index f8856701..e06a0eaf 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -21,6 +21,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/extras/offers" + "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" "github.com/mesos/mesos-go/api/v1/lib/extras/store" @@ -76,8 +77,8 @@ func main() { } wantsResources = mesos.Resources{ - mesos.CPUs(CPUs).Resource, - mesos.Memory(Memory).Resource, + resources.CPUs(CPUs).Resource, + resources.Memory(Memory).Resource, } taskPrototype = mesos.TaskInfo{ Name: TaskName, diff --git a/api/v1/lib/builders.go b/api/v1/lib/builders.go deleted file mode 100644 index 2d5ba6ab..00000000 --- a/api/v1/lib/builders.go +++ /dev/null @@ -1,76 +0,0 @@ -package mesos - -type ( - // ResourceBuilder simplifies construction of Resource objects - ResourceBuilder struct{ Resource } - // RangeBuilder simplifies construction of Range objects - RangeBuilder struct{ Ranges } -) - -func CPUs(value float64) *ResourceBuilder { - return BuildResource().Name("cpus").Scalar(value) -} - -func Memory(value float64) *ResourceBuilder { - return BuildResource().Name("mem").Scalar(value) -} - -func Disk(value float64) *ResourceBuilder { - return BuildResource().Name("disk").Scalar(value) -} - -func GPUs(value uint) *ResourceBuilder { - return BuildResource().Name("gpus").Scalar(float64(value)) -} - -func BuildRanges() *RangeBuilder { - return &RangeBuilder{Ranges: Ranges(nil)} -} - -// Span is a functional option for Ranges, defines the begin and end points of a -// continuous span within a range -func (rb *RangeBuilder) Span(bp, ep uint64) *RangeBuilder { - rb.Ranges = append(rb.Ranges, Value_Range{Begin: bp, End: ep}) - return rb -} - -func BuildResource() *ResourceBuilder { - return &ResourceBuilder{} -} -func (rb *ResourceBuilder) Name(name string) *ResourceBuilder { - rb.Resource.Name = name - return rb -} -func (rb *ResourceBuilder) Role(role string) *ResourceBuilder { - rb.Resource.Role = &role - return rb -} -func (rb *ResourceBuilder) Scalar(x float64) *ResourceBuilder { - rb.Resource.Type = SCALAR.Enum() - rb.Resource.Scalar = &Value_Scalar{Value: x} - return rb -} -func (rb *ResourceBuilder) Set(x ...string) *ResourceBuilder { - rb.Resource.Type = SET.Enum() - rb.Resource.Set = &Value_Set{Item: x} - return rb -} -func (rb *ResourceBuilder) Ranges(rs Ranges) *ResourceBuilder { - rb.Resource.Type = RANGES.Enum() - rb.Resource.Ranges = rb.Resource.Ranges.Add(&Value_Ranges{Range: rs}) - return rb -} -func (rb *ResourceBuilder) Disk(persistenceID, containerPath string) *ResourceBuilder { - rb.Resource.Disk = &Resource_DiskInfo{} - if containerPath != "" { - rb.Resource.Disk.Volume = &Volume{ContainerPath: containerPath} - } - if persistenceID != "" { - rb.Resource.Disk.Persistence = &Resource_DiskInfo_Persistence{ID: persistenceID} - } - return rb -} -func (rb *ResourceBuilder) Revocable() *ResourceBuilder { - rb.Resource.Revocable = &Resource_RevocableInfo{} - return rb -} diff --git a/api/v1/lib/extras/resources/builders.go b/api/v1/lib/extras/resources/builders.go new file mode 100644 index 00000000..57ce8400 --- /dev/null +++ b/api/v1/lib/extras/resources/builders.go @@ -0,0 +1,80 @@ +package resources + +import ( + "github.com/mesos/mesos-go/api/v1/lib" +) + +type ( + // Builder simplifies construction of Resource objects + Builder struct{ mesos.Resource } + // RangeBuilder simplifies construction of Range objects + RangeBuilder struct{ mesos.Ranges } +) + +func CPUs(value float64) *Builder { + return Build().Name("cpus").Scalar(value) +} + +func Memory(value float64) *Builder { + return Build().Name("mem").Scalar(value) +} + +func Disk(value float64) *Builder { + return Build().Name("disk").Scalar(value) +} + +func GPUs(value uint) *Builder { + return Build().Name("gpus").Scalar(float64(value)) +} + +func BuildRanges() *RangeBuilder { + return &RangeBuilder{Ranges: mesos.Ranges(nil)} +} + +// Span is a functional option for Ranges, defines the begin and end points of a +// continuous span within a range +func (rb *RangeBuilder) Span(bp, ep uint64) *RangeBuilder { + rb.Ranges = append(rb.Ranges, mesos.Value_Range{Begin: bp, End: ep}) + return rb +} + +func Build() *Builder { + return &Builder{} +} +func (rb *Builder) Name(name string) *Builder { + rb.Resource.Name = name + return rb +} +func (rb *Builder) Role(role string) *Builder { + rb.Resource.Role = &role + return rb +} +func (rb *Builder) Scalar(x float64) *Builder { + rb.Resource.Type = mesos.SCALAR.Enum() + rb.Resource.Scalar = &mesos.Value_Scalar{Value: x} + return rb +} +func (rb *Builder) Set(x ...string) *Builder { + rb.Resource.Type = mesos.SET.Enum() + rb.Resource.Set = &mesos.Value_Set{Item: x} + return rb +} +func (rb *Builder) Ranges(rs mesos.Ranges) *Builder { + rb.Resource.Type = mesos.RANGES.Enum() + rb.Resource.Ranges = rb.Resource.Ranges.Add(&mesos.Value_Ranges{Range: rs}) + return rb +} +func (rb *Builder) Disk(persistenceID, containerPath string) *Builder { + rb.Resource.Disk = &mesos.Resource_DiskInfo{} + if containerPath != "" { + rb.Resource.Disk.Volume = &mesos.Volume{ContainerPath: containerPath} + } + if persistenceID != "" { + rb.Resource.Disk.Persistence = &mesos.Resource_DiskInfo_Persistence{ID: persistenceID} + } + return rb +} +func (rb *Builder) Revocable() *Builder { + rb.Resource.Revocable = &mesos.Resource_RevocableInfo{} + return rb +} From 65c6dea85ba36dd937cf8ef8567dd60fcaf52167 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 21:19:35 +0000 Subject: [PATCH 24/67] controller: fix Config docs --- api/v1/lib/extras/scheduler/controller/controller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index 03a732e5..e56aafc9 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -12,8 +12,7 @@ type ( // Option modifies a Config, returns an Option that acts as an "undo" Option func(*Config) Option - // Config is a controller configuration. Public fields are REQUIRED. Optional properties are - // configured by applying Option funcs. + // Config is an opaque controller configuration. Properties are configured by applying Option funcs. Config struct { doneFunc func() bool frameworkIDFunc func() string From 0bd5000ca399e2230aa512863975f8f7128ec3d9 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 21:28:28 +0000 Subject: [PATCH 25/67] events: remove redundant AcknowledgeUpdates implementation --- .../lib/extras/scheduler/controller/rules.go | 9 +++-- api/v1/lib/scheduler/calls/errors.go | 13 ++++++++ api/v1/lib/scheduler/events/events.go | 33 ------------------- 3 files changed, 19 insertions(+), 36 deletions(-) create mode 100644 api/v1/lib/scheduler/calls/errors.go diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go index a4ff8f86..6f3e0609 100644 --- a/api/v1/lib/extras/scheduler/controller/rules.go +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -9,7 +9,6 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" - "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" ) // ErrEvent errors are generated by LiftErrors upon receiving an ERROR event from Mesos. @@ -80,7 +79,7 @@ func AckStatusUpdates(caller calls.Caller) Rule { } // AckStatusUpdatesF is a functional adapter for AckStatusUpdates, useful for cases where the caller may -// change over time. +// change over time. An error that occurs while ack'ing the status update is returned as a calls.AckError. func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { // aggressively attempt to ack updates: even if there's pre-existing error state attempt @@ -100,7 +99,11 @@ func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { ) err = calls.CallNoData(callerLookup(), ack) if err != nil { - err = &events.AckError{Ack: ack, Cause: err} + // TODO(jdef): not sure how important this is; if there's an error ack'ing + // because we beacame disconnected, then we'll just reconnect later and + // Mesos will ask us to ACK anyway -- why pay special attention to these + // call failures vs others? + err = &calls.AckError{Ack: ack, Cause: err} return nil, Error2(origErr, err) // drop } } diff --git a/api/v1/lib/scheduler/calls/errors.go b/api/v1/lib/scheduler/calls/errors.go new file mode 100644 index 00000000..3d32635d --- /dev/null +++ b/api/v1/lib/scheduler/calls/errors.go @@ -0,0 +1,13 @@ +package calls + +import ( + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// AckError wraps a caller-generated error and tracks the call that failed. +type AckError struct { + Ack *scheduler.Call + Cause error +} + +func (err *AckError) Error() string { return err.Cause.Error() } diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index 7366280c..951f07c4 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -4,7 +4,6 @@ import ( "sync" "github.com/mesos/mesos-go/api/v1/lib/scheduler" - "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" ) type ( @@ -159,38 +158,6 @@ func DefaultHandler(eh Handler) Option { } } -// AckError wraps a caller-generated error and tracks the call that failed. -type AckError struct { - Ack *scheduler.Call - Cause error -} - -func (err *AckError) Error() string { return err.Cause.Error() } - -// AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every -// UPDATE event that's received (that requests an ACK). -func AcknowledgeUpdates(callerLookup func() calls.Caller) Handler { - return WhenFunc(scheduler.Event_UPDATE, func(e *scheduler.Event) (err error) { - var ( - s = e.GetUpdate().GetStatus() - uuid = s.GetUUID() - ) - // only ACK non-empty UUID's, as per mesos scheduler spec - if len(uuid) > 0 { - ack := calls.Acknowledge( - s.GetAgentID().GetValue(), - s.TaskID.Value, - uuid, - ) - err = calls.CallNoData(callerLookup(), ack) - if err != nil { - err = &AckError{ack, err} - } - } - return - }) -} - // When // Deprecated in favor of Rules. func Once(h Handler) Handler { From 1f5c449fe2bd79407b6f776fe5f891fa9e406e5e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 21:31:29 +0000 Subject: [PATCH 26/67] readme: minor doc tweaks --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dc1a20b4..78f8735e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Go bindings for Apache Mesos -Very early version of a pure Go language bindings for Apache Mesos. +Pure Go language bindings for Apache Mesos, under development. As with other pure implementations, mesos-go uses the HTTP wire protocol to communicate directly with a running Mesos master and its slave instances. One of the objectives of this project is to provide an idiomatic Go API that makes it super easy to create Mesos frameworks using Go. @@ -12,6 +12,8 @@ One of the objectives of this project is to provide an idiomatic Go API that mak New projects should use the Mesos v1 API bindings, located in `api/v1`. Unless otherwise indicated, the remainder of this README describes the Mesos v1 API implementation. +Please **vendor** this library to avoid unpleasant surprises via `go get ...`. + The Mesos v0 API version of the bindings, located in `api/v0`, are more mature but will not see any major development besides critical compatibility and bug fixes. ### Compatibility From 7d7d1ffc3298370ca88f0c6e43db8448d06060d6 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 21:34:44 +0000 Subject: [PATCH 27/67] offers: from api/v1/lib/extras/offers/ to api/v1/lib/extras/scheduler/offers/ --- api/v1/cmd/msh/msh.go | 2 +- api/v1/lib/extras/{ => scheduler}/offers/filters.go | 0 api/v1/lib/extras/{ => scheduler}/offers/offers.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename api/v1/lib/extras/{ => scheduler}/offers/filters.go (100%) rename api/v1/lib/extras/{ => scheduler}/offers/offers.go (100%) diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index e06a0eaf..9099aafb 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -20,10 +20,10 @@ import ( "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/extras/latch" - "github.com/mesos/mesos-go/api/v1/lib/extras/offers" "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/offers" "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" diff --git a/api/v1/lib/extras/offers/filters.go b/api/v1/lib/extras/scheduler/offers/filters.go similarity index 100% rename from api/v1/lib/extras/offers/filters.go rename to api/v1/lib/extras/scheduler/offers/filters.go diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/scheduler/offers/offers.go similarity index 100% rename from api/v1/lib/extras/offers/offers.go rename to api/v1/lib/extras/scheduler/offers/offers.go From f86cb7419c930a797de4e7dcc84d3a362bb305e4 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 25 May 2017 23:53:26 +0000 Subject: [PATCH 28/67] example-scheduler: refactor failure into HandlerFunc to clarify event handler builder --- api/v1/cmd/example-scheduler/app/app.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index d21953f3..c0729944 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -92,11 +92,7 @@ func Run(cfg Config) error { func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { logger := controller.LogEvents() return controller.LiftErrors().Handle(events.HandlerSet{ - scheduler.Event_FAILURE: logger.HandleF(func(e *scheduler.Event) error { - f := e.GetFailure() - failure(f.ExecutorID, f.AgentID, f.Status) - return nil - }), + scheduler.Event_FAILURE: logger.HandleF(failure), scheduler.Event_OFFERS: trackOffersReceived(state).AndThen().HandleF( func(e *scheduler.Event) error { if state.config.verbose { @@ -122,7 +118,11 @@ func trackOffersReceived(state *internalState) eventrules.Rule { } } -func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { +func failure(e *scheduler.Event) error { + var ( + f = e.GetFailure() + eid, aid, stat = f.ExecutorID, f.AgentID, f.Status + ) if eid != nil { // executor failed.. msg := "executor '" + eid.Value + "' terminated" @@ -137,6 +137,7 @@ func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { // agent failed.. log.Println("agent '" + aid.Value + "' terminated") } + return nil } func resourceOffers(state *internalState, offers []mesos.Offer) error { From d2368b058a5b5249d339488ebedccb1646e56764 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 May 2017 00:58:04 +0000 Subject: [PATCH 29/67] rules: add If and Unless chaining funcs; added Concat convenience func --- api/v1/lib/extras/rules/rules.go | 28 ++++++++++++++++--- .../eventrules/eventrules_generated.go | 28 ++++++++++++++++--- .../extras/scheduler/eventrules/handlers.go | 2 +- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go index 00587709..4d99cbbc 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/rules/rules.go @@ -101,6 +101,7 @@ type ( // Rule implementations should not modify the given event parameter (to avoid side effects). // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. + // A nil Rule is valid: it is processed as a noop. Rule func({{.EventType}}, error, Chain) ({{.EventType}}, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, @@ -132,8 +133,8 @@ func (rs Rules) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, err return ch(rs[0](e, err, NewChain(rs))) } -// Rule adapts Rules to the Rule interface, for convenient call chaining. -func (rs Rules) Rule() Rule { return rs.Eval } +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { @@ -201,7 +202,23 @@ func Error(es ...error) error { return result.Err() } -// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} // Once returns a Rule that executes the receiver only once. func (r Rule) Once() Rule { @@ -217,10 +234,13 @@ func (r Rule) Once() Rule { // Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. // A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. func (r Rule) Poll(p <-chan struct{}) Rule { + // TODO(jdef): optimize for the case where p is nil (it always drops the events) return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { select { case <-p: // do something + // TODO(jdef): optimization: if we detect the chan is closed, affect a state change + // whereby this select is no longer invoked (and always pass control to r). return r(e, err, ch) default: // drop @@ -230,7 +250,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this is a noop. +// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). func (r Rule) EveryN(nthTime int) Rule { if nthTime < 2 { return r diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index bb4dd636..cf855222 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -16,6 +16,7 @@ type ( // Rule implementations should not modify the given event parameter (to avoid side effects). // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. + // A nil Rule is valid: it is processed as a noop. Rule func(*scheduler.Event, error, Chain) (*scheduler.Event, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, @@ -47,8 +48,8 @@ func (rs Rules) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, return ch(rs[0](e, err, NewChain(rs))) } -// Rule adapts Rules to the Rule interface, for convenient call chaining. -func (rs Rules) Rule() Rule { return rs.Eval } +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { @@ -116,7 +117,23 @@ func Error(es ...error) error { return result.Err() } -// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} // Once returns a Rule that executes the receiver only once. func (r Rule) Once() Rule { @@ -132,10 +149,13 @@ func (r Rule) Once() Rule { // Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. // A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. func (r Rule) Poll(p <-chan struct{}) Rule { + // TODO(jdef): optimize for the case where p is nil (it always drops the events) return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { select { case <-p: // do something + // TODO(jdef): optimization: if we detect the chan is closed, affect a state change + // whereby this select is no longer invoked (and always pass control to r). return r(e, err, ch) default: // drop @@ -145,7 +165,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this is a noop. +// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). func (r Rule) EveryN(nthTime int) Rule { if nthTime < 2 { return r diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers.go index 16a08b8d..4508a1ac 100644 --- a/api/v1/lib/extras/scheduler/eventrules/handlers.go +++ b/api/v1/lib/extras/scheduler/eventrules/handlers.go @@ -42,7 +42,7 @@ func (r Rule) HandleEvent(e *scheduler.Event) (err error) { // HandleEvent implements events.Handler for Rules func (rs Rules) HandleEvent(e *scheduler.Event) error { - return rs.Rule().HandleEvent(e) + return Rule(rs.Eval).HandleEvent(e) } /* From 9ea5be43a3163fbac27460fa04f74e64eeeec81f Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 May 2017 00:59:31 +0000 Subject: [PATCH 30/67] example-scheduler: tidy up event handler builder --- api/v1/cmd/example-scheduler/app/app.go | 135 ++++++++++++------------ 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index c0729944..36626b22 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -91,15 +91,12 @@ func Run(cfg Config) error { // buildEventHandler generates and returns a handler to process events received from the subscription. func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { logger := controller.LogEvents() - return controller.LiftErrors().Handle(events.HandlerSet{ + return controller.LiftErrors().DropOnError().Handle(events.HandlerSet{ scheduler.Event_FAILURE: logger.HandleF(failure), - scheduler.Event_OFFERS: trackOffersReceived(state).AndThen().HandleF( - func(e *scheduler.Event) error { - if state.config.verbose { - log.Println("received an OFFERS event") - } - return resourceOffers(state, e.GetOffers().GetOffers()) - }), + scheduler.Event_OFFERS: eventrules.Concat( + trackOffersReceived(state), + logger.If(state.config.verbose), + ).HandleF(resourceOffers(state)), scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), scheduler.Event_SUBSCRIBED: eventrules.Rules{ logger, @@ -140,83 +137,89 @@ func failure(e *scheduler.Event) error { return nil } -func resourceOffers(state *internalState, offers []mesos.Offer) error { - callOption := calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) - tasksLaunchedThisCycle := 0 - offersDeclined := 0 - for i := range offers { +func resourceOffers(state *internalState) events.HandlerFunc { + return func(e *scheduler.Event) error { var ( - remaining = mesos.Resources(offers[i].Resources) - tasks = []mesos.TaskInfo{} + offers = e.GetOffers().GetOffers() + callOption = calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) + tasksLaunchedThisCycle = 0 + offersDeclined = 0 ) + for i := range offers { + var ( + remaining = mesos.Resources(offers[i].Resources) + tasks = []mesos.TaskInfo{} + ) - if state.config.verbose { - log.Println("received offer id '" + offers[i].ID.Value + "' with resources " + remaining.String()) - } + if state.config.verbose { + log.Println("received offer id '" + offers[i].ID.Value + + "' with resources " + remaining.String()) + } - var wantsExecutorResources mesos.Resources - if len(offers[i].ExecutorIDs) == 0 { - wantsExecutorResources = mesos.Resources(state.executor.Resources) - } + var wantsExecutorResources mesos.Resources + if len(offers[i].ExecutorIDs) == 0 { + wantsExecutorResources = mesos.Resources(state.executor.Resources) + } - flattened := remaining.Flatten() + flattened := remaining.Flatten() - // avoid the expense of computing these if we can... - if state.config.summaryMetrics && state.config.resourceTypeMetrics { - for name, restype := range flattened.Types() { - if restype == mesos.SCALAR { - sum := flattened.SumScalars(mesos.NamedResources(name)) - state.metricsAPI.offeredResources(sum.GetValue(), name) + // avoid the expense of computing these if we can... + if state.config.summaryMetrics && state.config.resourceTypeMetrics { + for name, restype := range flattened.Types() { + if restype == mesos.SCALAR { + sum := flattened.SumScalars(mesos.NamedResources(name)) + state.metricsAPI.offeredResources(sum.GetValue(), name) + } } } - } - taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...) - for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) { - state.tasksLaunched++ - taskID := state.tasksLaunched + taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...) + for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) { + state.tasksLaunched++ + taskID := state.tasksLaunched - if state.config.verbose { - log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value) - } + if state.config.verbose { + log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value) + } - task := mesos.TaskInfo{ - TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}, - AgentID: offers[i].AgentID, - Executor: state.executor, - Resources: remaining.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign())), - } - task.Name = "Task " + task.TaskID.Value + task := mesos.TaskInfo{ + TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}, + AgentID: offers[i].AgentID, + Executor: state.executor, + Resources: remaining.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign())), + } + task.Name = "Task " + task.TaskID.Value - remaining.Subtract(task.Resources...) - tasks = append(tasks, task) + remaining.Subtract(task.Resources...) + tasks = append(tasks, task) - flattened = remaining.Flatten() - } + flattened = remaining.Flatten() + } - // build Accept call to launch all of the tasks we've assembled - accept := calls.Accept( - calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID), - ).With(callOption) + // build Accept call to launch all of the tasks we've assembled + accept := calls.Accept( + calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID), + ).With(callOption) - // send Accept call to mesos - err := calls.CallNoData(state.cli, accept) - if err != nil { - log.Printf("failed to launch tasks: %+v", err) - } else { - if n := len(tasks); n > 0 { - tasksLaunchedThisCycle += n + // send Accept call to mesos + err := calls.CallNoData(state.cli, accept) + if err != nil { + log.Printf("failed to launch tasks: %+v", err) } else { - offersDeclined++ + if n := len(tasks); n > 0 { + tasksLaunchedThisCycle += n + } else { + offersDeclined++ + } } } + state.metricsAPI.offersDeclined.Int(offersDeclined) + state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle) + if state.config.summaryMetrics { + state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) + } + return nil } - state.metricsAPI.offersDeclined.Int(offersDeclined) - state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle) - if state.config.summaryMetrics { - state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) - } - return nil } func statusUpdate(state *internalState) events.HandlerFunc { From 0aef62e849573874e957e361c4e35c3792f9c624 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 May 2017 01:20:27 +0000 Subject: [PATCH 31/67] offers: extract default key func impl --- api/v1/lib/extras/scheduler/offers/offers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/v1/lib/extras/scheduler/offers/offers.go b/api/v1/lib/extras/scheduler/offers/offers.go index 75ccdfef..34c275e5 100644 --- a/api/v1/lib/extras/scheduler/offers/offers.go +++ b/api/v1/lib/extras/scheduler/offers/offers.go @@ -87,7 +87,9 @@ func (offers Slice) FilterNot(filter Filter) Slice { return offers.Filter(not(fi func (offers Index) FilterNot(filter Filter) Index { return offers.Filter(not(filter)) } // DefaultKeyFunc indexes offers by their OfferID. -var DefaultKeyFunc = KeyFunc(func(o *Offer) interface{} { return o.GetID() }) +var DefaultKeyFunc = KeyFunc(KeyFuncByOfferID) + +func KeyFuncByOfferID(o *Offer) interface{} { return o.GetID() } // NewIndex returns a new Index constructed from the list of mesos offers. // If the KeyFunc is nil then offers are indexed by DefaultKeyFunc. From 05dd4ad44fd92f0f532cc98856340462fe9e12b0 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 May 2017 01:06:02 +0000 Subject: [PATCH 32/67] ignore proto-generated files when calculating code coverage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b2876e1e..1ff65fc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ before_install: install: - make install script: - - rm -rf $HOME/gopath/pkg && make coverage && $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=_output/coverage.out + - rm -rf $HOME/gopath/pkg && make coverage && $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=_output/coverage.out -ignore=$((tr '\n' , | sed -e 's/$,//g') < <(find api/v1/cmd -type f -name '*.go'|grep -v -e /vendor/ ; ls api/v1/lib{,/scheduler,/executor}/*.pb{,_ffjson}.go ; find api/v0 -type d)) From 01fcc24e64cbf940e247b14b5a825acc93e55195 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 28 May 2017 14:22:00 +0000 Subject: [PATCH 33/67] rules: bugfix and unit tests --- api/v1/lib/extras/rules/rules.go | 553 +++++++++++++++--- .../eventrules/eventrules_generated.go | 177 +++--- .../eventrules/eventrules_generated_test.go | 363 +++++++++++- 3 files changed, 906 insertions(+), 187 deletions(-) diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go index 4d99cbbc..a899b78a 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/rules/rules.go @@ -13,9 +13,10 @@ import ( type ( config struct { - Package string - Imports []string - EventType string + Package string + Imports []string + EventType string + EventPrototype string } ) @@ -56,6 +57,12 @@ func main() { } if c.EventType == "" { c.EventType = "Event" + c.EventPrototype = "Event{}" + } else if strings.HasPrefix(c.EventType, "*") { + // TODO(jdef) don't assume that event type is a struct or *struct + c.EventPrototype = "&" + c.EventType[1:] + "{}" + } else { + c.EventPrototype = c.EventType[1:] + "{}" } if output == "" { output = defaultOutput @@ -96,12 +103,18 @@ import ( ) type ( - // Rule executes a filter, rule, or decorator function; if the returned event is nil then - // no additional Rule func should be processed for the event. - // Rule implementations should not modify the given event parameter (to avoid side effects). - // If changes to the event object are needed, the suggested approach is to make a copy, - // modify the copy, and pass the copy to the chain. - // A nil Rule is valid: it is processed as a noop. + iface interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval({{.EventType}}, error, Chain) ({{.EventType}}, error) + } + + // Rule is the functional adaptation of iface. + // A nil Rule is valid: it is Eval'd as a noop. Rule func({{.EventType}}, error, Chain) ({{.EventType}}, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, @@ -117,20 +130,39 @@ type ( ErrorList []error ) -// chainIdentity is a Chain that returns the arguments as its results. -var chainIdentity = func(e {{.EventType}}, err error) ({{.EventType}}, error) { - return e, err +var ( + _ = iface(Rule(nil)) + _ = iface(Rules{}) + + // chainIdentity is a Chain that returns the arguments as its results. + chainIdentity = func(e {{.EventType}}, err error) ({{.EventType}}, error) { + return e, err + } +) + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if r != nil { + return r(e, err, ch) + } + return ch(e, err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. func (rs Rules) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return ch(rs.Chain()(e, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the ({{.EventType}}, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { if len(rs) == 0 { - return ch(e, err) // noop + return chainIdentity + } + return func(e {{.EventType}}, err error) ({{.EventType}}, error) { + return rs[0].Eval(e, err, rs[1:].Chain()) } - // we know there's at least 1 rule in the initial list; start with it and let the chain - // handle the iteration. - return ch(rs[0](e, err, NewChain(rs))) } // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. @@ -138,14 +170,14 @@ func Concat(rs ...Rule) Rule { return Rules(rs).Eval } // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { - switch len(es) { - case 0: - return "no errors" - case 1: - return es[0].Error() - default: - return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) - } + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } } // Error2 aggregates the given error params, returning nil if both are nil. @@ -155,9 +187,15 @@ func Error2(a, b error) error { if b == nil { return nil } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } return b } if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } return a } return Error(a, b) @@ -183,26 +221,30 @@ func IsErrorList(err error) bool { return false } -// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Error aggregates, and then flattens, a list of errors accrued during rule processing. // Returns nil if the given list of errors is empty or contains all nil errors. func Error(es ...error) error { - var result ErrorList - for _, err := range es { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { if err != nil { if multi, ok := err.(ErrorList); ok { - // flatten nested error lists - if len(multi) > 0 { - result = append(result, multi...) - } + result = append(result, flatten(multi)...) } else { result = append(result, err) } } } - return result.Err() + return ErrorList(result) } -// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool), OrElse(...Rule) // If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. func (r Rule) If(b bool) Rule { @@ -222,19 +264,29 @@ func (r Rule) Unless(b bool) Rule { // Once returns a Rule that executes the receiver only once. func (r Rule) Once() Rule { + if r == nil { + return nil + } var once sync.Once return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + ruleInvoked := false once.Do(func() { e, err = r(e, err, ch) + ruleInvoked = true }) + if !ruleInvoked { + e, err = ch(e, err) + } return e, err } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. -// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. +// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. func (r Rule) Poll(p <-chan struct{}) Rule { - // TODO(jdef): optimize for the case where p is nil (it always drops the events) + if p == nil || r == nil { + return nil + } return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { select { case <-p: @@ -243,8 +295,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { // whereby this select is no longer invoked (and always pass control to r). return r(e, err, ch) default: - // drop - return ch(nil, err) + return ch(e, err) } } } @@ -252,7 +303,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). func (r Rule) EveryN(nthTime int) Rule { - if nthTime < 2 { + if nthTime < 2 || r == nil { return r } var ( @@ -274,8 +325,19 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(e, err, ch) } - // else, drop - return ch(nil, err) + return ch(e, err) + } +} + +// Drop aborts the Chain and returns the ({{.EventType}}, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the ({{.EventType}}, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(e {{.EventType}}, err error, _ Chain) ({{.EventType}}, error) { + return r.Eval(e, err, chainIdentity) } } @@ -285,49 +347,16 @@ func DropOnError() Rule { } // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then -// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving -// rule. +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { - if err != nil || e == nil { + if err != nil { return e, err } - if r != nil { - return r(e, err, ch) - } - return ch(e, err) + return r.Eval(e, err, ch) } } -// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing -// for any of the following cases: -// - there are no more rules to process -// - the event has been zero'ed out (nil) -// Any nil rules in the list are processed as skipped (noop's). -func NewChain(rs Rules) Chain { - sz := len(rs) - if sz == 0 { - return chainIdentity - } - var ( - i = 0 - chain Chain - ) - chain = Chain(func(x {{.EventType}}, y error) ({{.EventType}}, error) { - i++ - if i >= sz || x == nil { - // we're at the end, or DROP was issued (x==nil) - return x, y - } else if rs[i] != nil { - return rs[i](x, y, chain) - } else { - return chain(x, y) - } - - }) - return chain -} - // AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then // all of the rules specified by the next parameter. The net effect is: execute the receiver rule // and only if there is no error state, continue processing the next rules, in order. @@ -341,14 +370,11 @@ func DropOnSuccess() Rule { func (r Rule) DropOnSuccess() Rule { return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { - if e != nil && err == nil { + if err == nil { // bypass remainder of chain return e, err } - if r != nil { - return r(e, err, ch) - } - return ch(e, err) + return r.Eval(e, err, ch) } } @@ -371,6 +397,8 @@ import ( {{ end -}} ) +func prototype() {{.EventType}} { return {{.EventPrototype}} } + func counter(i *int) Rule { return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { *i++ @@ -378,12 +406,26 @@ func counter(i *int) Rule { } } +func tracer(r Rule, name string, t *testing.T) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + t.Log("executing", name) + return r(e, err, ch) + } +} + func returnError(re error) Rule { return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { return ch(e, Error2(err, re)) } } +func chainCounter(i *int, ch Chain) Chain { + return func(e {{.EventType}}, err error) ({{.EventType}}, error) { + *i++ + return ch(e, err) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) @@ -400,33 +442,354 @@ func TestChainIdentity(t *testing.T) { } } +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ) + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e {{.EventType}} + err error + }{ + {nil, nil}, + {nil, a}, + {p, nil}, + {p, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + e, err = rule(tc.e, tc.err, chainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, err + e, err = Rules{}.Eval(tc.e, tc.err, chainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") b = errors.New("b") ) for i, tc := range []struct { - a error - b error - wants error + a error + b error + wants error + wantsMessage string }{ - {nil, nil, nil}, - {a, nil, a}, - {nil, b, b}, - {a, b, ErrorList{a, b}}, + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, } { - result := Error2(tc.a, tc.b) + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) // jump through hoops because we can't directly compare two errors with == if // they're both ErrorList. if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither - if !IsErrorList(result) && result == tc.wants { - continue + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + e, err := r(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) } - if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { - continue + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestPoll(t *testing.T) { + var ( + ch1 <-chan struct{} // always nil + ch2 = make(chan struct{}) // non-nil, blocking + ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking + ch4 = make(chan struct{}) // non-nil, closed + ) + ch3 <- struct{}{} + close(ch4) + for ti, tc := range []struct { + ch <-chan struct{} + wantsRuleCount []int + }{ + {ch1, []int{0, 0, 0, 0}}, + {ch2, []int{0, 0, 0, 0}}, + {ch3, []int{1, 1, 1, 1}}, + {ch4, []int{1, 2, 2, 2}}, + } { + var ( + i, j int + p = prototype() + r1 = counter(&i).Poll(tc.ch).Eval + r2 = Rule(nil).Poll(tc.ch).Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 2; x++ { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != nil { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := (k * 2) + x + 1; j != y { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", + ti, y, j) + } } } - t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) } } `)) diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index cf855222..2f6a49f1 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -11,12 +11,18 @@ import ( ) type ( - // Rule executes a filter, rule, or decorator function; if the returned event is nil then - // no additional Rule func should be processed for the event. - // Rule implementations should not modify the given event parameter (to avoid side effects). - // If changes to the event object are needed, the suggested approach is to make a copy, - // modify the copy, and pass the copy to the chain. - // A nil Rule is valid: it is processed as a noop. + iface interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(*scheduler.Event, error, Chain) (*scheduler.Event, error) + } + + // Rule is the functional adaptation of iface. + // A nil Rule is valid: it is Eval'd as a noop. Rule func(*scheduler.Event, error, Chain) (*scheduler.Event, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, @@ -32,20 +38,39 @@ type ( ErrorList []error ) -// chainIdentity is a Chain that returns the arguments as its results. -var chainIdentity = func(e *scheduler.Event, err error) (*scheduler.Event, error) { - return e, err +var ( + _ = iface(Rule(nil)) + _ = iface(Rules{}) + + // chainIdentity is a Chain that returns the arguments as its results. + chainIdentity = func(e *scheduler.Event, err error) (*scheduler.Event, error) { + return e, err + } +) + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if r != nil { + return r(e, err, ch) + } + return ch(e, err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. func (rs Rules) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return ch(rs.Chain()(e, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (*scheduler.Event, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { if len(rs) == 0 { - return ch(e, err) // noop + return chainIdentity + } + return func(e *scheduler.Event, err error) (*scheduler.Event, error) { + return rs[0].Eval(e, err, rs[1:].Chain()) } - // we know there's at least 1 rule in the initial list; start with it and let the chain - // handle the iteration. - return ch(rs[0](e, err, NewChain(rs))) } // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. @@ -53,14 +78,14 @@ func Concat(rs ...Rule) Rule { return Rules(rs).Eval } // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { - switch len(es) { - case 0: - return "no errors" - case 1: - return es[0].Error() - default: - return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) - } + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } } // Error2 aggregates the given error params, returning nil if both are nil. @@ -70,9 +95,15 @@ func Error2(a, b error) error { if b == nil { return nil } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } return b } if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } return a } return Error(a, b) @@ -98,26 +129,30 @@ func IsErrorList(err error) bool { return false } -// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Error aggregates, and then flattens, a list of errors accrued during rule processing. // Returns nil if the given list of errors is empty or contains all nil errors. func Error(es ...error) error { - var result ErrorList - for _, err := range es { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { if err != nil { if multi, ok := err.(ErrorList); ok { - // flatten nested error lists - if len(multi) > 0 { - result = append(result, multi...) - } + result = append(result, flatten(multi)...) } else { result = append(result, err) } } } - return result.Err() + return ErrorList(result) } -// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool), OrElse(...Rule) // If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. func (r Rule) If(b bool) Rule { @@ -137,19 +172,29 @@ func (r Rule) Unless(b bool) Rule { // Once returns a Rule that executes the receiver only once. func (r Rule) Once() Rule { + if r == nil { + return nil + } var once sync.Once return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + ruleInvoked := false once.Do(func() { e, err = r(e, err, ch) + ruleInvoked = true }) + if !ruleInvoked { + e, err = ch(e, err) + } return e, err } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. -// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. +// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. func (r Rule) Poll(p <-chan struct{}) Rule { - // TODO(jdef): optimize for the case where p is nil (it always drops the events) + if p == nil || r == nil { + return nil + } return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { select { case <-p: @@ -158,8 +203,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { // whereby this select is no longer invoked (and always pass control to r). return r(e, err, ch) default: - // drop - return ch(nil, err) + return ch(e, err) } } } @@ -167,7 +211,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). func (r Rule) EveryN(nthTime int) Rule { - if nthTime < 2 { + if nthTime < 2 || r == nil { return r } var ( @@ -189,8 +233,19 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(e, err, ch) } - // else, drop - return ch(nil, err) + return ch(e, err) + } +} + +// Drop aborts the Chain and returns the (*scheduler.Event, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (*scheduler.Event, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(e *scheduler.Event, err error, _ Chain) (*scheduler.Event, error) { + return r.Eval(e, err, chainIdentity) } } @@ -200,49 +255,16 @@ func DropOnError() Rule { } // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then -// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving -// rule. +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { - if err != nil || e == nil { + if err != nil { return e, err } - if r != nil { - return r(e, err, ch) - } - return ch(e, err) + return r.Eval(e, err, ch) } } -// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing -// for any of the following cases: -// - there are no more rules to process -// - the event has been zero'ed out (nil) -// Any nil rules in the list are processed as skipped (noop's). -func NewChain(rs Rules) Chain { - sz := len(rs) - if sz == 0 { - return chainIdentity - } - var ( - i = 0 - chain Chain - ) - chain = Chain(func(x *scheduler.Event, y error) (*scheduler.Event, error) { - i++ - if i >= sz || x == nil { - // we're at the end, or DROP was issued (x==nil) - return x, y - } else if rs[i] != nil { - return rs[i](x, y, chain) - } else { - return chain(x, y) - } - - }) - return chain -} - // AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then // all of the rules specified by the next parameter. The net effect is: execute the receiver rule // and only if there is no error state, continue processing the next rules, in order. @@ -256,14 +278,11 @@ func DropOnSuccess() Rule { func (r Rule) DropOnSuccess() Rule { return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { - if e != nil && err == nil { + if err == nil { // bypass remainder of chain return e, err } - if r != nil { - return r(e, err, ch) - } - return ch(e, err) + return r.Eval(e, err, ch) } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index f5a6f1de..962c5f42 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -11,6 +11,8 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) +func prototype() *scheduler.Event { return &scheduler.Event{} } + func counter(i *int) Rule { return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { *i++ @@ -18,12 +20,26 @@ func counter(i *int) Rule { } } +func tracer(r Rule, name string, t *testing.T) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + t.Log("executing", name) + return r(e, err, ch) + } +} + func returnError(re error) Rule { return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { return ch(e, Error2(err, re)) } } +func chainCounter(i *int, ch Chain) Chain { + return func(e *scheduler.Event, err error) (*scheduler.Event, error) { + *i++ + return ch(e, err) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) @@ -40,32 +56,353 @@ func TestChainIdentity(t *testing.T) { } } +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ) + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *scheduler.Event + err error + }{ + {nil, nil}, + {nil, a}, + {p, nil}, + {p, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + e, err = rule(tc.e, tc.err, chainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, err + e, err = Rules{}.Eval(tc.e, tc.err, chainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") b = errors.New("b") ) for i, tc := range []struct { - a error - b error - wants error + a error + b error + wants error + wantsMessage string }{ - {nil, nil, nil}, - {a, nil, a}, - {nil, b, b}, - {a, b, ErrorList{a, b}}, + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, } { - result := Error2(tc.a, tc.b) + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) // jump through hoops because we can't directly compare two errors with == if // they're both ErrorList. if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither - if !IsErrorList(result) && result == tc.wants { - continue + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + e, err := r(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) } - if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { - continue + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestPoll(t *testing.T) { + var ( + ch1 <-chan struct{} // always nil + ch2 = make(chan struct{}) // non-nil, blocking + ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking + ch4 = make(chan struct{}) // non-nil, closed + ) + ch3 <- struct{}{} + close(ch4) + for ti, tc := range []struct { + ch <-chan struct{} + wantsRuleCount []int + }{ + {ch1, []int{0, 0, 0, 0}}, + {ch2, []int{0, 0, 0, 0}}, + {ch3, []int{1, 1, 1, 1}}, + {ch4, []int{1, 2, 2, 2}}, + } { + var ( + i, j int + p = prototype() + r1 = counter(&i).Poll(tc.ch).Eval + r2 = Rule(nil).Poll(tc.ch).Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 2; x++ { + e, err := r(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != nil { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := (k * 2) + x + 1; j != y { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", + ti, y, j) + } } } - t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) } } From b0413bac611662a775ebcbf0edede67e4267f21e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 29 May 2017 21:09:41 +0000 Subject: [PATCH 34/67] cmd: fix rule processing examples --- api/v1/cmd/example-scheduler/app/app.go | 3 +-- api/v1/cmd/msh/msh.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 36626b22..dcb75c0f 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -110,8 +110,7 @@ func trackOffersReceived(state *internalState) eventrules.Rule { if err == nil { state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers())) } - return chain(e, nil) - + return chain(e, err) } } diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 9099aafb..d20fe570 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -156,7 +156,7 @@ func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { // we shouldn't have received offers, maybe the prior suppress call failed? err = calls.CallNoData(caller, calls.Suppress()) } - return nil, err // drop + return e, err // drop } } From 1e4237316815d35064776058db19b33a350de606 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 30 May 2017 11:05:30 +0000 Subject: [PATCH 35/67] rules: added Fail rule; additional unit testing --- api/v1/lib/extras/rules/rules.go | 81 ++++++++++++++++++- .../eventrules/eventrules_generated.go | 9 ++- .../eventrules/eventrules_generated_test.go | 72 ++++++++++++++++- 3 files changed, 156 insertions(+), 6 deletions(-) diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go index a899b78a..8303e991 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/rules/rules.go @@ -244,7 +244,7 @@ func flatten(errors []error) ErrorList { return ErrorList(result) } -// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool), OrElse(...Rule) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) // If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. func (r Rule) If(b bool) Rule { @@ -341,6 +341,13 @@ func (r Rule) ThenDrop() Rule { } } +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return ch(e, Error2(err, injected)) + } +} + // DropOnError returns a Rule that generates a nil event if the error state != nil func DropOnError() Rule { return Rule(nil).DropOnError() @@ -542,8 +549,6 @@ func TestAndThen(t *testing.T) { r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) - // r1 should execute the counter rule - // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { e, err := r(p, a, chainCounter(&j, chainIdentity)) if e != p { @@ -561,6 +566,37 @@ func TestAndThen(t *testing.T) { } } +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + e, err := tc.r(p, tc.initialError, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + func TestDropOnError(t *testing.T) { var ( i, j int @@ -586,6 +622,16 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } + e, err := r2(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } } func TestDropOnSuccess(t *testing.T) { @@ -612,6 +658,35 @@ func TestDropOnSuccess(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } + a := errors.New("a") + e, err := r2(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + e, err = r3(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } } func TestThenDrop(t *testing.T) { diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 2f6a49f1..9bd09ebe 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -152,7 +152,7 @@ func flatten(errors []error) ErrorList { return ErrorList(result) } -// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool), OrElse(...Rule) +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) // If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. func (r Rule) If(b bool) Rule { @@ -249,6 +249,13 @@ func (r Rule) ThenDrop() Rule { } } +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return ch(e, Error2(err, injected)) + } +} + // DropOnError returns a Rule that generates a nil event if the error state != nil func DropOnError() Rule { return Rule(nil).DropOnError() diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 962c5f42..658b6397 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -156,8 +156,6 @@ func TestAndThen(t *testing.T) { r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) - // r1 should execute the counter rule - // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { e, err := r(p, a, chainCounter(&j, chainIdentity)) if e != p { @@ -175,6 +173,37 @@ func TestAndThen(t *testing.T) { } } +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + e, err := tc.r(p, tc.initialError, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + func TestDropOnError(t *testing.T) { var ( i, j int @@ -200,6 +229,16 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } + e, err := r2(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } } func TestDropOnSuccess(t *testing.T) { @@ -226,6 +265,35 @@ func TestDropOnSuccess(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } + a := errors.New("a") + e, err := r2(p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + e, err = r3(p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } } func TestThenDrop(t *testing.T) { From 6264b4f148b9eec4f5b694e101619c4f685c9ebb Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 12:19:35 +0000 Subject: [PATCH 36/67] support for context.Context w/ scheduler: events, rules, and reference controller --- api/v1/cmd/example-scheduler/app/app.go | 30 ++-- api/v1/cmd/example-scheduler/app/state.go | 7 +- api/v1/cmd/msh/msh.go | 22 +-- api/v1/lib/extras/rules/rules.go | 143 ++++++++++-------- .../extras/scheduler/controller/controller.go | 45 +++--- .../lib/extras/scheduler/controller/rules.go | 31 ++-- .../eventrules/eventrules_generated.go | 79 +++++----- .../eventrules/eventrules_generated_test.go | 64 ++++---- .../extras/scheduler/eventrules/handlers.go | 18 ++- api/v1/lib/scheduler/events/decorators.go | 8 +- api/v1/lib/scheduler/events/events.go | 37 ++--- api/v1/lib/scheduler/events/metrics.go | 7 +- 12 files changed, 265 insertions(+), 226 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index dcb75c0f..8b155bd3 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -1,6 +1,7 @@ package app import ( + "context" "errors" "io" "log" @@ -31,10 +32,9 @@ func (err StateError) Error() string { return string(err) } func Run(cfg Config) error { log.Printf("scheduler running with configuration: %+v", cfg) - shutdown := make(chan struct{}) - defer close(shutdown) + ctx, cancel := context.WithCancel(context.Background()) - state, err := newInternalState(cfg) + state, err := newInternalState(cfg, cancel) if err != nil { return err } @@ -57,9 +57,9 @@ func Run(cfg Config) error { }.Apply(state.cli) err = controller.Run( + ctx, buildFrameworkInfo(state.config), state.cli, - controller.WithDone(state.done.Closed), controller.WithEventHandler( buildEventHandler(state, frameworkIDStore), eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), @@ -67,7 +67,7 @@ func Run(cfg Config) error { ), controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), controller.WithRegistrationTokens( - backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), + backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, ctx.Done()), ), controller.WithSubscriptionTerminated(func(err error) { if err != nil { @@ -75,7 +75,7 @@ func Run(cfg Config) error { log.Println(err) } if _, ok := err.(StateError); ok { - state.done.Close() + state.shutdown() } return } @@ -106,15 +106,15 @@ func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) e } func trackOffersReceived(state *internalState) eventrules.Rule { - return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) { if err == nil { state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers())) } - return chain(e, err) + return chain(ctx, e, err) } } -func failure(e *scheduler.Event) error { +func failure(_ context.Context, e *scheduler.Event) error { var ( f = e.GetFailure() eid, aid, stat = f.ExecutorID, f.AgentID, f.Status @@ -137,7 +137,7 @@ func failure(e *scheduler.Event) error { } func resourceOffers(state *internalState) events.HandlerFunc { - return func(e *scheduler.Event) error { + return func(_ context.Context, e *scheduler.Event) error { var ( offers = e.GetOffers().GetOffers() callOption = calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) @@ -222,7 +222,7 @@ func resourceOffers(state *internalState) events.HandlerFunc { } func statusUpdate(state *internalState) events.HandlerFunc { - return func(e *scheduler.Event) error { + return func(_ context.Context, e *scheduler.Event) error { s := e.GetUpdate().GetStatus() if state.config.verbose { msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() @@ -239,7 +239,7 @@ func statusUpdate(state *internalState) events.HandlerFunc { if state.tasksFinished == state.totalTasks { log.Println("mission accomplished, terminating") - state.done.Close() + state.shutdown() } else { tryReviveOffers(state) } @@ -250,7 +250,7 @@ func statusUpdate(state *internalState) events.HandlerFunc { " with reason " + s.GetReason().String() + " from source " + s.GetSource().String() + " with message '" + s.GetMessage() + "'") - state.done.Close() + state.shutdown() } return nil } @@ -273,9 +273,9 @@ func tryReviveOffers(state *internalState) { // logAllEvents logs every observed event; this is somewhat expensive to do func logAllEvents(h events.Handler) events.Handler { - return events.HandlerFunc(func(e *scheduler.Event) error { + return events.HandlerFunc(func(ctx context.Context, e *scheduler.Event) error { log.Printf("%+v\n", *e) - return h.HandleEvent(e) + return h.HandleEvent(ctx, e) }) } diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index 96c05edc..6864bfa4 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -13,7 +13,6 @@ import ( proto "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" - "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" @@ -184,7 +183,7 @@ func loadCredentials(userConfig credentials) (result credentials, err error) { return } -func newInternalState(cfg Config) (*internalState, error) { +func newInternalState(cfg Config, shutdown func()) (*internalState, error) { metricsAPI := initMetrics(cfg) executorInfo, err := prepareExecutorInfo( cfg.executor, @@ -210,7 +209,7 @@ func newInternalState(cfg Config) (*internalState, error) { metricsAPI: metricsAPI, cli: buildHTTPSched(cfg, creds), random: rand.New(rand.NewSource(time.Now().Unix())), - done: latch.New(), + shutdown: shutdown, } return state, nil } @@ -227,6 +226,6 @@ type internalState struct { reviveTokens <-chan struct{} metricsAPI *metricsAPI err error - done latch.Interface + shutdown func() random *rand.Rand } diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index d20fe570..20bbde9f 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -10,6 +10,7 @@ package main // import ( + "context" "flag" "fmt" "io" @@ -19,7 +20,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" - "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" @@ -105,20 +105,20 @@ func main() { func run() error { var ( - done = latch.New() - caller = calls.Decorators{ + ctx, cancel = context.WithCancel(context.Background()) + caller = calls.Decorators{ calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), }.Apply(buildClient()) ) return controller.Run( + ctx, &mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)}, caller, - controller.WithDone(done.Closed), controller.WithEventHandler(buildEventHandler(caller)), controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), controller.WithSubscriptionTerminated(func(err error) { - defer done.Close() + defer cancel() if err == io.EOF { log.Println("disconnected") } @@ -143,12 +143,12 @@ func buildEventHandler(caller calls.Caller) events.Handler { } func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { - return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) { if err != nil { - return chain(e, err) + return chain(ctx, e, err) } if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress { - return chain(e, err) + return chain(ctx, e, err) } off := offers.Slice(e.GetOffers().GetOffers()) err = calls.CallNoData(caller, calls.Decline(off.IDs()...).With(refuseSeconds)) @@ -156,12 +156,12 @@ func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { // we shouldn't have received offers, maybe the prior suppress call failed? err = calls.CallNoData(caller, calls.Suppress()) } - return e, err // drop + return ctx, e, err // drop } } func resourceOffers(caller calls.Caller) events.HandlerFunc { - return func(e *scheduler.Event) (err error) { + return func(_ context.Context, e *scheduler.Event) (err error) { var ( off = e.GetOffers().GetOffers() index = offers.NewIndex(off, nil) @@ -197,7 +197,7 @@ func resourceOffers(caller calls.Caller) events.HandlerFunc { } } -func statusUpdate(e *scheduler.Event) error { +func statusUpdate(_ context.Context, e *scheduler.Event) error { s := e.GetUpdate().GetStatus() switch st := s.GetState(); st { case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING: diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go index 8303e991..db80fe6d 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/rules/rules.go @@ -95,6 +95,7 @@ var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( + "context" "fmt" "sync" {{range .Imports}} @@ -103,23 +104,23 @@ import ( ) type ( - iface interface { + evaler interface { // Eval executes a filter, rule, or decorator function; if the returned event is nil then // no additional rule evaluation should be processed for the event. // Eval implementations should not modify the given event parameter (to avoid side effects). // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. // Eval implementations SHOULD be safe to execute concurrently. - Eval({{.EventType}}, error, Chain) ({{.EventType}}, error) + Eval(context.Context, {{.EventType}}, error, Chain) (context.Context, {{.EventType}}, error) } - // Rule is the functional adaptation of iface. + // Rule is the functional adaptation of evaler. // A nil Rule is valid: it is Eval'd as a noop. - Rule func({{.EventType}}, error, Chain) ({{.EventType}}, error) + Rule func(context.Context, {{.EventType}}, error, Chain) (context.Context, {{.EventType}}, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, // no additional rules are processed. - Chain func({{.EventType}}, error) ({{.EventType}}, error) + Chain func(context.Context, {{.EventType}}, error) (context.Context, {{.EventType}}, error) // Rules is a list of rules to be processed, in order. Rules []Rule @@ -131,37 +132,37 @@ type ( ) var ( - _ = iface(Rule(nil)) - _ = iface(Rules{}) + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(e {{.EventType}}, err error) ({{.EventType}}, error) { - return e, err + chainIdentity = func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { + return ctx, e, err } ) // Eval is a convenience func that processes a nil Rule as a noop. -func (r Rule) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { +func (r Rule) Eval(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { if r != nil { - return r(e, err, ch) + return r(ctx, e, err, ch) } - return ch(e, err) + return ch(ctx, e, err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. -func (rs Rules) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { - return ch(rs.Chain()(e, err)) +func (rs Rules) Eval(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return ch(rs.Chain()(ctx, e, err)) } -// Chain returns a Chain that evaluates the given Rules, in order, propagating the ({{.EventType}}, error) +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.EventType}}, error) // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { return chainIdentity } - return func(e {{.EventType}}, err error) ({{.EventType}}, error) { - return rs[0].Eval(e, err, rs[1:].Chain()) + return func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { + return rs[0].Eval(ctx, e, err, rs[1:].Chain()) } } @@ -268,16 +269,16 @@ func (r Rule) Once() Rule { return nil } var once sync.Once - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { ruleInvoked := false once.Do(func() { - e, err = r(e, err, ch) + ctx, e, err = r(ctx, e, err, ch) ruleInvoked = true }) if !ruleInvoked { - e, err = ch(e, err) + ctx, e, err = ch(ctx, e, err) } - return e, err + return ctx, e, err } } @@ -287,15 +288,15 @@ func (r Rule) Poll(p <-chan struct{}) Rule { if p == nil || r == nil { return nil } - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { select { case <-p: // do something // TODO(jdef): optimization: if we detect the chan is closed, affect a state change // whereby this select is no longer invoked (and always pass control to r). - return r(e, err, ch) + return r(ctx, e, err, ch) default: - return ch(e, err) + return ch(ctx, e, err) } } } @@ -321,30 +322,30 @@ func (r Rule) EveryN(nthTime int) Rule { return false } ) - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { if forward() { - return r(e, err, ch) + return r(ctx, e, err, ch) } - return ch(e, err) + return ch(ctx, e, err) } } -// Drop aborts the Chain and returns the ({{.EventType}}, error) tuple as-is. +// Drop aborts the Chain and returns the (context.Context, {{.EventType}}, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() } -// ThenDrop executes the receiving rule, but aborts the Chain, and returns the ({{.EventType}}, error) tuple as-is. +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.EventType}}, error) tuple as-is. func (r Rule) ThenDrop() Rule { - return func(e {{.EventType}}, err error, _ Chain) ({{.EventType}}, error) { - return r.Eval(e, err, chainIdentity) + return func(ctx context.Context, e {{.EventType}}, err error, _ Chain) (context.Context, {{.EventType}}, error) { + return r.Eval(ctx, e, err, chainIdentity) } } // Fail returns a Rule that injects the given error. func Fail(injected error) Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { - return ch(e, Error2(err, injected)) + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return ch(ctx, e, Error2(err, injected)) } } @@ -356,11 +357,11 @@ func DropOnError() Rule { // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then // the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { if err != nil { - return e, err + return ctx, e, err } - return r.Eval(e, err, ch) + return r.Eval(ctx, e, err, ch) } } @@ -376,12 +377,12 @@ func DropOnSuccess() Rule { } func (r Rule) DropOnSuccess() Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { if err == nil { // bypass remainder of chain - return e, err + return ctx, e, err } - return r.Eval(e, err, ch) + return r.Eval(ctx, e, err, ch) } } @@ -396,6 +397,7 @@ var testTemplate = template.Must(template.New("").Parse(`package {{.Package}} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( + "context" "errors" "reflect" "testing" @@ -407,29 +409,29 @@ import ( func prototype() {{.EventType}} { return {{.EventPrototype}} } func counter(i *int) Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { *i++ - return ch(e, err) + return ch(ctx, e, err) } } func tracer(r Rule, name string, t *testing.T) Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { t.Log("executing", name) - return r(e, err, ch) + return r(ctx, e, err, ch) } } func returnError(re error) Rule { - return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { - return ch(e, Error2(err, re)) + return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return ch(ctx, e, Error2(err, re)) } } func chainCounter(i *int, ch Chain) Chain { - return func(e {{.EventType}}, err error) ({{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { *i++ - return ch(e, err) + return ch(ctx, e, err) } } @@ -437,7 +439,7 @@ func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) - e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -451,8 +453,9 @@ func TestChainIdentity(t *testing.T) { func TestRules(t *testing.T) { var ( - p = prototype() - a = errors.New("a") + p = prototype() + a = errors.New("a") + ctx = context.Background() ) // multiple rules in Rules should execute, dropping nil rules along the way @@ -474,7 +477,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - e, err = rule(tc.e, tc.err, chainIdentity) + _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -487,7 +490,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, err - e, err = Rules{}.Eval(tc.e, tc.err, chainIdentity) + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -545,12 +548,13 @@ func TestAndThen(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) for k, r := range []Rule{r1, r2} { - e, err := r(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -570,6 +574,7 @@ func TestOnFailure(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() a = errors.New("a") r1 = counter(&i) r2 = Fail(a).OnFailure(counter(&i)) @@ -581,7 +586,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - e, err := tc.r(p, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -601,6 +606,7 @@ func TestDropOnError(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).DropOnError() a = errors.New("a") @@ -608,7 +614,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - e, err := r(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -622,7 +628,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - e, err := r2(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -638,13 +644,14 @@ func TestDropOnSuccess(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).DropOnSuccess() ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -659,7 +666,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - e, err := r2(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -674,7 +681,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - e, err = r3(p, nil, chainCounter(&j, chainIdentity)) + _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -694,12 +701,13 @@ func TestThenDrop(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).ThenDrop() ) // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -721,13 +729,14 @@ func TestDrop(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = Rules{Drop(), counter(&i)}.Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -748,13 +757,14 @@ func TestIf(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).If(true).Eval r2 = counter(&i).If(false).Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -774,13 +784,14 @@ func TestUnless(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Unless(false).Eval r2 = counter(&i).Unless(true).Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -800,12 +811,13 @@ func TestOnce(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Once().Eval r2 = Rule(nil).Once().Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -843,12 +855,13 @@ func TestPoll(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Poll(tc.ch).Eval r2 = Rule(nil).Poll(tc.ch).Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index e56aafc9..45808ecb 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -1,6 +1,8 @@ package controller import ( + "context" + "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/encoding" "github.com/mesos/mesos-go/api/v1/lib/scheduler" @@ -14,7 +16,6 @@ type ( // Config is an opaque controller configuration. Properties are configured by applying Option funcs. Config struct { - doneFunc func() bool frameworkIDFunc func() string handler events.Handler registrationTokens <-chan struct{} @@ -45,16 +46,6 @@ func WithFrameworkID(frameworkIDFunc func() string) Option { } } -// WithDone sets a fetcher func that returns true when the controller should exit. -// doneFunc is optional; nil equates to a func that always returns false. -func WithDone(doneFunc func() bool) Option { - return func(c *Config) Option { - old := c.doneFunc - c.doneFunc = doneFunc - return WithDone(old) - } -} - // WithSubscriptionTerminated sets a handler that is invoked at the end of every subscription cycle; the // given error may be nil if no error occurred. subscriptionTerminated is optional; if nil then errors are // swallowed. @@ -84,12 +75,19 @@ func (c *Config) tryFrameworkID() (result string) { return } -func (c *Config) tryDone() (result bool) { return c.doneFunc != nil && c.doneFunc() } +func isDone(ctx context.Context) (result bool) { + select { + case <-ctx.Done(): + return true + default: + return false + } +} // Run executes a control loop that registers a framework with Mesos and processes the scheduler events // that flow through the subscription. Upon disconnection, if the current configuration reports "not done" // then the controller will attempt to re-register the framework and continue processing events. -func Run(framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) (lastErr error) { +func Run(ctx context.Context, framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) (lastErr error) { var config Config for _, opt := range options { if opt != nil { @@ -100,16 +98,21 @@ func Run(framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) config.handler = DefaultHandler } subscribe := calls.Subscribe(framework) - for !config.tryDone() { + for !isDone(ctx) { frameworkID := config.tryFrameworkID() if framework.GetFailoverTimeout() > 0 && frameworkID != "" { subscribe.With(calls.SubscribeTo(frameworkID)) } if config.registrationTokens != nil { - <-config.registrationTokens + select { + case <-config.registrationTokens: + // continue + case <-ctx.Done(): + return ctx.Err() + } } resp, err := caller.Call(subscribe) - lastErr = processSubscription(config, resp, err) + lastErr = processSubscription(ctx, config, resp, err) if config.subscriptionTerminated != nil { config.subscriptionTerminated(lastErr) } @@ -117,23 +120,23 @@ func Run(framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) return } -func processSubscription(config Config, resp mesos.Response, err error) error { +func processSubscription(ctx context.Context, config Config, resp mesos.Response, err error) error { if resp != nil { defer resp.Close() } if err == nil { - err = eventLoop(config, resp) + err = eventLoop(ctx, config, resp) } return err } // eventLoop returns the framework ID received by mesos (if any); callers should check for a // framework ID regardless of whether error != nil. -func eventLoop(config Config, eventDecoder encoding.Decoder) (err error) { - for err == nil && !config.tryDone() { +func eventLoop(ctx context.Context, config Config, eventDecoder encoding.Decoder) (err error) { + for err == nil && !isDone(ctx) { var e scheduler.Event if err = eventDecoder.Decode(&e); err == nil { - err = config.handler.HandleEvent(&e) + err = config.handler.HandleEvent(ctx, &e) } } return err diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go index 6f3e0609..0bbb9a02 100644 --- a/api/v1/lib/extras/scheduler/controller/rules.go +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "log" "time" @@ -21,17 +22,17 @@ func (e ErrEvent) Error() string { // LiftErrors extract the error message from a scheduler error event and returns it as an ErrEvent // so that downstream rules/handlers may continue processing. func LiftErrors() Rule { - return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { if err != nil { - return chain(e, err) + return chain(ctx, e, err) } if e.GetType() == scheduler.Event_ERROR { // it's recommended that we abort and re-try subscribing; returning an // error here will cause the event loop to terminate and the connection // will be reset. - return chain(e, ErrEvent(e.GetError().GetMessage())) + return chain(ctx, e, ErrEvent(e.GetError().GetMessage())) } - return chain(e, nil) + return chain(ctx, e, nil) } } @@ -42,9 +43,9 @@ type StateError string func (err StateError) Error() string { return string(err) } func TrackSubscription(frameworkIDStore store.Singleton, failoverTimeout time.Duration) Rule { - return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { if err != nil { - return chain(e, err) + return chain(ctx, e, err) } if e.GetType() == scheduler.Event_SUBSCRIBED { var ( @@ -52,22 +53,22 @@ func TrackSubscription(frameworkIDStore store.Singleton, failoverTimeout time.Du frameworkID = e.GetSubscribed().GetFrameworkID().GetValue() ) if err != nil && err != store.ErrNotFound { - return chain(e, err) + return chain(ctx, e, err) } // order of `if` statements are important: tread carefully w/ respect to future changes if frameworkID == "" { // sanity check, should **never** happen - return chain(e, StateError("mesos sent an empty frameworkID?!")) + return chain(ctx, e, StateError("mesos sent an empty frameworkID?!")) } if storedFrameworkID != "" && storedFrameworkID != frameworkID && failoverTimeout > 0 { - return chain(e, StateError(fmt.Sprintf( + return chain(ctx, e, StateError(fmt.Sprintf( "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", failoverTimeout))) } if storedFrameworkID != frameworkID { frameworkIDStore.Set(frameworkID) } } - return chain(e, nil) + return chain(ctx, e, nil) } } @@ -81,7 +82,7 @@ func AckStatusUpdates(caller calls.Caller) Rule { // AckStatusUpdatesF is a functional adapter for AckStatusUpdates, useful for cases where the caller may // change over time. An error that occurs while ack'ing the status update is returned as a calls.AckError. func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { - return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { // aggressively attempt to ack updates: even if there's pre-existing error state attempt // to acknowledge all status updates. origErr := err @@ -104,11 +105,11 @@ func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { // Mesos will ask us to ACK anyway -- why pay special attention to these // call failures vs others? err = &calls.AckError{Ack: ack, Cause: err} - return nil, Error2(origErr, err) // drop + return ctx, e, Error2(origErr, err) // drop (do not propagate to chain) } } } - return chain(e, origErr) + return chain(ctx, e, origErr) } } @@ -121,8 +122,8 @@ var ( // LogEvents returns a rule that logs scheduler events to the EventLogger func LogEvents() Rule { - return Rule(func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + return Rule(func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { EventLogger(e) - return chain(e, err) + return chain(ctx, e, err) }) } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 9bd09ebe..c19affd3 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -4,6 +4,7 @@ package eventrules // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( + "context" "fmt" "sync" @@ -11,23 +12,23 @@ import ( ) type ( - iface interface { + evaler interface { // Eval executes a filter, rule, or decorator function; if the returned event is nil then // no additional rule evaluation should be processed for the event. // Eval implementations should not modify the given event parameter (to avoid side effects). // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. // Eval implementations SHOULD be safe to execute concurrently. - Eval(*scheduler.Event, error, Chain) (*scheduler.Event, error) + Eval(context.Context, *scheduler.Event, error, Chain) (context.Context, *scheduler.Event, error) } - // Rule is the functional adaptation of iface. + // Rule is the functional adaptation of evaler. // A nil Rule is valid: it is Eval'd as a noop. - Rule func(*scheduler.Event, error, Chain) (*scheduler.Event, error) + Rule func(context.Context, *scheduler.Event, error, Chain) (context.Context, *scheduler.Event, error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, // no additional rules are processed. - Chain func(*scheduler.Event, error) (*scheduler.Event, error) + Chain func(context.Context, *scheduler.Event, error) (context.Context, *scheduler.Event, error) // Rules is a list of rules to be processed, in order. Rules []Rule @@ -39,37 +40,37 @@ type ( ) var ( - _ = iface(Rule(nil)) - _ = iface(Rules{}) + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(e *scheduler.Event, err error) (*scheduler.Event, error) { - return e, err + chainIdentity = func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + return ctx, e, err } ) // Eval is a convenience func that processes a nil Rule as a noop. -func (r Rule) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { +func (r Rule) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { if r != nil { - return r(e, err, ch) + return r(ctx, e, err, ch) } - return ch(e, err) + return ch(ctx, e, err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. -func (rs Rules) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { - return ch(rs.Chain()(e, err)) +func (rs Rules) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(rs.Chain()(ctx, e, err)) } -// Chain returns a Chain that evaluates the given Rules, in order, propagating the (*scheduler.Event, error) +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *scheduler.Event, error) // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { return chainIdentity } - return func(e *scheduler.Event, err error) (*scheduler.Event, error) { - return rs[0].Eval(e, err, rs[1:].Chain()) + return func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + return rs[0].Eval(ctx, e, err, rs[1:].Chain()) } } @@ -176,16 +177,16 @@ func (r Rule) Once() Rule { return nil } var once sync.Once - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { ruleInvoked := false once.Do(func() { - e, err = r(e, err, ch) + ctx, e, err = r(ctx, e, err, ch) ruleInvoked = true }) if !ruleInvoked { - e, err = ch(e, err) + ctx, e, err = ch(ctx, e, err) } - return e, err + return ctx, e, err } } @@ -195,15 +196,15 @@ func (r Rule) Poll(p <-chan struct{}) Rule { if p == nil || r == nil { return nil } - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { select { case <-p: // do something // TODO(jdef): optimization: if we detect the chan is closed, affect a state change // whereby this select is no longer invoked (and always pass control to r). - return r(e, err, ch) + return r(ctx, e, err, ch) default: - return ch(e, err) + return ch(ctx, e, err) } } } @@ -229,30 +230,30 @@ func (r Rule) EveryN(nthTime int) Rule { return false } ) - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { if forward() { - return r(e, err, ch) + return r(ctx, e, err, ch) } - return ch(e, err) + return ch(ctx, e, err) } } -// Drop aborts the Chain and returns the (*scheduler.Event, error) tuple as-is. +// Drop aborts the Chain and returns the (context.Context, *scheduler.Event, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() } -// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (*scheduler.Event, error) tuple as-is. +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Event, error) tuple as-is. func (r Rule) ThenDrop() Rule { - return func(e *scheduler.Event, err error, _ Chain) (*scheduler.Event, error) { - return r.Eval(e, err, chainIdentity) + return func(ctx context.Context, e *scheduler.Event, err error, _ Chain) (context.Context, *scheduler.Event, error) { + return r.Eval(ctx, e, err, chainIdentity) } } // Fail returns a Rule that injects the given error. func Fail(injected error) Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { - return ch(e, Error2(err, injected)) + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(ctx, e, Error2(err, injected)) } } @@ -264,11 +265,11 @@ func DropOnError() Rule { // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then // the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { if err != nil { - return e, err + return ctx, e, err } - return r.Eval(e, err, ch) + return r.Eval(ctx, e, err, ch) } } @@ -284,12 +285,12 @@ func DropOnSuccess() Rule { } func (r Rule) DropOnSuccess() Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { if err == nil { // bypass remainder of chain - return e, err + return ctx, e, err } - return r.Eval(e, err, ch) + return r.Eval(ctx, e, err, ch) } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 658b6397..0a0c0f52 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -4,6 +4,7 @@ package eventrules // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( + "context" "errors" "reflect" "testing" @@ -14,29 +15,29 @@ import ( func prototype() *scheduler.Event { return &scheduler.Event{} } func counter(i *int) Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { *i++ - return ch(e, err) + return ch(ctx, e, err) } } func tracer(r Rule, name string, t *testing.T) Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { t.Log("executing", name) - return r(e, err, ch) + return r(ctx, e, err, ch) } } func returnError(re error) Rule { - return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { - return ch(e, Error2(err, re)) + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(ctx, e, Error2(err, re)) } } func chainCounter(i *int, ch Chain) Chain { - return func(e *scheduler.Event, err error) (*scheduler.Event, error) { + return func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { *i++ - return ch(e, err) + return ch(ctx, e, err) } } @@ -44,7 +45,7 @@ func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) - e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -58,8 +59,9 @@ func TestChainIdentity(t *testing.T) { func TestRules(t *testing.T) { var ( - p = prototype() - a = errors.New("a") + p = prototype() + a = errors.New("a") + ctx = context.Background() ) // multiple rules in Rules should execute, dropping nil rules along the way @@ -81,7 +83,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - e, err = rule(tc.e, tc.err, chainIdentity) + _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -94,7 +96,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, err - e, err = Rules{}.Eval(tc.e, tc.err, chainIdentity) + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -152,12 +154,13 @@ func TestAndThen(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) for k, r := range []Rule{r1, r2} { - e, err := r(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -177,6 +180,7 @@ func TestOnFailure(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() a = errors.New("a") r1 = counter(&i) r2 = Fail(a).OnFailure(counter(&i)) @@ -188,7 +192,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - e, err := tc.r(p, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -208,6 +212,7 @@ func TestDropOnError(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).DropOnError() a = errors.New("a") @@ -215,7 +220,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - e, err := r(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -229,7 +234,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - e, err := r2(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -245,13 +250,14 @@ func TestDropOnSuccess(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).DropOnSuccess() ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -266,7 +272,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - e, err := r2(p, a, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -281,7 +287,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - e, err = r3(p, nil, chainCounter(&j, chainIdentity)) + _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -301,12 +307,13 @@ func TestThenDrop(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = counter(&i).ThenDrop() ) // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -328,13 +335,14 @@ func TestDrop(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i) r2 = Rules{Drop(), counter(&i)}.Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -355,13 +363,14 @@ func TestIf(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).If(true).Eval r2 = counter(&i).If(false).Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -381,13 +390,14 @@ func TestUnless(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Unless(false).Eval r2 = counter(&i).Unless(true).Eval ) // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -407,12 +417,13 @@ func TestOnce(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Once().Eval r2 = Rule(nil).Once().Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -450,12 +461,13 @@ func TestPoll(t *testing.T) { var ( i, j int p = prototype() + ctx = context.Background() r1 = counter(&i).Poll(tc.ch).Eval r2 = Rule(nil).Poll(tc.ch).Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { - e, err := r(p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers.go index 4508a1ac..af60b9cf 100644 --- a/api/v1/lib/extras/scheduler/eventrules/handlers.go +++ b/api/v1/lib/extras/scheduler/eventrules/handlers.go @@ -1,6 +1,8 @@ package eventrules import ( + "context" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" ) @@ -10,14 +12,14 @@ func Handle(h events.Handler) Rule { if h == nil { return nil } - return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { - newErr := h.HandleEvent(e) - return chain(e, Error2(err, newErr)) + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) } } // HandleF is the functional equivalent of Handle -func HandleF(h events.HandlerFunc) Rule { +func HandleF(ctx context.Context, h events.HandlerFunc) Rule { return Handle(events.Handler(h)) } @@ -32,17 +34,17 @@ func (r Rule) HandleF(h events.HandlerFunc) Rule { } // HandleEvent implements events.Handler for Rule -func (r Rule) HandleEvent(e *scheduler.Event) (err error) { +func (r Rule) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { if r == nil { return nil } - _, err = r(e, nil, chainIdentity) + _, _, err = r(ctx, e, nil, chainIdentity) return } // HandleEvent implements events.Handler for Rules -func (rs Rules) HandleEvent(e *scheduler.Event) error { - return Rule(rs.Eval).HandleEvent(e) +func (rs Rules) HandleEvent(ctx context.Context, e *scheduler.Event) error { + return Rule(rs.Eval).HandleEvent(ctx, e) } /* diff --git a/api/v1/lib/scheduler/events/decorators.go b/api/v1/lib/scheduler/events/decorators.go index e4699af9..8da87fdb 100644 --- a/api/v1/lib/scheduler/events/decorators.go +++ b/api/v1/lib/scheduler/events/decorators.go @@ -1,6 +1,8 @@ package events import ( + "context" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) @@ -37,12 +39,12 @@ func (d Decorator) When(f func() bool) Decorator { // generates a new decorator every time the Decorator func is invoked. // probably OK for now. decorated := d(h) - return HandlerFunc(func(e *scheduler.Event) (err error) { + return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { if f() { // let the decorated handler process this - err = decorated.HandleEvent(e) + err = decorated.HandleEvent(ctx, e) } else { - err = h.HandleEvent(e) + err = h.HandleEvent(ctx, e) } return }) diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index 951f07c4..cd8ff241 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -1,6 +1,7 @@ package events import ( + "context" "sync" "github.com/mesos/mesos-go/api/v1/lib/scheduler" @@ -10,11 +11,11 @@ type ( // Handler is invoked upon the occurrence of some scheduler event that is generated // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) Handler interface { - HandleEvent(*scheduler.Event) error + HandleEvent(context.Context, *scheduler.Event) error } // HandlerFunc is a functional adaptation of the Handler interface - HandlerFunc func(*scheduler.Event) error + HandlerFunc func(context.Context, *scheduler.Event) error HandlerSet map[scheduler.Event_Type]Handler HandlerFuncSet map[scheduler.Event_Type]HandlerFunc @@ -40,9 +41,11 @@ type ( ) // HandleEvent implements Handler for HandlerFunc -func (f HandlerFunc) HandleEvent(e *scheduler.Event) error { return f(e) } +func (f HandlerFunc) HandleEvent(ctx context.Context, e *scheduler.Event) error { return f(ctx, e) } -func NoopHandler() HandlerFunc { return func(_ *scheduler.Event) error { return nil } } +func NoopHandler() HandlerFunc { + return func(_ context.Context, _ *scheduler.Event) error { return nil } +} // NewMux generates and returns a new, empty Mux instance. func NewMux(opts ...Option) *Mux { @@ -68,13 +71,13 @@ func (m *Mux) With(opts ...Option) Option { } // HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *scheduler.Event) error { - ok, err := m.handlers.tryHandleEvent(e) +func (m *Mux) HandleEvent(ctx context.Context, e *scheduler.Event) error { + ok, err := m.handlers.tryHandleEvent(ctx, e) if ok { return err } if m.defaultHandler != nil { - return m.defaultHandler.HandleEvent(e) + return m.defaultHandler.HandleEvent(ctx, e) } return nil } @@ -95,15 +98,15 @@ func Handle(et scheduler.Event_Type, eh Handler) Option { } // HandleEvent implements Handler for HandlerSet -func (hs HandlerSet) HandleEvent(e *scheduler.Event) (err error) { - _, err = hs.tryHandleEvent(e) +func (hs HandlerSet) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { + _, err = hs.tryHandleEvent(ctx, e) return } // tryHandleEvent returns true if the event was handled by a member of the HandlerSet -func (hs HandlerSet) tryHandleEvent(e *scheduler.Event) (bool, error) { +func (hs HandlerSet) tryHandleEvent(ctx context.Context, e *scheduler.Event) (bool, error) { if h := hs[e.GetType()]; h != nil { - return true, h.HandleEvent(e) + return true, h.HandleEvent(ctx, e) } return false, nil } @@ -162,9 +165,9 @@ func DefaultHandler(eh Handler) Option { // Deprecated in favor of Rules. func Once(h Handler) Handler { var once sync.Once - return HandlerFunc(func(e *scheduler.Event) (err error) { + return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { once.Do(func() { - err = h.HandleEvent(e) + err = h.HandleEvent(ctx, e) }) return }) @@ -177,9 +180,9 @@ func OnceFunc(h HandlerFunc) Handler { return Once(h) } // When // Deprecated in favor of Rules. func When(p Predicate, h Handler) Handler { - return HandlerFunc(func(e *scheduler.Event) (err error) { + return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { if p.Predicate().Apply(e) { - err = h.HandleEvent(e) + err = h.HandleEvent(ctx, e) } return }) @@ -191,10 +194,10 @@ func WhenFunc(p Predicate, h HandlerFunc) Handler { return When(p, h) } // HandleEvent implements Handler for Handlers. // Deprecated in favor of Rules. -func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { +func (hs Handlers) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { for _, h := range hs { if h != nil { - if err = h.HandleEvent(e); err != nil { + if err = h.HandleEvent(ctx, e); err != nil { break } } diff --git a/api/v1/lib/scheduler/events/metrics.go b/api/v1/lib/scheduler/events/metrics.go index 6693a00c..5055f042 100644 --- a/api/v1/lib/scheduler/events/metrics.go +++ b/api/v1/lib/scheduler/events/metrics.go @@ -1,6 +1,9 @@ package events +// TODO(jdef) move this code to the extras tree, it doesn't belong in the core lib + import ( + "context" "strings" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" @@ -12,9 +15,9 @@ func Metrics(harness xmetrics.Harness) Decorator { if h == nil { return h } - return HandlerFunc(func(e *scheduler.Event) error { + return HandlerFunc(func(ctx context.Context, e *scheduler.Event) error { typename := strings.ToLower(e.GetType().String()) - return harness(func() error { return h.HandleEvent(e) }, typename) + return harness(func() error { return h.HandleEvent(ctx, e) }, typename) }) } } From e964294680b755373716fccbe99aab271061b674 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 12:57:33 +0000 Subject: [PATCH 37/67] support for context.Context w/ scheduler: callers --- api/v1/cmd/example-scheduler/app/app.go | 16 ++++++------- api/v1/cmd/msh/msh.go | 12 +++++----- .../extras/scheduler/controller/controller.go | 2 +- .../lib/extras/scheduler/controller/rules.go | 2 +- api/v1/lib/httpcli/http.go | 9 +++++++ api/v1/lib/httpcli/httpsched/httpsched.go | 24 ++++++++++++------- api/v1/lib/httpcli/httpsched/state.go | 15 ++++++------ api/v1/lib/scheduler/calls/caller.go | 22 ++++++++++------- api/v1/lib/scheduler/calls/metrics.go | 7 ++++-- 9 files changed, 66 insertions(+), 43 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 8b155bd3..0646e774 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -137,7 +137,7 @@ func failure(_ context.Context, e *scheduler.Event) error { } func resourceOffers(state *internalState) events.HandlerFunc { - return func(_ context.Context, e *scheduler.Event) error { + return func(ctx context.Context, e *scheduler.Event) error { var ( offers = e.GetOffers().GetOffers() callOption = calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) @@ -201,7 +201,7 @@ func resourceOffers(state *internalState) events.HandlerFunc { ).With(callOption) // send Accept call to mesos - err := calls.CallNoData(state.cli, accept) + err := calls.CallNoData(ctx, state.cli, accept) if err != nil { log.Printf("failed to launch tasks: %+v", err) } else { @@ -222,7 +222,7 @@ func resourceOffers(state *internalState) events.HandlerFunc { } func statusUpdate(state *internalState) events.HandlerFunc { - return func(_ context.Context, e *scheduler.Event) error { + return func(ctx context.Context, e *scheduler.Event) error { s := e.GetUpdate().GetStatus() if state.config.verbose { msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() @@ -241,7 +241,7 @@ func statusUpdate(state *internalState) events.HandlerFunc { log.Println("mission accomplished, terminating") state.shutdown() } else { - tryReviveOffers(state) + tryReviveOffers(ctx, state) } case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: @@ -256,12 +256,12 @@ func statusUpdate(state *internalState) events.HandlerFunc { } } -func tryReviveOffers(state *internalState) { +func tryReviveOffers(ctx context.Context, state *internalState) { // limit the rate at which we request offer revival select { case <-state.reviveTokens: // not done yet, revive offers! - err := calls.CallNoData(state.cli, calls.Revive()) + err := calls.CallNoData(ctx, state.cli, calls.Revive()) if err != nil { log.Printf("failed to revive offers: %+v", err) return @@ -302,11 +302,11 @@ func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics b // logCalls logs a specific message string when a particular call-type is observed func logCalls(messages map[scheduler.Call_Type]string) calls.Decorator { return func(caller calls.Caller) calls.Caller { - return calls.CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { + return calls.CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { if message, ok := messages[c.GetType()]; ok { log.Println(message) } - return caller.Call(c) + return caller.Call(ctx, c) }) } } diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 20bbde9f..9df1b5e3 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -151,17 +151,17 @@ func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { return chain(ctx, e, err) } off := offers.Slice(e.GetOffers().GetOffers()) - err = calls.CallNoData(caller, calls.Decline(off.IDs()...).With(refuseSeconds)) + err = calls.CallNoData(ctx, caller, calls.Decline(off.IDs()...).With(refuseSeconds)) if err == nil { // we shouldn't have received offers, maybe the prior suppress call failed? - err = calls.CallNoData(caller, calls.Suppress()) + err = calls.CallNoData(ctx, caller, calls.Suppress()) } return ctx, e, err // drop } } func resourceOffers(caller calls.Caller) events.HandlerFunc { - return func(_ context.Context, e *scheduler.Event) (err error) { + return func(ctx context.Context, e *scheduler.Event) (err error) { var ( off = e.GetOffers().GetOffers() index = offers.NewIndex(off, nil) @@ -173,7 +173,7 @@ func resourceOffers(caller calls.Caller) events.HandlerFunc { task.AgentID = match.AgentID task.Resources = mesos.Resources(match.Resources).Find(wantsResources.Flatten(Role.Assign())) - err = calls.CallNoData(caller, calls.Accept( + err = calls.CallNoData(ctx, caller, calls.Accept( calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), )) if err != nil { @@ -186,12 +186,12 @@ func resourceOffers(caller calls.Caller) events.HandlerFunc { } // decline all but the possible match delete(index, match.GetID()) - err = calls.CallNoData(caller, calls.Decline(index.IDs()...).With(refuseSeconds)) + err = calls.CallNoData(ctx, caller, calls.Decline(index.IDs()...).With(refuseSeconds)) if err != nil { return } if declineAndSuppress { - err = calls.CallNoData(caller, calls.Suppress()) + err = calls.CallNoData(ctx, caller, calls.Suppress()) } return } diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index 45808ecb..a229b49a 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -111,7 +111,7 @@ func Run(ctx context.Context, framework *mesos.FrameworkInfo, caller calls.Calle return ctx.Err() } } - resp, err := caller.Call(subscribe) + resp, err := caller.Call(ctx, subscribe) lastErr = processSubscription(ctx, config, resp, err) if config.subscriptionTerminated != nil { config.subscriptionTerminated(lastErr) diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go index 0bbb9a02..192c0578 100644 --- a/api/v1/lib/extras/scheduler/controller/rules.go +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -98,7 +98,7 @@ func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { s.TaskID.Value, uuid, ) - err = calls.CallNoData(callerLookup(), ack) + err = calls.CallNoData(ctx, callerLookup(), ack) if err != nil { // TODO(jdef): not sure how important this is; if there's an error ack'ing // because we beacame disconnected, then we'll just reconnect later and diff --git a/api/v1/lib/httpcli/http.go b/api/v1/lib/httpcli/http.go index 0258eb47..a7b5372c 100644 --- a/api/v1/lib/httpcli/http.go +++ b/api/v1/lib/httpcli/http.go @@ -2,6 +2,7 @@ package httpcli import ( "bytes" + "context" "crypto/tls" "fmt" "io" @@ -316,6 +317,14 @@ func Header(k, v string) RequestOpt { return func(r *http.Request) { r.Header.Ad // Close returns a RequestOpt that determines whether to close the underlying connection after sending the request. func Close(b bool) RequestOpt { return func(r *http.Request) { r.Close = b } } +// Context returns a RequestOpt that sets the request's Context (ctx must be non-nil) +func Context(ctx context.Context) RequestOpt { + return func(r *http.Request) { + r2 := r.WithContext(ctx) + *r = *r2 + } +} + type Config struct { client *http.Client dialer *net.Dialer diff --git a/api/v1/lib/httpcli/httpsched/httpsched.go b/api/v1/lib/httpcli/httpsched/httpsched.go index 0f53d905..dc8b899b 100644 --- a/api/v1/lib/httpcli/httpsched/httpsched.go +++ b/api/v1/lib/httpcli/httpsched/httpsched.go @@ -1,6 +1,7 @@ package httpsched import ( + "context" "log" "net/http" "net/url" @@ -43,7 +44,7 @@ type ( calls.Caller // httpDo is intentionally package-private; clients of this package may extend a Caller // generated by this package by overriding the Call func but may not customize httpDo. - httpDo(encoding.Marshaler, ...httpcli.RequestOpt) (mesos.Response, error) + httpDo(context.Context, encoding.Marshaler, ...httpcli.RequestOpt) (mesos.Response, error) } callerInternal interface { @@ -64,22 +65,22 @@ type ( } ) -func (ct *callerTemporary) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { +func (ct *callerTemporary) httpDo(ctx context.Context, m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { ct.callerInternal.WithTemporary(ct.opt, func() error { if len(opt) == 0 { opt = ct.requestOpts } else if len(ct.requestOpts) > 0 { opt = append(opt[:], ct.requestOpts...) } - resp, err = ct.callerInternal.httpDo(m, opt...) + resp, err = ct.callerInternal.httpDo(ctx, m, opt...) return nil }) return } -func (ct *callerTemporary) Call(call *scheduler.Call) (resp mesos.Response, err error) { +func (ct *callerTemporary) Call(ctx context.Context, call *scheduler.Call) (resp mesos.Response, err error) { ct.callerInternal.WithTemporary(ct.opt, func() error { - resp, err = ct.callerInternal.Call(call) + resp, err = ct.callerInternal.Call(ctx, call) return nil }) return @@ -113,7 +114,7 @@ func NewCaller(cl *httpcli.Client, opts ...Option) calls.Caller { // httpDo decorates the inherited behavior w/ support for HTTP redirection to follow Mesos leadership changes. // NOTE: this implementation will change the state of the client upon Mesos leadership changes. -func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { +func (cli *client) httpDo(ctx context.Context, m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { var ( done chan struct{} // avoid allocating these chans unless we actually need to redirect redirectBackoff <-chan struct{} @@ -130,6 +131,7 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp close(done) } }() + opt = append(opt, httpcli.Context(ctx)) for attempt := 0; ; attempt++ { resp, err = cli.Client.Do(m, opt...) redirectErr, ok := err.(*mesosRedirectionError) @@ -139,7 +141,11 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp if attempt < cli.redirect.MaxAttempts { log.Println("redirecting to " + redirectErr.newURL) cli.With(httpcli.Endpoint(redirectErr.newURL)) - <-getBackoff() + select { + case <-getBackoff(): + case <-ctx.Done(): + return nil, ctx.Err() + } continue } return @@ -147,8 +153,8 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp } // Call implements Client -func (cli *client) Call(call *scheduler.Call) (mesos.Response, error) { - return cli.httpDo(call) +func (cli *client) Call(ctx context.Context, call *scheduler.Call) (mesos.Response, error) { + return cli.httpDo(ctx, call) } type mesosRedirectionError struct{ newURL string } diff --git a/api/v1/lib/httpcli/httpsched/state.go b/api/v1/lib/httpcli/httpsched/state.go index a273b243..6d4d9c3c 100644 --- a/api/v1/lib/httpcli/httpsched/state.go +++ b/api/v1/lib/httpcli/httpsched/state.go @@ -1,6 +1,7 @@ package httpsched import ( + "context" "fmt" "log" "net/http" @@ -42,7 +43,7 @@ type ( err error // err is the error from the most recently executed call } - stateFn func(*state) stateFn + stateFn func(context.Context, *state) stateFn ) func maybeLogged(f httpcli.DoFunc) httpcli.DoFunc { @@ -108,7 +109,7 @@ func disconnectionDecoder(decoder encoding.Decoder, disconnect func()) encoding. }) } -func disconnectedFn(state *state) stateFn { +func disconnectedFn(ctx context.Context, state *state) stateFn { // (a) validate call = SUBSCRIBE if state.call.GetType() != scheduler.Call_SUBSCRIBE { state.resp = nil @@ -144,7 +145,7 @@ func disconnectedFn(state *state) stateFn { ) // (c) execute the call, save the result in resp, err - stateResp, stateErr := subscribeCaller.Call(state.call) + stateResp, stateErr := subscribeCaller.Call(ctx, state.call) state.err = stateErr // (d) if err != nil return disconnectedFn since we're unsubscribed @@ -198,7 +199,7 @@ func errorIndicatesSubscriptionLoss(err error) (result bool) { return } -func connectedFn(state *state) stateFn { +func connectedFn(ctx context.Context, state *state) stateFn { // (a) validate call != SUBSCRIBE if state.call.GetType() == scheduler.Call_SUBSCRIBE { state.resp = nil @@ -216,7 +217,7 @@ func connectedFn(state *state) stateFn { } // (b) execute call, save the result in resp, err - state.resp, state.err = state.caller.Call(state.call) + state.resp, state.err = state.caller.Call(ctx, state.call) if errorIndicatesSubscriptionLoss(state.err) { // properly transition back to a disconnected state if mesos thinks that we're unsubscribed @@ -227,11 +228,11 @@ func connectedFn(state *state) stateFn { return connectedFn } -func (state *state) Call(call *scheduler.Call) (resp mesos.Response, err error) { +func (state *state) Call(ctx context.Context, call *scheduler.Call) (resp mesos.Response, err error) { state.m.Lock() defer state.m.Unlock() state.call = call - state.fn = state.fn(state) + state.fn = state.fn(ctx, state) if debug && state.err != nil { log.Print(*call, state.err) diff --git a/api/v1/lib/scheduler/calls/caller.go b/api/v1/lib/scheduler/calls/caller.go index 772dc850..48ae2acf 100644 --- a/api/v1/lib/scheduler/calls/caller.go +++ b/api/v1/lib/scheduler/calls/caller.go @@ -1,6 +1,8 @@ package calls import ( + "context" + "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) @@ -9,11 +11,11 @@ import ( type ( Caller interface { // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. - Call(*scheduler.Call) (mesos.Response, error) + Call(context.Context, *scheduler.Call) (mesos.Response, error) } // CallerFunc is the functional adaptation of the Caller interface - CallerFunc func(*scheduler.Call) (mesos.Response, error) + CallerFunc func(context.Context, *scheduler.Call) (mesos.Response, error) // Decorator funcs usually return a Caller whose behavior has been somehow modified Decorator func(Caller) Caller @@ -23,7 +25,9 @@ type ( ) // Call implements the Caller interface for CallerFunc -func (f CallerFunc) Call(c *scheduler.Call) (mesos.Response, error) { return f(c) } +func (f CallerFunc) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + return f(ctx, c) +} // Apply is a convenient, nil-safe applicator that returns the result of d(c) iff d != nil; otherwise c func (d Decorator) Apply(c Caller) (result Caller) { @@ -79,9 +83,9 @@ func (ds Decorators) Combine() (result Decorator) { // Deprecated in favor of SubscribedCaller; should remove after v0.0.3. func FrameworkCaller(frameworkID string) Decorator { return func(h Caller) Caller { - return CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { + return CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { c.FrameworkID = &mesos.FrameworkID{Value: frameworkID} - return h.Call(c) + return h.Call(ctx, c) }) } } @@ -91,14 +95,14 @@ func FrameworkCaller(frameworkID string) Decorator { // - calls are not modified when the generated framework ID is "" func SubscribedCaller(frameworkID func() string) Decorator { return func(h Caller) Caller { - return CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { + return CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { // never overwrite framework ID for subscribe calls; the scheduler must do that part if c.GetType() != scheduler.Call_SUBSCRIBE { if fid := frameworkID(); fid != "" { c.FrameworkID = &mesos.FrameworkID{Value: fid} } } - return h.Call(c) + return h.Call(ctx, c) }) } } @@ -107,8 +111,8 @@ var noopDecorator = Decorator(func(h Caller) Caller { return h }) // CallNoData is a convenience func that executes the given Call using the provided Caller // and always drops the response data. -func CallNoData(caller Caller, call *scheduler.Call) error { - resp, err := caller.Call(call) +func CallNoData(ctx context.Context, caller Caller, call *scheduler.Call) error { + resp, err := caller.Call(ctx, call) if resp != nil { resp.Close() } diff --git a/api/v1/lib/scheduler/calls/metrics.go b/api/v1/lib/scheduler/calls/metrics.go index 2319a7bc..c3779205 100644 --- a/api/v1/lib/scheduler/calls/metrics.go +++ b/api/v1/lib/scheduler/calls/metrics.go @@ -1,6 +1,9 @@ package calls +// TODO(jdef): move this code to the extras tree, it doesn't belong here + import ( + "context" "strings" "github.com/mesos/mesos-go/api/v1/lib" @@ -11,10 +14,10 @@ import ( func CallerMetrics(harness xmetrics.Harness) Decorator { return func(caller Caller) (metricsCaller Caller) { if caller != nil { - metricsCaller = CallerFunc(func(c *scheduler.Call) (res mesos.Response, err error) { + metricsCaller = CallerFunc(func(ctx context.Context, c *scheduler.Call) (res mesos.Response, err error) { typename := strings.ToLower(c.GetType().String()) harness(func() error { - res, err = caller.Call(c) + res, err = caller.Call(ctx, c) return err // need to count these }, typename) return From 8d530585a4eca1ab5ffeff1d5f018e0979990059 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 14:25:32 +0000 Subject: [PATCH 38/67] scheduler events: eliminate Decorators in favor of eventrules --- api/v1/cmd/example-scheduler/app/app.go | 38 +++++------ api/v1/lib/encoding/codec.go | 7 +- .../extras/scheduler/controller/controller.go | 4 +- .../extras/scheduler/eventrules/metrics.go | 20 ++++++ api/v1/lib/scheduler/events/decorators.go | 68 ------------------- api/v1/lib/scheduler/events/metrics.go | 23 ------- 6 files changed, 46 insertions(+), 114 deletions(-) create mode 100644 api/v1/lib/extras/scheduler/eventrules/metrics.go delete mode 100644 api/v1/lib/scheduler/events/decorators.go delete mode 100644 api/v1/lib/scheduler/events/metrics.go diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 0646e774..3ba57aa9 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -60,11 +60,7 @@ func Run(cfg Config) error { ctx, buildFrameworkInfo(state.config), state.cli, - controller.WithEventHandler( - buildEventHandler(state, frameworkIDStore), - eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), - events.Decorator(logAllEvents).If(state.config.verbose), - ), + controller.WithEventHandler(buildEventHandler(state, frameworkIDStore)), controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), controller.WithRegistrationTokens( backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, ctx.Done()), @@ -90,18 +86,20 @@ func Run(cfg Config) error { // buildEventHandler generates and returns a handler to process events received from the subscription. func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { - logger := controller.LogEvents() - return controller.LiftErrors().DropOnError().Handle(events.HandlerSet{ + // disable brief logs when verbose logs are enabled (there's no sense logging twice!) + logger := controller.LogEvents().Unless(state.config.verbose) + return eventrules.Concat( + eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), + logAllEvents().If(state.config.verbose), + controller.LiftErrors().DropOnError(), + ).Handle(events.HandlerSet{ scheduler.Event_FAILURE: logger.HandleF(failure), - scheduler.Event_OFFERS: eventrules.Concat( - trackOffersReceived(state), - logger.If(state.config.verbose), - ).HandleF(resourceOffers(state)), - scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), - scheduler.Event_SUBSCRIBED: eventrules.Rules{ + scheduler.Event_OFFERS: trackOffersReceived(state).HandleF(resourceOffers(state)), + scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), + scheduler.Event_SUBSCRIBED: eventrules.Concat( logger, controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout), - }, + ), }) } @@ -272,21 +270,21 @@ func tryReviveOffers(ctx context.Context, state *internalState) { } // logAllEvents logs every observed event; this is somewhat expensive to do -func logAllEvents(h events.Handler) events.Handler { - return events.HandlerFunc(func(ctx context.Context, e *scheduler.Event) error { +func logAllEvents() eventrules.Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch eventrules.Chain) (context.Context, *scheduler.Event, error) { log.Printf("%+v\n", *e) - return h.HandleEvent(ctx, e) - }) + return ch(ctx, e, err) + } } // eventMetrics logs metrics for every processed API event -func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) events.Decorator { +func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) eventrules.Rule { timed := metricsAPI.eventReceivedLatency if !timingMetrics { timed = nil } harness := xmetrics.NewHarness(metricsAPI.eventReceivedCount, metricsAPI.eventErrorCount, timed, clock) - return events.Metrics(harness) + return eventrules.Metrics(harness) } // callMetrics logs metrics for every outgoing Mesos call diff --git a/api/v1/lib/encoding/codec.go b/api/v1/lib/encoding/codec.go index 557b7aef..ebcd6ccd 100644 --- a/api/v1/lib/encoding/codec.go +++ b/api/v1/lib/encoding/codec.go @@ -48,7 +48,12 @@ type Codec struct { } // String implements the fmt.Stringer interface. -func (c *Codec) String() string { return c.Name } +func (c *Codec) String() string { + if c == nil { + return "" + } + return c.Name +} func (c *Codec) RequestContentType() string { return c.MediaTypes[0] } func (c *Codec) ResponseContentType() string { return c.MediaTypes[1] } diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index a229b49a..ee296d83 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -26,10 +26,10 @@ type ( // WithEventHandler sets the consumer of scheduler events. The controller's internal event processing // loop is aborted if a Handler returns a non-nil error, after which the controller may attempt // to re-register (subscribe) with Mesos. -func WithEventHandler(handler events.Handler, ds ...events.Decorator) Option { +func WithEventHandler(handler events.Handler) Option { return func(c *Config) Option { old := c.handler - c.handler = events.Decorators(ds).Apply(handler) + c.handler = handler return WithEventHandler(old) } } diff --git a/api/v1/lib/extras/scheduler/eventrules/metrics.go b/api/v1/lib/extras/scheduler/eventrules/metrics.go new file mode 100644 index 00000000..7713182f --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/metrics.go @@ -0,0 +1,20 @@ +package eventrules + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + typename := strings.ToLower(e.GetType().String()) + err = harness(func() error { + ctx, e, err = ch(ctx, e, err) + return err + }, typename) + return ctx, e, err + } +} diff --git a/api/v1/lib/scheduler/events/decorators.go b/api/v1/lib/scheduler/events/decorators.go deleted file mode 100644 index 8da87fdb..00000000 --- a/api/v1/lib/scheduler/events/decorators.go +++ /dev/null @@ -1,68 +0,0 @@ -package events - -import ( - "context" - - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -type ( - // Decorator functions typically modify behavior of the given delegate Handler. - Decorator func(Handler) Handler - - // Decorators aggregates Decorator functions - Decorators []Decorator -) - -var noopDecorator = Decorator(func(h Handler) Handler { return h }) - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// When returns a Decorator that evaluates the bool func every time the Handler is invoked. -// When f returns true, the Decorated Handler is invoked, otherwise the original Handler is. -func (d Decorator) When(f func() bool) Decorator { - if d == nil || f == nil { - return noopDecorator - } - return func(h Handler) Handler { - // generates a new decorator every time the Decorator func is invoked. - // probably OK for now. - decorated := d(h) - return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { - if f() { - // let the decorated handler process this - err = decorated.HandleEvent(ctx, e) - } else { - err = h.HandleEvent(ctx, e) - } - return - }) - } -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Handler that is ultimately returned. -func (ds Decorators) Apply(h Handler) Handler { - for _, d := range ds { - h = d.Apply(h) - } - return h -} - -func (d Decorator) Apply(h Handler) Handler { - if d != nil { - h = d(h) - } - return h -} diff --git a/api/v1/lib/scheduler/events/metrics.go b/api/v1/lib/scheduler/events/metrics.go deleted file mode 100644 index 5055f042..00000000 --- a/api/v1/lib/scheduler/events/metrics.go +++ /dev/null @@ -1,23 +0,0 @@ -package events - -// TODO(jdef) move this code to the extras tree, it doesn't belong in the core lib - -import ( - "context" - "strings" - - xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -func Metrics(harness xmetrics.Harness) Decorator { - return func(h Handler) Handler { - if h == nil { - return h - } - return HandlerFunc(func(ctx context.Context, e *scheduler.Event) error { - typename := strings.ToLower(e.GetType().String()) - return harness(func() error { return h.HandleEvent(ctx, e) }, typename) - }) - } -} From 2ef7721aa38f84029fe849678b8743211cef5197 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 18:32:18 +0000 Subject: [PATCH 39/67] scheduler/calls: eliminate Decorator support, replace with extras/scheduler/callrules --- Makefile | 1 + api/v1/cmd/example-scheduler/app/app.go | 39 +- api/v1/cmd/msh/msh.go | 13 +- api/v1/lib/extras/rules/rules.go | 317 +++++++--- .../lib/extras/scheduler/callrules/callers.go | 71 +++ .../callrules/callrules_generated.go | 302 ++++++++++ .../callrules/callrules_generated_test.go | 554 ++++++++++++++++++ api/v1/lib/extras/scheduler/callrules/gen.go | 3 + .../lib/extras/scheduler/callrules/metrics.go | 21 + .../eventrules/eventrules_generated.go | 2 + .../extras/scheduler/eventrules/handlers.go | 6 +- .../extras/scheduler/eventrules/metrics.go | 2 +- api/v1/lib/scheduler/calls/caller.go | 86 --- api/v1/lib/scheduler/calls/metrics.go | 28 - 14 files changed, 1226 insertions(+), 219 deletions(-) create mode 100644 api/v1/lib/extras/scheduler/callrules/callers.go create mode 100644 api/v1/lib/extras/scheduler/callrules/callrules_generated.go create mode 100644 api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go create mode 100644 api/v1/lib/extras/scheduler/callrules/gen.go create mode 100644 api/v1/lib/extras/scheduler/callrules/metrics.go delete mode 100644 api/v1/lib/scheduler/calls/metrics.go diff --git a/Makefile b/Makefile index d2142b4f..6158a7a9 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,7 @@ sync: .PHONY: generate generate: go generate ./api/v1/lib/extras/scheduler/eventrules + go generate ./api/v1/lib/extras/scheduler/callrules GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 3ba57aa9..61ce0a2f 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -11,6 +11,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" "github.com/mesos/mesos-go/api/v1/lib/extras/store" @@ -43,25 +44,25 @@ func Run(cfg Config) error { // probably tolerate X number of subsequent subscribe failures before bailing. we'll need // to track the lastCallAttempted along with subsequentSubscribeTimeouts. - frameworkIDStore := store.DecorateSingleton( + fidStore := store.DecorateSingleton( store.NewInMemorySingleton(), store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { log.Println("FrameworkID", v) return nil })) - state.cli = calls.Decorators{ - callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), + state.cli = callrules.Concat( + callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)), logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}), - calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), // automatically set the frameworkID for all outgoing calls - }.Apply(state.cli) + callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), + ).Caller(state.cli) err = controller.Run( ctx, buildFrameworkInfo(state.config), state.cli, - controller.WithEventHandler(buildEventHandler(state, frameworkIDStore)), - controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithEventHandler(buildEventHandler(state, fidStore)), + controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)), controller.WithRegistrationTokens( backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, ctx.Done()), ), @@ -85,12 +86,12 @@ func Run(cfg Config) error { } // buildEventHandler generates and returns a handler to process events received from the subscription. -func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { +func buildEventHandler(state *internalState, fidStore store.Singleton) events.Handler { // disable brief logs when verbose logs are enabled (there's no sense logging twice!) logger := controller.LogEvents().Unless(state.config.verbose) return eventrules.Concat( - eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), logAllEvents().If(state.config.verbose), + eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), controller.LiftErrors().DropOnError(), ).Handle(events.HandlerSet{ scheduler.Event_FAILURE: logger.HandleF(failure), @@ -98,7 +99,7 @@ func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) e scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), scheduler.Event_SUBSCRIBED: eventrules.Concat( logger, - controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout), + controller.TrackSubscription(fidStore, state.config.failoverTimeout), ), }) } @@ -288,23 +289,21 @@ func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics } // callMetrics logs metrics for every outgoing Mesos call -func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) calls.Decorator { +func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) callrules.Rule { timed := metricsAPI.callLatency if !timingMetrics { timed = nil } harness := xmetrics.NewHarness(metricsAPI.callCount, metricsAPI.callErrorCount, timed, clock) - return calls.CallerMetrics(harness) + return callrules.Metrics(harness) } // logCalls logs a specific message string when a particular call-type is observed -func logCalls(messages map[scheduler.Call_Type]string) calls.Decorator { - return func(caller calls.Caller) calls.Caller { - return calls.CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { - if message, ok := messages[c.GetType()]; ok { - log.Println(message) - } - return caller.Call(ctx, c) - }) +func logCalls(messages map[scheduler.Call_Type]string) callrules.Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch callrules.Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if message, ok := messages[c.GetType()]; ok { + log.Println(message) + } + return ch(ctx, c, r, err) } } diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 9df1b5e3..348a248a 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -21,6 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/offers" @@ -45,7 +46,7 @@ var ( CPUs = float64(0.010) Memory = float64(64) - frameworkIDStore store.Singleton + fidStore store.Singleton declineAndSuppress bool refuseSeconds = calls.RefuseSeconds(5 * time.Second) wantsResources mesos.Resources @@ -60,7 +61,7 @@ func init() { flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command") flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command") - frameworkIDStore = store.DecorateSingleton( + fidStore = store.DecorateSingleton( store.NewInMemorySingleton(), store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { log.Println("FrameworkID", v) @@ -106,9 +107,7 @@ func main() { func run() error { var ( ctx, cancel = context.WithCancel(context.Background()) - caller = calls.Decorators{ - calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), - }.Apply(buildClient()) + caller = callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)).Caller(buildClient()) ) return controller.Run( @@ -116,7 +115,7 @@ func run() error { &mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)}, caller, controller.WithEventHandler(buildEventHandler(caller)), - controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)), controller.WithSubscriptionTerminated(func(err error) { defer cancel() if err == io.EOF { @@ -136,7 +135,7 @@ func buildEventHandler(caller calls.Caller) events.Handler { logger := controller.LogEvents() return controller.LiftErrors().Handle(events.HandlerSet{ scheduler.Event_FAILURE: logger, - scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(frameworkIDStore, 0)}, + scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)}, scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), }) diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go index db80fe6d..954b0fad 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/rules/rules.go @@ -13,10 +13,12 @@ import ( type ( config struct { - Package string - Imports []string - EventType string - EventPrototype string + Package string + Imports []string + EventType string + EventPrototype string + ReturnType string + ReturnPrototype string } ) @@ -32,6 +34,39 @@ func (c *config) Set(s string) error { return nil } +func (c *config) ReturnVar(names ...string) string { + if c.ReturnType == "" || len(names) == 0 { + return "" + } + return "var " + strings.Join(names, ",") + " " + c.ReturnType +} + +func (c *config) ReturnArg(name string) string { + if c.ReturnType == "" { + return "" + } + if name == "" { + return c.ReturnType + } + if strings.HasSuffix(name, ",") { + return strings.TrimSpace(name[:len(name)-1]+" "+c.ReturnType) + ", " + } + return name + " " + c.ReturnType +} + +func (c *config) ReturnRef(name string) string { + if c.ReturnType == "" || name == "" { + return "" + } + if strings.HasSuffix(name, ",") { + if len(name) < 2 { + panic("expected ref name before comma") + } + return name[:len(name)-1] + ", " + } + return name +} + func main() { var ( c = config{ @@ -48,6 +83,8 @@ func main() { flag.StringVar(&c.Package, "package", c.Package, "destination package") flag.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") + flag.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") + flag.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") flag.StringVar(&output, "output", output, "path of the to-be-generated file") flag.Var(&c, "import", "packages to import") flag.Parse() @@ -64,6 +101,10 @@ func main() { } else { c.EventPrototype = c.EventType[1:] + "{}" } + if c.ReturnType != "" && c.ReturnPrototype == "" { + log.Fatal("return_prototype is required when return_type is set") + } + if output == "" { output = defaultOutput } @@ -79,14 +120,20 @@ func main() { log.Fatal(err) } defer f.Close() - rulesTemplate.Execute(f, c) + err = rulesTemplate.Execute(f, &c) + if err != nil { + log.Fatal(err) + } // unit test template f, err = os.Create(testOutput) if err != nil { log.Fatal(err) } - testTemplate.Execute(f, c) + err = testTemplate.Execute(f, &c) + if err != nil { + log.Fatal(err) + } } var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} @@ -99,8 +146,8 @@ import ( "fmt" "sync" {{range .Imports}} - {{ printf "%q" . }} -{{ end -}} + {{ printf "%q" . -}} +{{end}} ) type ( @@ -111,16 +158,16 @@ type ( // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. // Eval implementations SHOULD be safe to execute concurrently. - Eval(context.Context, {{.EventType}}, error, Chain) (context.Context, {{.EventType}}, error) + Eval(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) } // Rule is the functional adaptation of evaler. // A nil Rule is valid: it is Eval'd as a noop. - Rule func(context.Context, {{.EventType}}, error, Chain) (context.Context, {{.EventType}}, error) + Rule func(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, // no additional rules are processed. - Chain func(context.Context, {{.EventType}}, error) (context.Context, {{.EventType}}, error) + Chain func(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) // Rules is a list of rules to be processed, in order. Rules []Rule @@ -136,23 +183,23 @@ var ( _ = evaler(Rules{}) // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { - return ctx, e, err + chainIdentity = func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return ctx, e, {{.ReturnRef "z," -}} err } ) // Eval is a convenience func that processes a nil Rule as a noop. -func (r Rule) Eval(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { +func (r Rule) Eval(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { if r != nil { - return r(ctx, e, err, ch) + return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } - return ch(ctx, e, err) + return ch(ctx, e, {{.ReturnRef "z," -}} err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. -func (rs Rules) Eval(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { - return ch(rs.Chain()(ctx, e, err)) +func (rs Rules) Eval(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return ch(rs.Chain()(ctx, e, {{.ReturnRef "z," -}} err)) } // Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.EventType}}, error) @@ -161,8 +208,8 @@ func (rs Rules) Chain() Chain { if len(rs) == 0 { return chainIdentity } - return func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { - return rs[0].Eval(ctx, e, err, rs[1:].Chain()) + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return rs[0].Eval(ctx, e, {{.ReturnRef "z," -}} err, rs[1:].Chain()) } } @@ -269,16 +316,16 @@ func (r Rule) Once() Rule { return nil } var once sync.Once - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { ruleInvoked := false once.Do(func() { - ctx, e, err = r(ctx, e, err, ch) + ctx, e, {{.ReturnRef "z," -}} err = r(ctx, e, {{.ReturnRef "z," -}} err, ch) ruleInvoked = true }) if !ruleInvoked { - ctx, e, err = ch(ctx, e, err) + ctx, e, {{.ReturnRef "z," -}} err = ch(ctx, e, {{.ReturnRef "z," -}} err) } - return ctx, e, err + return ctx, e, {{.ReturnRef "z," -}} err } } @@ -288,15 +335,17 @@ func (r Rule) Poll(p <-chan struct{}) Rule { if p == nil || r == nil { return nil } - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { select { case <-p: // do something // TODO(jdef): optimization: if we detect the chan is closed, affect a state change // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, err, ch) + return r(ctx, e, {{.ReturnRef "z," -}} err, ch) + case <-ctx.Done(): + return ctx, e, {{.ReturnRef "z," -}} Error2(err, ctx.Err()) default: - return ch(ctx, e, err) + return ch(ctx, e, {{.ReturnRef "z," -}} err) } } } @@ -322,11 +371,11 @@ func (r Rule) EveryN(nthTime int) Rule { return false } ) - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { if forward() { - return r(ctx, e, err, ch) + return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } - return ch(ctx, e, err) + return ch(ctx, e, {{.ReturnRef "z," -}} err) } } @@ -335,17 +384,17 @@ func Drop() Rule { return Rule(nil).ThenDrop() } -// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.EventType}}, error) tuple as-is. +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) tuple as-is. func (r Rule) ThenDrop() Rule { - return func(ctx context.Context, e {{.EventType}}, err error, _ Chain) (context.Context, {{.EventType}}, error) { - return r.Eval(ctx, e, err, chainIdentity) + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, _ Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, chainIdentity) } } // Fail returns a Rule that injects the given error. func Fail(injected error) Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { - return ch(ctx, e, Error2(err, injected)) + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, injected)) } } @@ -357,11 +406,11 @@ func DropOnError() Rule { // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then // the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { if err != nil { - return ctx, e, err + return ctx, e, {{.ReturnRef "z," -}} err } - return r.Eval(ctx, e, err, ch) + return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, ch) } } @@ -377,12 +426,12 @@ func DropOnSuccess() Rule { } func (r Rule) DropOnSuccess() Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { if err == nil { // bypass remainder of chain - return ctx, e, err + return ctx, e, {{.ReturnRef "z," -}} err } - return r.Eval(ctx, e, err, ch) + return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, ch) } } @@ -402,44 +451,46 @@ import ( "reflect" "testing" {{range .Imports}} - {{ printf "%q" . }} -{{ end -}} + {{ printf "%q" . -}} +{{end}} ) func prototype() {{.EventType}} { return {{.EventPrototype}} } func counter(i *int) Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { *i++ - return ch(ctx, e, err) + return ch(ctx, e, {{.ReturnRef "z," -}} err) } } func tracer(r Rule, name string, t *testing.T) Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { t.Log("executing", name) - return r(ctx, e, err, ch) + return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } } func returnError(re error) Rule { - return func(ctx context.Context, e {{.EventType}}, err error, ch Chain) (context.Context, {{.EventType}}, error) { - return ch(ctx, e, Error2(err, re)) + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, re)) } } func chainCounter(i *int, ch Chain) Chain { - return func(ctx context.Context, e {{.EventType}}, err error) (context.Context, {{.EventType}}, error) { + return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { *i++ - return ch(ctx, e, err) + return ch(ctx, e, {{.ReturnRef "z," -}} err) } } func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) - - _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) +{{if .ReturnType}} + {{.ReturnVar "z0"}} +{{end}} + _, e, {{.ReturnRef "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.ReturnRef "z0," -}} nil, chainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -458,16 +509,29 @@ func TestRules(t *testing.T) { ctx = context.Background() ) + {{if .ReturnType -}} + {{.ReturnVar "z0"}} + var zp = {{.ReturnPrototype}} + {{end -}} + // multiple rules in Rules should execute, dropping nil rules along the way for _, tc := range []struct { e {{.EventType}} + {{if .ReturnType}} + {{- .ReturnArg "z "}} + {{end -}} err error }{ - {nil, nil}, - {nil, a}, - {p, nil}, - {p, a}, - } { + {nil, {{.ReturnRef "z0," -}} nil}, + {nil, {{.ReturnRef "z0," -}} a}, + {p, {{.ReturnRef "z0," -}} nil}, + {p, {{.ReturnRef "z0," -}} a}, +{{if .ReturnType}} + {nil, {{.ReturnRef "zp," -}} nil}, + {nil, {{.ReturnRef "zp," -}} a}, + {p, {{.ReturnRef "zp," -}} nil}, + {p, {{.ReturnRef "zp," -}} a}, +{{end}} } { var ( i int rule = Concat( @@ -477,11 +541,16 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) + _, e, {{.ReturnRef "zz," -}} err = rule(ctx, tc.e, {{.ReturnRef "tc.z," -}} tc.err, chainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } + {{if .ReturnType -}} + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + {{end -}} if err != tc.err { t.Errorf("expected %q error instead of %q", tc.err, err) } @@ -489,11 +558,16 @@ func TestRules(t *testing.T) { t.Error("expected 2 rule executions instead of", i) } - // empty Rules should not change event, err - _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) + // empty Rules should not change event, {{.ReturnRef "z," -}} err + _, e, {{.ReturnRef "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.ReturnRef "tc.z," -}} tc.err, chainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } + {{if .ReturnType -}} + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + {{end -}} if err != tc.err { t.Errorf("expected %q error instead of %q", tc.err, err) } @@ -553,11 +627,19 @@ func TestAndThen(t *testing.T) { r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != a { t.Error("unexpected error", err) } @@ -579,6 +661,9 @@ func TestOnFailure(t *testing.T) { r1 = counter(&i) r2 = Fail(a).OnFailure(counter(&i)) ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} for k, tc := range []struct { r Rule initialError error @@ -586,10 +671,15 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := tc.r(ctx, p, {{.ReturnRef "zp," -}} tc.initialError, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != a { t.Error("unexpected error", err) } @@ -611,13 +701,21 @@ func TestDropOnError(t *testing.T) { r2 = counter(&i).DropOnError() a = errors.New("a") ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != a { t.Error("unexpected error", err) } @@ -628,10 +726,15 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r2(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -648,13 +751,21 @@ func TestDropOnSuccess(t *testing.T) { r1 = counter(&i) r2 = counter(&i).DropOnSuccess() ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -666,10 +777,15 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r2(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != a { t.Error("unexpected error", err) } @@ -681,10 +797,15 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err = r3(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -705,12 +826,20 @@ func TestThenDrop(t *testing.T) { r1 = counter(&i) r2 = counter(&i).ThenDrop() ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != anErr { t.Errorf("expected %v instead of error %v", anErr, err) } @@ -733,13 +862,21 @@ func TestDrop(t *testing.T) { r1 = counter(&i) r2 = Rules{Drop(), counter(&i)}.Eval ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != anErr { t.Errorf("expected %v instead of error %v", anErr, err) } @@ -761,13 +898,21 @@ func TestIf(t *testing.T) { r1 = counter(&i).If(true).Eval r2 = counter(&i).If(false).Eval ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -788,13 +933,21 @@ func TestUnless(t *testing.T) { r1 = counter(&i).Unless(false).Eval r2 = counter(&i).Unless(true).Eval ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -815,12 +968,20 @@ func TestOnce(t *testing.T) { r1 = counter(&i).Once().Eval r2 = Rule(nil).Once().Eval ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Error("unexpected error", err) } @@ -859,12 +1020,20 @@ func TestPoll(t *testing.T) { r1 = counter(&i).Poll(tc.ch).Eval r2 = Rule(nil).Poll(tc.ch).Eval ) + {{if .ReturnType -}} + var zp = {{.ReturnPrototype}} + {{end -}} for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } + {{if .ReturnType -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} if err != nil { t.Errorf("test case %d failed: unexpected error %v", ti, err) } diff --git a/api/v1/lib/extras/scheduler/callrules/callers.go b/api/v1/lib/extras/scheduler/callrules/callers.go new file mode 100644 index 00000000..cfb8ab9a --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callers.go @@ -0,0 +1,71 @@ +package callrules + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" +) + +// Call returns a Rule that invokes the given Caller +func Call(caller calls.Caller) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c *scheduler.Call, _ mesos.Response, _ error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf calls.CallerFunc) Rule { + return Call(calls.Caller(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller calls.Caller) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf calls.CallerFunc) Rule { + return r.Caller(calls.Caller(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = calls.Caller(Rule(nil)) + _ = calls.Caller(Rules(nil)) +) + +// WithFrameworkID returns a Rule that injects a framework ID to outgoing calls, with the following exceptions: +// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) +// - calls are not modified when the detected framework ID is "" +func WithFrameworkID(frameworkID func() string) Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + // never overwrite framework ID for subscribe calls; the scheduler must do that part + if c.GetType() != scheduler.Call_SUBSCRIBE { + if fid := frameworkID(); fid != "" { + c2 := *c + c2.FrameworkID = &mesos.FrameworkID{Value: fid} + c = &c2 + } + } + return ch(ctx, c, r, err) + } +} diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go new file mode 100644 index 00000000..03b4472f --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -0,0 +1,302 @@ +package callrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *scheduler.Call, mesos.Response, error, Chain) (context.Context, *scheduler.Call, mesos.Response, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *scheduler.Call, mesos.Response, error, Chain) (context.Context, *scheduler.Call, mesos.Response, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *scheduler.Call, mesos.Response, error) (context.Context, *scheduler.Call, mesos.Response, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) + + // chainIdentity is a Chain that returns the arguments as its results. + chainIdentity = func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + return ctx, e, z, err + } +) + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if r != nil { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(rs.Chain()(ctx, e, z, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *scheduler.Call, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return chainIdentity + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, z, err = r(ctx, e, z, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, z, err = ch(ctx, e, z, err) + } + return ctx, e, z, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. +// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + if p == nil || r == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + select { + case <-p: + // do something + // TODO(jdef): optimization: if we detect the chan is closed, affect a state change + // whereby this select is no longer invoked (and always pass control to r). + return r(ctx, e, z, err, ch) + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return ch(ctx, e, z, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 || r == nil { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if forward() { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) + } +} + +// Drop aborts the Chain and returns the (context.Context, *scheduler.Call, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Call, mesos.Response, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, _ Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return r.Eval(ctx, e, z, err, chainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if err != nil { + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go new file mode 100644 index 00000000..7a618dc8 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -0,0 +1,554 @@ +package callrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func prototype() *scheduler.Call { return &scheduler.Call{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + t.Log("executing", name) + return r(ctx, e, z, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + var z0 mesos.Response + + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + var z0 mesos.Response + var zp = &mesos.ResponseWrapper{} + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *scheduler.Call + z mesos.Response + err error + }{ + {nil, z0, nil}, + {nil, z0, a}, + {p, z0, nil}, + {p, z0, a}, + + {nil, zp, nil}, + {nil, zp, a}, + {p, zp, nil}, + {p, zp, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, chainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, z, err + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, chainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + var zp = &mesos.ResponseWrapper{} + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestPoll(t *testing.T) { + var ( + ch1 <-chan struct{} // always nil + ch2 = make(chan struct{}) // non-nil, blocking + ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking + ch4 = make(chan struct{}) // non-nil, closed + ) + ch3 <- struct{}{} + close(ch4) + for ti, tc := range []struct { + ch <-chan struct{} + wantsRuleCount []int + }{ + {ch1, []int{0, 0, 0, 0}}, + {ch2, []int{0, 0, 0, 0}}, + {ch3, []int{1, 1, 1, 1}}, + {ch4, []int{1, 2, 2, 2}}, + } { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Poll(tc.ch).Eval + r2 = Rule(nil).Poll(tc.ch).Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 2; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := (k * 2) + x + 1; j != y { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", + ti, y, j) + } + } + } + } +} diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go new file mode 100644 index 00000000..99777a4b --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -0,0 +1,3 @@ +package callrules + +//go:generate go run ../../rules/rules.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} diff --git a/api/v1/lib/extras/scheduler/callrules/metrics.go b/api/v1/lib/extras/scheduler/callrules/metrics.go new file mode 100644 index 00000000..1adc1219 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/metrics.go @@ -0,0 +1,21 @@ +package callrules + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + typename := strings.ToLower(c.GetType().String()) + harness(func() error { + _, _, r, err = ch(ctx, c, r, err) + return err // need to count these + }, typename) + return ctx, c, r, err + } +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index c19affd3..d3e630ac 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -203,6 +203,8 @@ func (r Rule) Poll(p <-chan struct{}) Rule { // TODO(jdef): optimization: if we detect the chan is closed, affect a state change // whereby this select is no longer invoked (and always pass control to r). return r(ctx, e, err, ch) + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) default: return ch(ctx, e, err) } diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers.go index af60b9cf..0190f99b 100644 --- a/api/v1/lib/extras/scheduler/eventrules/handlers.go +++ b/api/v1/lib/extras/scheduler/eventrules/handlers.go @@ -7,7 +7,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" ) -// Handler generates a rule that executes the given handler. +// Handle generates a rule that executes the given handler. func Handle(h events.Handler) Rule { if h == nil { return nil @@ -19,11 +19,11 @@ func Handle(h events.Handler) Rule { } // HandleF is the functional equivalent of Handle -func HandleF(ctx context.Context, h events.HandlerFunc) Rule { +func HandleF(h events.HandlerFunc) Rule { return Handle(events.Handler(h)) } -// Handler returns a rule that invokes the given Handler +// Handle returns a Rule that invokes the receiver, then the given Handler func (r Rule) Handle(h events.Handler) Rule { return Rules{r, Handle(h)}.Eval } diff --git a/api/v1/lib/extras/scheduler/eventrules/metrics.go b/api/v1/lib/extras/scheduler/eventrules/metrics.go index 7713182f..14df3a7a 100644 --- a/api/v1/lib/extras/scheduler/eventrules/metrics.go +++ b/api/v1/lib/extras/scheduler/eventrules/metrics.go @@ -11,7 +11,7 @@ import ( func Metrics(harness metrics.Harness) Rule { return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { typename := strings.ToLower(e.GetType().String()) - err = harness(func() error { + harness(func() error { ctx, e, err = ch(ctx, e, err) return err }, typename) diff --git a/api/v1/lib/scheduler/calls/caller.go b/api/v1/lib/scheduler/calls/caller.go index 48ae2acf..0538476a 100644 --- a/api/v1/lib/scheduler/calls/caller.go +++ b/api/v1/lib/scheduler/calls/caller.go @@ -16,12 +16,6 @@ type ( // CallerFunc is the functional adaptation of the Caller interface CallerFunc func(context.Context, *scheduler.Call) (mesos.Response, error) - - // Decorator funcs usually return a Caller whose behavior has been somehow modified - Decorator func(Caller) Caller - - // Decorators is a convenience type that applies multiple Decorator functions to a Caller - Decorators []Decorator ) // Call implements the Caller interface for CallerFunc @@ -29,86 +23,6 @@ func (f CallerFunc) Call(ctx context.Context, c *scheduler.Call) (mesos.Response return f(ctx, c) } -// Apply is a convenient, nil-safe applicator that returns the result of d(c) iff d != nil; otherwise c -func (d Decorator) Apply(c Caller) (result Caller) { - if d != nil { - result = d(c) - } else { - result = c - } - return -} - -// Apply is a convenience function that applies the combined product of the decorators to the given Caller. -func (ds Decorators) Apply(c Caller) Caller { - return ds.Combine()(c) -} - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Caller that is ultimately returned. -func (ds Decorators) Combine() (result Decorator) { - actual := make(Decorators, 0, len(ds)) - for _, d := range ds { - if d != nil { - actual = append(actual, d) - } - } - if len(actual) == 0 { - result = noopDecorator - } else { - result = Decorator(func(h Caller) Caller { - for _, d := range actual { - h = d(h) - } - return h - }) - } - return -} - -// FrameworkCaller generates and returns a Decorator that applies the given frameworkID to all calls. -// Deprecated in favor of SubscribedCaller; should remove after v0.0.3. -func FrameworkCaller(frameworkID string) Decorator { - return func(h Caller) Caller { - return CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { - c.FrameworkID = &mesos.FrameworkID{Value: frameworkID} - return h.Call(ctx, c) - }) - } -} - -// SubscribedCaller returns a Decorator that injects a framework ID to all calls, with the following exceptions: -// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) -// - calls are not modified when the generated framework ID is "" -func SubscribedCaller(frameworkID func() string) Decorator { - return func(h Caller) Caller { - return CallerFunc(func(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { - // never overwrite framework ID for subscribe calls; the scheduler must do that part - if c.GetType() != scheduler.Call_SUBSCRIBE { - if fid := frameworkID(); fid != "" { - c.FrameworkID = &mesos.FrameworkID{Value: fid} - } - } - return h.Call(ctx, c) - }) - } -} - -var noopDecorator = Decorator(func(h Caller) Caller { return h }) - // CallNoData is a convenience func that executes the given Call using the provided Caller // and always drops the response data. func CallNoData(ctx context.Context, caller Caller, call *scheduler.Call) error { diff --git a/api/v1/lib/scheduler/calls/metrics.go b/api/v1/lib/scheduler/calls/metrics.go deleted file mode 100644 index c3779205..00000000 --- a/api/v1/lib/scheduler/calls/metrics.go +++ /dev/null @@ -1,28 +0,0 @@ -package calls - -// TODO(jdef): move this code to the extras tree, it doesn't belong here - -import ( - "context" - "strings" - - "github.com/mesos/mesos-go/api/v1/lib" - xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -func CallerMetrics(harness xmetrics.Harness) Decorator { - return func(caller Caller) (metricsCaller Caller) { - if caller != nil { - metricsCaller = CallerFunc(func(ctx context.Context, c *scheduler.Call) (res mesos.Response, err error) { - typename := strings.ToLower(c.GetType().String()) - harness(func() error { - res, err = caller.Call(ctx, c) - return err // need to count these - }, typename) - return - }) - } - return - } -} From ab4d4f044780e39bbdb4319991688c7dd31ad7c1 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 18:45:27 +0000 Subject: [PATCH 40/67] scheduler/events: remove Predicate and Handler decorators (in favor of eventrules) --- api/v1/lib/scheduler/events.go | 18 -------- api/v1/lib/scheduler/events/events.go | 52 ----------------------- api/v1/lib/scheduler/events/predicates.go | 12 ------ 3 files changed, 82 deletions(-) delete mode 100644 api/v1/lib/scheduler/events.go delete mode 100644 api/v1/lib/scheduler/events/predicates.go diff --git a/api/v1/lib/scheduler/events.go b/api/v1/lib/scheduler/events.go deleted file mode 100644 index 26bcea9e..00000000 --- a/api/v1/lib/scheduler/events.go +++ /dev/null @@ -1,18 +0,0 @@ -package scheduler - -// EventPredicate funcs evaluate scheduler events and determine some pass/fail outcome; useful for -// filtering or conditionally handling events. -type EventPredicate func(*Event) bool - -// Apply returns the result of the predicate function; always true if the predicate is nil. -func (ep EventPredicate) Apply(e *Event) (result bool) { - if ep == nil { - result = true - } else { - result = ep(e) - } - return -} - -// Predicate implements scheduler/events.Predicate -func (t Event_Type) Predicate() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index cd8ff241..2f405510 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -2,7 +2,6 @@ package events import ( "context" - "sync" "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) @@ -31,13 +30,6 @@ type ( // Option is a functional configuration option that returns an "undo" option that // reverts the change made by the option. Option func(*Mux) Option - - // Handlers aggregates Handler things - Handlers []Handler - - Predicate interface { - Predicate() scheduler.EventPredicate - } ) // HandleEvent implements Handler for HandlerFunc @@ -160,47 +152,3 @@ func DefaultHandler(eh Handler) Option { return DefaultHandler(old) } } - -// When -// Deprecated in favor of Rules. -func Once(h Handler) Handler { - var once sync.Once - return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { - once.Do(func() { - err = h.HandleEvent(ctx, e) - }) - return - }) -} - -// When -// Deprecated in favor of Rules. -func OnceFunc(h HandlerFunc) Handler { return Once(h) } - -// When -// Deprecated in favor of Rules. -func When(p Predicate, h Handler) Handler { - return HandlerFunc(func(ctx context.Context, e *scheduler.Event) (err error) { - if p.Predicate().Apply(e) { - err = h.HandleEvent(ctx, e) - } - return - }) -} - -// WhenFunc -// Deprecated in favor of Rules. -func WhenFunc(p Predicate, h HandlerFunc) Handler { return When(p, h) } - -// HandleEvent implements Handler for Handlers. -// Deprecated in favor of Rules. -func (hs Handlers) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { - for _, h := range hs { - if h != nil { - if err = h.HandleEvent(ctx, e); err != nil { - break - } - } - } - return -} diff --git a/api/v1/lib/scheduler/events/predicates.go b/api/v1/lib/scheduler/events/predicates.go deleted file mode 100644 index 6aefb64a..00000000 --- a/api/v1/lib/scheduler/events/predicates.go +++ /dev/null @@ -1,12 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -type PredicateBool func() bool - -// Predicate implements scheduler.events.Predicate -func (b PredicateBool) Predicate() scheduler.EventPredicate { - return func(_ *scheduler.Event) bool { return b() } -} From c834bab1102de1d29f724b0e7b4a2c7340415a26 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 18:47:48 +0000 Subject: [PATCH 41/67] scheduler/event: renamed events.go to handlers.go --- api/v1/lib/scheduler/events/{events.go => handlers.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename api/v1/lib/scheduler/events/{events.go => handlers.go} (100%) diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/handlers.go similarity index 100% rename from api/v1/lib/scheduler/events/events.go rename to api/v1/lib/scheduler/events/handlers.go From 6e58eb487d8667fede3c5ac2eda03169a22f2b6a Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 19:02:43 +0000 Subject: [PATCH 42/67] scheduler/events: remove Mux and Options (alleviate API bloat) --- api/v1/lib/scheduler/events/handlers.go | 124 ++---------------------- 1 file changed, 7 insertions(+), 117 deletions(-) diff --git a/api/v1/lib/scheduler/events/handlers.go b/api/v1/lib/scheduler/events/handlers.go index 2f405510..44455852 100644 --- a/api/v1/lib/scheduler/events/handlers.go +++ b/api/v1/lib/scheduler/events/handlers.go @@ -18,18 +18,6 @@ type ( HandlerSet map[scheduler.Event_Type]Handler HandlerFuncSet map[scheduler.Event_Type]HandlerFunc - - // Mux maps event types to Handlers (only one Handler for each type). A "default" - // Handler implementation may be provided to handle cases in which there is no - // registered Handler for specific event type. - Mux struct { - handlers HandlerSet - defaultHandler Handler - } - - // Option is a functional configuration option that returns an "undo" option that - // reverts the change made by the option. - Option func(*Mux) Option ) // HandleEvent implements Handler for HandlerFunc @@ -39,116 +27,18 @@ func NoopHandler() HandlerFunc { return func(_ context.Context, _ *scheduler.Event) error { return nil } } -// NewMux generates and returns a new, empty Mux instance. -func NewMux(opts ...Option) *Mux { - m := &Mux{ - handlers: make(HandlerSet), - } - m.With(opts...) - return m -} - -// With applies the given options to the Mux and returns the result of invoking -// the last Option func. If no options are provided then a no-op Option is returned. -func (m *Mux) With(opts ...Option) Option { - var last Option // defaults to noop - last = Option(func(x *Mux) Option { return last }) - - for _, o := range opts { - if o != nil { - last = o(m) - } - } - return last -} - -// HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(ctx context.Context, e *scheduler.Event) error { - ok, err := m.handlers.tryHandleEvent(ctx, e) - if ok { - return err - } - if m.defaultHandler != nil { - return m.defaultHandler.HandleEvent(ctx, e) - } - return nil -} - -// Handle returns an option that configures a Handler to handle a specific event type. -// If the specified Handler is nil then any currently registered Handler for the given -// event type is deleted upon application of the returned Option. -func Handle(et scheduler.Event_Type, eh Handler) Option { - return func(m *Mux) Option { - old := m.handlers[et] - if eh == nil { - delete(m.handlers, et) - } else { - m.handlers[et] = eh - } - return Handle(et, old) - } -} - // HandleEvent implements Handler for HandlerSet func (hs HandlerSet) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { - _, err = hs.tryHandleEvent(ctx, e) - return -} - -// tryHandleEvent returns true if the event was handled by a member of the HandlerSet -func (hs HandlerSet) tryHandleEvent(ctx context.Context, e *scheduler.Event) (bool, error) { if h := hs[e.GetType()]; h != nil { - return true, h.HandleEvent(ctx, e) + return h.HandleEvent(ctx, e) } - return false, nil -} - -// Map returns an Option that configures multiple Handler objects. -func (handlers HandlerSet) ToOption() (option Option) { - option = func(m *Mux) Option { - type history struct { - et scheduler.Event_Type - h Handler - } - old := make([]history, len(handlers)) - for et, h := range handlers { - old = append(old, history{et, m.handlers[et]}) - m.handlers[et] = h - } - return func(m *Mux) Option { - for i := range old { - if old[i].h == nil { - delete(m.handlers, old[i].et) - } else { - m.handlers[old[i].et] = old[i].h - } - } - return option - } - } - return -} - -// HandlerSet converts a HandlerFuncSet -func (handlers HandlerFuncSet) HandlerSet() HandlerSet { - h := make(HandlerSet, len(handlers)) - for k, v := range handlers { - h[k] = v - } - return h -} - -// ToOption converts a HandlerFuncSet -func (hs HandlerFuncSet) ToOption() (option Option) { - return hs.HandlerSet().ToOption() + return nil } -// DefaultHandler returns an option that configures the default handler that's invoked -// in cases where there is no Handler registered for specific event type. -func DefaultHandler(eh Handler) Option { - return func(m *Mux) Option { - old := m.defaultHandler - m.defaultHandler = eh - return DefaultHandler(old) +// HandleEvent implements Handler for HandlerFuncSet +func (hs HandlerFuncSet) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) } + return nil } From 1ad848cbf9332202f27f89eb80a027beb3a0e695 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 19:40:12 +0000 Subject: [PATCH 43/67] scheduler/events: rename HandlerSet and HandlerFuncSet to Handlers, HandlerFuncs; add Otherwise decorator --- api/v1/cmd/example-scheduler/app/app.go | 4 +- api/v1/cmd/msh/msh.go | 4 +- api/v1/lib/scheduler/events/handlers.go | 56 +++++++++++++++++++++---- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 61ce0a2f..0368e62d 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -93,7 +93,7 @@ func buildEventHandler(state *internalState, fidStore store.Singleton) events.Ha logAllEvents().If(state.config.verbose), eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), controller.LiftErrors().DropOnError(), - ).Handle(events.HandlerSet{ + ).Handle(events.Handlers{ scheduler.Event_FAILURE: logger.HandleF(failure), scheduler.Event_OFFERS: trackOffersReceived(state).HandleF(resourceOffers(state)), scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), @@ -101,7 +101,7 @@ func buildEventHandler(state *internalState, fidStore store.Singleton) events.Ha logger, controller.TrackSubscription(fidStore, state.config.failoverTimeout), ), - }) + }.Otherwise(logger.HandleEvent)) } func trackOffersReceived(state *internalState) eventrules.Rule { diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index 348a248a..fb390759 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -133,12 +133,12 @@ func buildClient() calls.Caller { func buildEventHandler(caller calls.Caller) events.Handler { logger := controller.LogEvents() - return controller.LiftErrors().Handle(events.HandlerSet{ + return controller.LiftErrors().Handle(events.Handlers{ scheduler.Event_FAILURE: logger, scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)}, scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), - }) + }.Otherwise(logger.HandleEvent)) } func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { diff --git a/api/v1/lib/scheduler/events/handlers.go b/api/v1/lib/scheduler/events/handlers.go index 44455852..c1b2dae5 100644 --- a/api/v1/lib/scheduler/events/handlers.go +++ b/api/v1/lib/scheduler/events/handlers.go @@ -16,29 +16,67 @@ type ( // HandlerFunc is a functional adaptation of the Handler interface HandlerFunc func(context.Context, *scheduler.Event) error - HandlerSet map[scheduler.Event_Type]Handler - HandlerFuncSet map[scheduler.Event_Type]HandlerFunc + // Handlers executes an event Handler according to the event's type + Handlers map[scheduler.Event_Type]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[scheduler.Event_Type]HandlerFunc ) // HandleEvent implements Handler for HandlerFunc func (f HandlerFunc) HandleEvent(ctx context.Context, e *scheduler.Event) error { return f(ctx, e) } -func NoopHandler() HandlerFunc { - return func(_ context.Context, _ *scheduler.Event) error { return nil } -} +var noopHandler = func(_ context.Context, _ *scheduler.Event) error { return nil } -// HandleEvent implements Handler for HandlerSet -func (hs HandlerSet) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { +// NoopHandler returns a HandlerFunc that does nothing and always returns nil +func NoopHandler() HandlerFunc { return noopHandler } + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } return nil } -// HandleEvent implements Handler for HandlerFuncSet -func (hs HandlerFuncSet) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } return nil } + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *scheduler.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *scheduler.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) From 912cf3618c1bafb3695aba5f8b5a3dc14c5b4650 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 20:18:27 +0000 Subject: [PATCH 44/67] require go1.7 or higher in order to use context.Context --- .travis.yml | 1 - README.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ff65fc0..44fc65e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ language: go git: submodules: false go: - - 1.6.x - 1.7.x - 1.8 before_install: diff --git a/README.md b/README.md index 78f8735e..e0758c55 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ The Mesos v0 API version of the bindings, located in `api/v0`, are more mature b - Modular design for easy readability/extensibility ### Pre-Requisites -- Go 1.6 or higher +- Go 1.7 or higher - A standard and working Go workspace setup - Apache Mesos 1.0 or newer From 64e83b2c2646881a643c0b99f718ee597fda2ffd Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 21:18:35 +0000 Subject: [PATCH 45/67] generate: modularize code generator- separate templates from generator --- Makefile | 4 +- api/v1/lib/extras/gen/gen.go | 168 ++++++++++++++++++ api/v1/lib/extras/{rules => gen}/rules.go | 133 +------------- .../callrules/callrules_generated.go | 2 +- .../callrules/callrules_generated_test.go | 2 +- api/v1/lib/extras/scheduler/callrules/gen.go | 2 +- .../eventrules/eventrules_generated.go | 2 +- .../eventrules/eventrules_generated_test.go | 2 +- api/v1/lib/extras/scheduler/eventrules/gen.go | 2 +- 9 files changed, 180 insertions(+), 137 deletions(-) create mode 100644 api/v1/lib/extras/gen/gen.go rename api/v1/lib/extras/{rules => gen}/rules.go (89%) diff --git a/Makefile b/Makefile index 6158a7a9..a9d6d8cb 100644 --- a/Makefile +++ b/Makefile @@ -79,8 +79,8 @@ sync: .PHONY: generate generate: - go generate ./api/v1/lib/extras/scheduler/eventrules - go generate ./api/v1/lib/extras/scheduler/callrules + go generate -x ./api/v1/lib/extras/scheduler/eventrules + go generate -x ./api/v1/lib/extras/scheduler/callrules GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go new file mode 100644 index 00000000..7f99bf0c --- /dev/null +++ b/api/v1/lib/extras/gen/gen.go @@ -0,0 +1,168 @@ +// +build ignore + +package main + +import ( + "errors" + "flag" + "fmt" + "log" + "os" + "strings" + "text/template" +) + +type Config struct { + Package string + Imports []string + EventType string + EventPrototype string + ReturnType string + ReturnPrototype string + Args string // arguments that we were invoked with +} + +func (c *Config) String() string { + if c == nil { + return "" + } + return fmt.Sprintf("%#v", ([]string)(c.Imports)) +} + +func (c *Config) Set(s string) error { + c.Imports = append(c.Imports, s) + return nil +} + +func (c *Config) ReturnVar(names ...string) string { + if c.ReturnType == "" || len(names) == 0 { + return "" + } + return "var " + strings.Join(names, ",") + " " + c.ReturnType +} + +func (c *Config) ReturnArg(name string) string { + if c.ReturnType == "" { + return "" + } + if name == "" { + return c.ReturnType + } + if strings.HasSuffix(name, ",") { + return strings.TrimSpace(name[:len(name)-1]+" "+c.ReturnType) + ", " + } + return name + " " + c.ReturnType +} + +func (c *Config) ReturnRef(name string) string { + if c.ReturnType == "" || name == "" { + return "" + } + if strings.HasSuffix(name, ",") { + if len(name) < 2 { + panic("expected ref name before comma") + } + return name[:len(name)-1] + ", " + } + return name +} + +func (c *Config) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&c.Package, "package", c.Package, "destination package") + fs.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") + fs.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") + fs.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") + fs.Var(c, "import", "packages to import") +} + +func NewConfig() *Config { + var ( + c = Config{ + Package: os.Getenv("GOPACKAGE"), + EventType: "Event", + } + ) + return &c +} + +func Run(src, test *template.Template, args ...string) { + if len(args) < 1 { + panic(errors.New("expected at least one arg")) + } + var ( + c = NewConfig() + defaultOutput = "foo.go" + output string + ) + if c.Package != "" { + defaultOutput = c.Package + "_generated.go" + } + + fs := flag.NewFlagSet(args[0], flag.PanicOnError) + fs.StringVar(&output, "output", output, "path of the to-be-generated file") + c.AddFlags(fs) + + if err := fs.Parse(args[1:]); err != nil { + if err == flag.ErrHelp { + fs.PrintDefaults() + } + panic(err) + } + + c.Args = strings.Join(args[1:], " ") + + if c.Package == "" { + c.Package = "foo" + } + if c.EventType == "" { + c.EventType = "Event" + c.EventPrototype = "Event{}" + } else if strings.HasPrefix(c.EventType, "*") { + // TODO(jdef) don't assume that event type is a struct or *struct + c.EventPrototype = "&" + c.EventType[1:] + "{}" + } else { + c.EventPrototype = c.EventType[1:] + "{}" + } + if c.ReturnType != "" && c.ReturnPrototype == "" { + panic(errors.New("return_prototype is required when return_type is set")) + } + + if output == "" { + output = defaultOutput + } + + genmap := make(map[string]*template.Template) + if src != nil { + genmap[output] = src + } + if test != nil { + testOutput := output + "_test" + if strings.HasSuffix(output, ".go") { + testOutput = output[:len(output)-3] + "_test.go" + } + genmap[testOutput] = test + } + if len(genmap) == 0 { + panic(errors.New("neither src or test templates were provided")) + } + + Generate(genmap, c, func(err error) { panic(err) }) +} + +func Generate(items map[string]*template.Template, data interface{}, eh func(error)) { + for filename, t := range items { + func() { + f, err := os.Create(filename) + if err != nil { + eh(err) + return + } + defer f.Close() + log.Println("generating file", filename) + err = t.Execute(f, data) + if err != nil { + eh(err) + } + }() + } +} diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/gen/rules.go similarity index 89% rename from api/v1/lib/extras/rules/rules.go rename to api/v1/lib/extras/gen/rules.go index 954b0fad..8cd9ebe0 100644 --- a/api/v1/lib/extras/rules/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -3,142 +3,17 @@ package main import ( - "flag" - "fmt" - "log" "os" - "strings" "text/template" ) -type ( - config struct { - Package string - Imports []string - EventType string - EventPrototype string - ReturnType string - ReturnPrototype string - } -) - -func (c *config) String() string { - if c == nil { - return "" - } - return fmt.Sprintf("%#v", ([]string)(c.Imports)) -} - -func (c *config) Set(s string) error { - c.Imports = append(c.Imports, s) - return nil -} - -func (c *config) ReturnVar(names ...string) string { - if c.ReturnType == "" || len(names) == 0 { - return "" - } - return "var " + strings.Join(names, ",") + " " + c.ReturnType -} - -func (c *config) ReturnArg(name string) string { - if c.ReturnType == "" { - return "" - } - if name == "" { - return c.ReturnType - } - if strings.HasSuffix(name, ",") { - return strings.TrimSpace(name[:len(name)-1]+" "+c.ReturnType) + ", " - } - return name + " " + c.ReturnType -} - -func (c *config) ReturnRef(name string) string { - if c.ReturnType == "" || name == "" { - return "" - } - if strings.HasSuffix(name, ",") { - if len(name) < 2 { - panic("expected ref name before comma") - } - return name[:len(name)-1] + ", " - } - return name -} - func main() { - var ( - c = config{ - Package: os.Getenv("GOPACKAGE"), - EventType: "Event", - } - defaultOutput = "foo.go" - ) - if c.Package != "" { - defaultOutput = c.Package + "_generated.go" - } - - output := defaultOutput - - flag.StringVar(&c.Package, "package", c.Package, "destination package") - flag.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") - flag.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") - flag.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") - flag.StringVar(&output, "output", output, "path of the to-be-generated file") - flag.Var(&c, "import", "packages to import") - flag.Parse() - - if c.Package == "" { - c.Package = "foo" - } - if c.EventType == "" { - c.EventType = "Event" - c.EventPrototype = "Event{}" - } else if strings.HasPrefix(c.EventType, "*") { - // TODO(jdef) don't assume that event type is a struct or *struct - c.EventPrototype = "&" + c.EventType[1:] + "{}" - } else { - c.EventPrototype = c.EventType[1:] + "{}" - } - if c.ReturnType != "" && c.ReturnPrototype == "" { - log.Fatal("return_prototype is required when return_type is set") - } - - if output == "" { - output = defaultOutput - } - - testOutput := output + "_test" - if strings.HasSuffix(output, ".go") { - testOutput = output[:len(output)-3] + "_test.go" - } - - // main template - f, err := os.Create(output) - if err != nil { - log.Fatal(err) - } - defer f.Close() - err = rulesTemplate.Execute(f, &c) - if err != nil { - log.Fatal(err) - } - - // unit test template - f, err = os.Create(testOutput) - if err != nil { - log.Fatal(err) - } - err = testTemplate.Execute(f, &c) - if err != nil { - log.Fatal(err) - } + Run(rulesTemplate, rulesTestTemplate, os.Args...) } var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} -// go generate +// go generate {{.Args}} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( @@ -440,9 +315,9 @@ func (r Rule) OnFailure(next ...Rule) Rule { } `)) -var testTemplate = template.Must(template.New("").Parse(`package {{.Package}} +var rulesTestTemplate = template.Must(template.New("").Parse(`package {{.Package}} -// go generate +// go generate {{.Args}} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 03b4472f..55d0f3e7 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -1,6 +1,6 @@ package callrules -// go generate +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 7a618dc8..060cb9fb 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -1,6 +1,6 @@ package callrules -// go generate +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go index 99777a4b..dcadf5f4 100644 --- a/api/v1/lib/extras/scheduler/callrules/gen.go +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -1,3 +1,3 @@ package callrules -//go:generate go run ../../rules/rules.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index d3e630ac..db38dc8e 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -1,6 +1,6 @@ package eventrules -// go generate +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 0a0c0f52..4a96815c 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -1,6 +1,6 @@ package eventrules -// go generate +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go index 44a4854c..b950626c 100644 --- a/api/v1/lib/extras/scheduler/eventrules/gen.go +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -1,3 +1,3 @@ package eventrules -//go:generate go run ../../rules/rules.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event From 95a5cd09d381788f6f4f7158e9cdcf62bebf2238 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 3 Jun 2017 21:28:59 +0000 Subject: [PATCH 46/67] scheduler/events: code generation for Handler API --- Makefile | 1 + api/v1/lib/extras/gen/handlers.go | 100 ++++++++++++++++++ .../{handlers.go => events_generated.go} | 3 + api/v1/lib/scheduler/events/gen.go | 3 + 4 files changed, 107 insertions(+) create mode 100644 api/v1/lib/extras/gen/handlers.go rename api/v1/lib/scheduler/events/{handlers.go => events_generated.go} (94%) create mode 100644 api/v1/lib/scheduler/events/gen.go diff --git a/Makefile b/Makefile index a9d6d8cb..15a2d790 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ sync: generate: go generate -x ./api/v1/lib/extras/scheduler/eventrules go generate -x ./api/v1/lib/extras/scheduler/callrules + go generate -x ./api/v1/lib/scheduler/events GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) diff --git a/api/v1/lib/extras/gen/handlers.go b/api/v1/lib/extras/gen/handlers.go new file mode 100644 index 00000000..fdc40f29 --- /dev/null +++ b/api/v1/lib/extras/gen/handlers.go @@ -0,0 +1,100 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +type ( + // Handler is invoked upon the occurrence of some scheduler event that is generated + // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) + Handler interface { + HandleEvent(context.Context, {{.EventType}}) error + } + + // HandlerFunc is a functional adaptation of the Handler interface + HandlerFunc func(context.Context, {{.EventType}}) error + + // Handlers executes an event Handler according to the event's type + Handlers map[scheduler.Event_Type]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[scheduler.Event_Type]HandlerFunc +) + +// HandleEvent implements Handler for HandlerFunc +func (f HandlerFunc) HandleEvent(ctx context.Context, e {{.EventType}}) error { return f(ctx, e) } + +var noopHandler = func(_ context.Context, _ {{.EventType}}) error { return nil } + +// NoopHandler returns a HandlerFunc that does nothing and always returns nil +func NoopHandler() HandlerFunc { return noopHandler } + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e {{.EventType}}) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e {{.EventType}}) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e {{.EventType}}) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e {{.EventType}}) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) +`)) diff --git a/api/v1/lib/scheduler/events/handlers.go b/api/v1/lib/scheduler/events/events_generated.go similarity index 94% rename from api/v1/lib/scheduler/events/handlers.go rename to api/v1/lib/scheduler/events/events_generated.go index c1b2dae5..5727f10c 100644 --- a/api/v1/lib/scheduler/events/handlers.go +++ b/api/v1/lib/scheduler/events/events_generated.go @@ -1,5 +1,8 @@ package events +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +// GENERATED CODE FOLLOWS; DO NOT EDIT. + import ( "context" diff --git a/api/v1/lib/scheduler/events/gen.go b/api/v1/lib/scheduler/events/gen.go new file mode 100644 index 00000000..62339cd4 --- /dev/null +++ b/api/v1/lib/scheduler/events/gen.go @@ -0,0 +1,3 @@ +package events + +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event From 26d4abad6f68b93f6c94358a8cd7f6475fc5355c Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 01:52:12 +0000 Subject: [PATCH 47/67] executor/events: auto-generated event handler API --- Makefile | 1 + api/v1/cmd/example-executor/main.go | 39 ++++---- api/v1/lib/executor/events/decorators.go | 66 -------------- api/v1/lib/executor/events/events.go | 91 ------------------- .../lib/executor/events/events_generated.go | 85 +++++++++++++++++ api/v1/lib/executor/events/gen.go | 3 + api/v1/lib/extras/gen/gen.go | 75 +++++++++++++-- api/v1/lib/extras/gen/handlers.go | 4 +- .../lib/scheduler/events/events_generated.go | 2 +- api/v1/lib/scheduler/events/gen.go | 2 +- 10 files changed, 181 insertions(+), 187 deletions(-) delete mode 100644 api/v1/lib/executor/events/decorators.go delete mode 100644 api/v1/lib/executor/events/events.go create mode 100644 api/v1/lib/executor/events/events_generated.go create mode 100644 api/v1/lib/executor/events/gen.go diff --git a/Makefile b/Makefile index 15a2d790..a79627f6 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ sync: generate: go generate -x ./api/v1/lib/extras/scheduler/eventrules go generate -x ./api/v1/lib/extras/scheduler/callrules + go generate -x ./api/v1/lib/executor/events go generate -x ./api/v1/lib/scheduler/events GOPKG := github.com/mesos/mesos-go diff --git a/api/v1/cmd/example-executor/main.go b/api/v1/cmd/example-executor/main.go index c4f5ad4b..6ed236f3 100644 --- a/api/v1/cmd/example-executor/main.go +++ b/api/v1/cmd/example-executor/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "io" "log" @@ -139,54 +140,58 @@ func unacknowledgedUpdates(state *internalState) executor.CallOpt { } func eventLoop(state *internalState, decoder encoding.Decoder, h events.Handler) (err error) { + ctx := context.TODO() for err == nil && !state.shouldQuit { // housekeeping sendFailedTasks(state) var e executor.Event if err = decoder.Decode(&e); err == nil { - err = h.HandleEvent(&e) + err = h.HandleEvent(ctx, &e) } } return err } func buildEventHandler(state *internalState) events.Handler { - return events.NewMux( - events.Handle(executor.Event_SUBSCRIBED, events.HandlerFunc(func(e *executor.Event) error { + return events.HandlerFuncs{ + executor.Event_SUBSCRIBED: func(_ context.Context, e *executor.Event) error { log.Println("SUBSCRIBED") state.framework = e.Subscribed.FrameworkInfo state.executor = e.Subscribed.ExecutorInfo state.agent = e.Subscribed.AgentInfo return nil - })), - events.Handle(executor.Event_LAUNCH, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_LAUNCH: func(_ context.Context, e *executor.Event) error { launch(state, e.Launch.Task) return nil - })), - events.Handle(executor.Event_KILL, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_KILL: func(_ context.Context, e *executor.Event) error { log.Println("warning: KILL not implemented") return nil - })), - events.Handle(executor.Event_ACKNOWLEDGED, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_ACKNOWLEDGED: func(_ context.Context, e *executor.Event) error { delete(state.unackedTasks, e.Acknowledged.TaskID) delete(state.unackedUpdates, string(e.Acknowledged.UUID)) return nil - })), - events.Handle(executor.Event_MESSAGE, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_MESSAGE: func(_ context.Context, e *executor.Event) error { log.Printf("MESSAGE: received %d bytes of message data", len(e.Message.Data)) return nil - })), - events.Handle(executor.Event_SHUTDOWN, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_SHUTDOWN: func(_ context.Context, e *executor.Event) error { log.Println("SHUTDOWN received") state.shouldQuit = true return nil - })), - events.Handle(executor.Event_ERROR, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_ERROR: func(_ context.Context, e *executor.Event) error { log.Println("ERROR received") return errMustAbort - })), - ) + }, + }.Otherwise(func(_ context.Context, e *executor.Event) error { + log.Fatal("unexpected event", e) + return nil + }) } func sendFailedTasks(state *internalState) { diff --git a/api/v1/lib/executor/events/decorators.go b/api/v1/lib/executor/events/decorators.go deleted file mode 100644 index 4eb31eb5..00000000 --- a/api/v1/lib/executor/events/decorators.go +++ /dev/null @@ -1,66 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/executor" -) - -type ( - // Decorator functions typically modify behavior of the given delegate Handler. - Decorator func(Handler) Handler - - // Decorators aggregates Decorator functions - Decorators []Decorator -) - -var noopDecorator = Decorator(func(h Handler) Handler { return h }) - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// When returns a Decorator that evaluates the bool func every time the Handler is invoked. -// When f returns true, the Decorated Handler is invoked, otherwise the original Handler is. -func (d Decorator) When(f func() bool) Decorator { - if d == nil || f == nil { - return noopDecorator - } - return func(h Handler) Handler { - // generates a new decorator every time the Decorator func is invoked. - // probably OK for now. - decorated := d(h) - return HandlerFunc(func(e *executor.Event) (err error) { - if f() { - // let the decorated handler process this - err = decorated.HandleEvent(e) - } else { - err = h.HandleEvent(e) - } - return - }) - } -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Handler that is ultimately returned. -func (ds Decorators) Apply(h Handler) Handler { - for _, d := range ds { - h = d.Apply(h) - } - return h -} - -func (d Decorator) Apply(h Handler) Handler { - if d != nil { - h = d(h) - } - return h -} diff --git a/api/v1/lib/executor/events/events.go b/api/v1/lib/executor/events/events.go deleted file mode 100644 index 1e6d68c2..00000000 --- a/api/v1/lib/executor/events/events.go +++ /dev/null @@ -1,91 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/executor" -) - -type ( - // Handler is invoked upon the occurrence of some executor event that is generated - // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) - Handler interface { - HandleEvent(*executor.Event) error - } - - // HandlerFunc is a functional adaptation of the Handler interface - HandlerFunc func(*executor.Event) error - - // Mux maps event types to Handlers (only one Handler for each type). A "default" - // Handler implementation may be provided to handle cases in which there is no - // registered Handler for specific event type. - Mux struct { - handlers map[executor.Event_Type]Handler - defaultHandler Handler - } - - // Option is a functional configuration option that returns an "undo" option that - // reverts the change made by the option. - Option func(*Mux) Option -) - -// HandleEvents implements Handler for HandlerFunc -func (f HandlerFunc) HandleEvent(e *executor.Event) error { return f(e) } - -// NewMux generates and returns a new, empty Mux instance. -func NewMux(opts ...Option) *Mux { - m := &Mux{ - handlers: make(map[executor.Event_Type]Handler), - } - m.With(opts...) - return m -} - -// With applies the given options to the Mux and returns the result of invoking -// the last Option func. If no options are provided then a no-op Option is returned. -func (m *Mux) With(opts ...Option) Option { - var last Option // defaults to noop - last = Option(func(x *Mux) Option { return last }) - - for _, o := range opts { - if o != nil { - last = o(m) - } - } - return last -} - -// HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *executor.Event) (err error) { - h, found := m.handlers[e.GetType()] - if !found { - h = m.defaultHandler - } - if h != nil { - err = h.HandleEvent(e) - } - return -} - -// Handle returns an option that configures a Handler to handle a specific event type. -// If the specified Handler is nil then any currently registered Handler for the given -// event type is deleted upon application of the returned Option. -func Handle(et executor.Event_Type, eh Handler) Option { - return func(m *Mux) Option { - old := m.handlers[et] - if eh == nil { - delete(m.handlers, et) - } else { - m.handlers[et] = eh - } - return Handle(et, old) - } -} - -// DefaultHandler returns an option that configures the default handler that's invoked -// in cases where there is no Handler registered for specific event type. -func DefaultHandler(eh Handler) Option { - return func(m *Mux) Option { - old := m.defaultHandler - m.defaultHandler = eh - return DefaultHandler(old) - } -} diff --git a/api/v1/lib/executor/events/events_generated.go b/api/v1/lib/executor/events/events_generated.go new file mode 100644 index 00000000..7ea7bdb9 --- /dev/null +++ b/api/v1/lib/executor/events/events_generated.go @@ -0,0 +1,85 @@ +package events + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -event_type *executor.Event -type ET:executor.Event_Type +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + // Handler is invoked upon the occurrence of some scheduler event that is generated + // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) + Handler interface { + HandleEvent(context.Context, *executor.Event) error + } + + // HandlerFunc is a functional adaptation of the Handler interface + HandlerFunc func(context.Context, *executor.Event) error + + // Handlers executes an event Handler according to the event's type + Handlers map[executor.Event_Type]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[executor.Event_Type]HandlerFunc +) + +// HandleEvent implements Handler for HandlerFunc +func (f HandlerFunc) HandleEvent(ctx context.Context, e *executor.Event) error { return f(ctx, e) } + +var noopHandler = func(_ context.Context, _ *executor.Event) error { return nil } + +// NoopHandler returns a HandlerFunc that does nothing and always returns nil +func NoopHandler() HandlerFunc { return noopHandler } + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *executor.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *executor.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) diff --git a/api/v1/lib/executor/events/gen.go b/api/v1/lib/executor/events/gen.go new file mode 100644 index 00000000..b28a7934 --- /dev/null +++ b/api/v1/lib/executor/events/gen.go @@ -0,0 +1,3 @@ +package events + +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -event_type *executor.Event -type ET:executor.Event_Type diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go index 7f99bf0c..b6bbf686 100644 --- a/api/v1/lib/extras/gen/gen.go +++ b/api/v1/lib/extras/gen/gen.go @@ -12,15 +12,26 @@ import ( "text/template" ) -type Config struct { - Package string - Imports []string - EventType string - EventPrototype string - ReturnType string - ReturnPrototype string - Args string // arguments that we were invoked with -} +type ( + Type struct { + Notation string + Spec string + Prototype string + } + + TypeMap map[string]Type + + Config struct { + Package string + Imports []string + EventType string // Deprecated in favor of Types + EventPrototype string // Deprecated in favor of Types + ReturnType string + ReturnPrototype string + Args string // arguments that we were invoked with + Types TypeMap + } +) func (c *Config) String() string { if c == nil { @@ -34,6 +45,35 @@ func (c *Config) Set(s string) error { return nil } +func (tm *TypeMap) Set(s string) error { + tok := strings.SplitN(s, ":", 3) + + if len(tok) < 2 { + return errors.New("expected {notation}:{type-spec} syntax, instead of " + s) + } + + if *tm == nil { + *tm = make(TypeMap) + } + + t := (*tm)[tok[0]] + t.Notation, t.Spec = tok[0], tok[1] + + if len(tok) == 3 { + t.Prototype = tok[2] + } + + (*tm)[tok[0]] = t + return nil +} + +func (tm *TypeMap) String() string { + if tm == nil { + return "" + } + return fmt.Sprintf("%#v", *tm) +} + func (c *Config) ReturnVar(names ...string) string { if c.ReturnType == "" || len(names) == 0 { return "" @@ -67,12 +107,29 @@ func (c *Config) ReturnRef(name string) string { return name } +func (c *Config) Type(notation string) string { + t, ok := c.Types[notation] + if !ok { + panic(fmt.Errorf("unknown type notation %q", notation)) + } + return t.Spec +} + +func (c *Config) Prototype(notation string) string { + t, ok := c.Types[notation] + if !ok { + panic(fmt.Errorf("unknown type notation %q", notation)) + } + return t.Prototype +} + func (c *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&c.Package, "package", c.Package, "destination package") fs.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") fs.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") fs.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") fs.Var(c, "import", "packages to import") + fs.Var(&c.Types, "type", "auxilliary type mappings in {notation}:{type-spec}:{prototype-expr} format") } func NewConfig() *Config { diff --git a/api/v1/lib/extras/gen/handlers.go b/api/v1/lib/extras/gen/handlers.go index fdc40f29..06be8132 100644 --- a/api/v1/lib/extras/gen/handlers.go +++ b/api/v1/lib/extras/gen/handlers.go @@ -34,10 +34,10 @@ type ( HandlerFunc func(context.Context, {{.EventType}}) error // Handlers executes an event Handler according to the event's type - Handlers map[scheduler.Event_Type]Handler + Handlers map[{{.Type "ET"}}]Handler // HandlerFuncs executes an event HandlerFunc according to the event's type - HandlerFuncs map[scheduler.Event_Type]HandlerFunc + HandlerFuncs map[{{.Type "ET"}}]HandlerFunc ) // HandleEvent implements Handler for HandlerFunc diff --git a/api/v1/lib/scheduler/events/events_generated.go b/api/v1/lib/scheduler/events/events_generated.go index 5727f10c..744c4cee 100644 --- a/api/v1/lib/scheduler/events/events_generated.go +++ b/api/v1/lib/scheduler/events/events_generated.go @@ -1,6 +1,6 @@ package events -// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event -type ET:scheduler.Event_Type // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/scheduler/events/gen.go b/api/v1/lib/scheduler/events/gen.go index 62339cd4..70fd131c 100644 --- a/api/v1/lib/scheduler/events/gen.go +++ b/api/v1/lib/scheduler/events/gen.go @@ -1,3 +1,3 @@ package events -//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event -type ET:scheduler.Event_Type From 10e19db3edc8525f7b5f7039dd097bb917f6562b Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 02:42:54 +0000 Subject: [PATCH 48/67] handlers: stop using -event_type for generation, use -type instead --- api/v1/lib/executor/events/events_generated.go | 2 +- api/v1/lib/executor/events/gen.go | 2 +- api/v1/lib/extras/gen/gen.go | 4 ++-- api/v1/lib/extras/gen/handlers.go | 16 ++++++++-------- api/v1/lib/scheduler/events/events_generated.go | 2 +- api/v1/lib/scheduler/events/gen.go | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/v1/lib/executor/events/events_generated.go b/api/v1/lib/executor/events/events_generated.go index 7ea7bdb9..57cf862b 100644 --- a/api/v1/lib/executor/events/events_generated.go +++ b/api/v1/lib/executor/events/events_generated.go @@ -1,6 +1,6 @@ package events -// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -event_type *executor.Event -type ET:executor.Event_Type +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/executor/events/gen.go b/api/v1/lib/executor/events/gen.go index b28a7934..673ecd79 100644 --- a/api/v1/lib/executor/events/gen.go +++ b/api/v1/lib/executor/events/gen.go @@ -1,3 +1,3 @@ package events -//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -event_type *executor.Event -type ET:executor.Event_Type +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go index b6bbf686..9c274569 100644 --- a/api/v1/lib/extras/gen/gen.go +++ b/api/v1/lib/extras/gen/gen.go @@ -110,7 +110,7 @@ func (c *Config) ReturnRef(name string) string { func (c *Config) Type(notation string) string { t, ok := c.Types[notation] if !ok { - panic(fmt.Errorf("unknown type notation %q", notation)) + return "" } return t.Spec } @@ -118,7 +118,7 @@ func (c *Config) Type(notation string) string { func (c *Config) Prototype(notation string) string { t, ok := c.Types[notation] if !ok { - panic(fmt.Errorf("unknown type notation %q", notation)) + return "" } return t.Prototype } diff --git a/api/v1/lib/extras/gen/handlers.go b/api/v1/lib/extras/gen/handlers.go index 06be8132..a440cc4b 100644 --- a/api/v1/lib/extras/gen/handlers.go +++ b/api/v1/lib/extras/gen/handlers.go @@ -27,11 +27,11 @@ type ( // Handler is invoked upon the occurrence of some scheduler event that is generated // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) Handler interface { - HandleEvent(context.Context, {{.EventType}}) error + HandleEvent(context.Context, {{.Type "E"}}) error } // HandlerFunc is a functional adaptation of the Handler interface - HandlerFunc func(context.Context, {{.EventType}}) error + HandlerFunc func(context.Context, {{.Type "E"}}) error // Handlers executes an event Handler according to the event's type Handlers map[{{.Type "ET"}}]Handler @@ -41,15 +41,15 @@ type ( ) // HandleEvent implements Handler for HandlerFunc -func (f HandlerFunc) HandleEvent(ctx context.Context, e {{.EventType}}) error { return f(ctx, e) } +func (f HandlerFunc) HandleEvent(ctx context.Context, e {{.Type "E"}}) error { return f(ctx, e) } -var noopHandler = func(_ context.Context, _ {{.EventType}}) error { return nil } +var noopHandler = func(_ context.Context, _ {{.Type "E"}}) error { return nil } // NoopHandler returns a HandlerFunc that does nothing and always returns nil func NoopHandler() HandlerFunc { return noopHandler } // HandleEvent implements Handler for Handlers -func (hs Handlers) HandleEvent(ctx context.Context, e {{.EventType}}) (err error) { +func (hs Handlers) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } @@ -57,7 +57,7 @@ func (hs Handlers) HandleEvent(ctx context.Context, e {{.EventType}}) (err error } // HandleEvent implements Handler for HandlerFuncs -func (hs HandlerFuncs) HandleEvent(ctx context.Context, e {{.EventType}}) (err error) { +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } @@ -70,7 +70,7 @@ func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { if f == nil { return hs.HandleEvent } - return func(ctx context.Context, e {{.EventType}}) error { + return func(ctx context.Context, e {{.Type "E"}}) error { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } @@ -84,7 +84,7 @@ func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { if f == nil { return hs.HandleEvent } - return func(ctx context.Context, e {{.EventType}}) error { + return func(ctx context.Context, e {{.Type "E"}}) error { if h := hs[e.GetType()]; h != nil { return h.HandleEvent(ctx, e) } diff --git a/api/v1/lib/scheduler/events/events_generated.go b/api/v1/lib/scheduler/events/events_generated.go index 744c4cee..383b2b8c 100644 --- a/api/v1/lib/scheduler/events/events_generated.go +++ b/api/v1/lib/scheduler/events/events_generated.go @@ -1,6 +1,6 @@ package events -// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event -type ET:scheduler.Event_Type +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/scheduler/events/gen.go b/api/v1/lib/scheduler/events/gen.go index 70fd131c..259a07cb 100644 --- a/api/v1/lib/scheduler/events/gen.go +++ b/api/v1/lib/scheduler/events/gen.go @@ -1,3 +1,3 @@ package events -//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event -type ET:scheduler.Event_Type +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type From 738e00f0ac8d827f127bda313736abafadf8ca46 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 02:57:57 +0000 Subject: [PATCH 49/67] rules generation: stop using -event_type in favor of -type --- api/v1/lib/extras/gen/gen.go | 23 ++++----- api/v1/lib/extras/gen/handlers.go | 2 + api/v1/lib/extras/gen/rules.go | 47 ++++++++++--------- .../callrules/callrules_generated.go | 2 +- .../callrules/callrules_generated_test.go | 2 +- api/v1/lib/extras/scheduler/callrules/gen.go | 2 +- .../eventrules/eventrules_generated.go | 2 +- .../eventrules/eventrules_generated_test.go | 2 +- api/v1/lib/extras/scheduler/eventrules/gen.go | 2 +- 9 files changed, 41 insertions(+), 43 deletions(-) diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go index 9c274569..cace882c 100644 --- a/api/v1/lib/extras/gen/gen.go +++ b/api/v1/lib/extras/gen/gen.go @@ -24,8 +24,6 @@ type ( Config struct { Package string Imports []string - EventType string // Deprecated in favor of Types - EventPrototype string // Deprecated in favor of Types ReturnType string ReturnPrototype string Args string // arguments that we were invoked with @@ -107,6 +105,14 @@ func (c *Config) ReturnRef(name string) string { return name } +func (c *Config) RequireType(notation string) (string, error) { + _, ok := c.Types[notation] + if !ok { + return "", fmt.Errorf("type %q is required but not specified", notation) + } + return "", nil +} + func (c *Config) Type(notation string) string { t, ok := c.Types[notation] if !ok { @@ -125,7 +131,6 @@ func (c *Config) Prototype(notation string) string { func (c *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&c.Package, "package", c.Package, "destination package") - fs.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") fs.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") fs.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") fs.Var(c, "import", "packages to import") @@ -135,8 +140,7 @@ func (c *Config) AddFlags(fs *flag.FlagSet) { func NewConfig() *Config { var ( c = Config{ - Package: os.Getenv("GOPACKAGE"), - EventType: "Event", + Package: os.Getenv("GOPACKAGE"), } ) return &c @@ -171,15 +175,6 @@ func Run(src, test *template.Template, args ...string) { if c.Package == "" { c.Package = "foo" } - if c.EventType == "" { - c.EventType = "Event" - c.EventPrototype = "Event{}" - } else if strings.HasPrefix(c.EventType, "*") { - // TODO(jdef) don't assume that event type is a struct or *struct - c.EventPrototype = "&" + c.EventType[1:] + "{}" - } else { - c.EventPrototype = c.EventType[1:] + "{}" - } if c.ReturnType != "" && c.ReturnPrototype == "" { panic(errors.New("return_prototype is required when return_type is set")) } diff --git a/api/v1/lib/extras/gen/handlers.go b/api/v1/lib/extras/gen/handlers.go index a440cc4b..873d2de7 100644 --- a/api/v1/lib/extras/gen/handlers.go +++ b/api/v1/lib/extras/gen/handlers.go @@ -23,6 +23,8 @@ import ( {{end}} ) +{{.RequireType "E" -}} +{{.RequireType "ET" -}} type ( // Handler is invoked upon the occurrence of some scheduler event that is generated // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 8cd9ebe0..5e8d4eca 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -25,6 +25,7 @@ import ( {{end}} ) +{{.RequireType "E" -}} type ( evaler interface { // Eval executes a filter, rule, or decorator function; if the returned event is nil then @@ -33,16 +34,16 @@ type ( // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. // Eval implementations SHOULD be safe to execute concurrently. - Eval(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) + Eval(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) } // Rule is the functional adaptation of evaler. // A nil Rule is valid: it is Eval'd as a noop. - Rule func(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) + Rule func(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, // no additional rules are processed. - Chain func(context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) + Chain func(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) // Rules is a list of rules to be processed, in order. Rules []Rule @@ -58,13 +59,13 @@ var ( _ = evaler(Rules{}) // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + chainIdentity = func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return ctx, e, {{.ReturnRef "z," -}} err } ) // Eval is a convenience func that processes a nil Rule as a noop. -func (r Rule) Eval(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { +func (r Rule) Eval(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { if r != nil { return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } @@ -73,17 +74,17 @@ func (r Rule) Eval(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. -func (rs Rules) Eval(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { +func (rs Rules) Eval(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return ch(rs.Chain()(ctx, e, {{.ReturnRef "z," -}} err)) } -// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.EventType}}, error) +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.Type "E"}}, error) // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { return chainIdentity } - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return rs[0].Eval(ctx, e, {{.ReturnRef "z," -}} err, rs[1:].Chain()) } } @@ -191,7 +192,7 @@ func (r Rule) Once() Rule { return nil } var once sync.Once - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { ruleInvoked := false once.Do(func() { ctx, e, {{.ReturnRef "z," -}} err = r(ctx, e, {{.ReturnRef "z," -}} err, ch) @@ -210,7 +211,7 @@ func (r Rule) Poll(p <-chan struct{}) Rule { if p == nil || r == nil { return nil } - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { select { case <-p: // do something @@ -246,7 +247,7 @@ func (r Rule) EveryN(nthTime int) Rule { return false } ) - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { if forward() { return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } @@ -254,21 +255,21 @@ func (r Rule) EveryN(nthTime int) Rule { } } -// Drop aborts the Chain and returns the (context.Context, {{.EventType}}, error) tuple as-is. +// Drop aborts the Chain and returns the (context.Context, {{.Type "E"}}, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() } -// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) tuple as-is. +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) tuple as-is. func (r Rule) ThenDrop() Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, _ Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, _ Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, chainIdentity) } } // Fail returns a Rule that injects the given error. func Fail(injected error) Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, injected)) } } @@ -281,7 +282,7 @@ func DropOnError() Rule { // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then // the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { if err != nil { return ctx, e, {{.ReturnRef "z," -}} err } @@ -301,7 +302,7 @@ func DropOnSuccess() Rule { } func (r Rule) DropOnSuccess() Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { if err == nil { // bypass remainder of chain return ctx, e, {{.ReturnRef "z," -}} err @@ -330,30 +331,30 @@ import ( {{end}} ) -func prototype() {{.EventType}} { return {{.EventPrototype}} } +func prototype() {{.Type "E"}} { return {{.Prototype "E"}} } func counter(i *int) Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { *i++ return ch(ctx, e, {{.ReturnRef "z," -}} err) } } func tracer(r Rule, name string, t *testing.T) Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { t.Log("executing", name) return r(ctx, e, {{.ReturnRef "z," -}} err, ch) } } func returnError(re error) Rule { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, re)) } } func chainCounter(i *int, ch Chain) Chain { - return func(ctx context.Context, e {{.EventType}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.EventType}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { *i++ return ch(ctx, e, {{.ReturnRef "z," -}} err) } @@ -391,7 +392,7 @@ func TestRules(t *testing.T) { // multiple rules in Rules should execute, dropping nil rules along the way for _, tc := range []struct { - e {{.EventType}} + e {{.Type "E"}} {{if .ReturnType}} {{- .ReturnArg "z "}} {{end -}} diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 55d0f3e7..819bd443 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -1,6 +1,6 @@ package callrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 060cb9fb..63b62c82 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -1,6 +1,6 @@ package callrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go index dcadf5f4..2090d776 100644 --- a/api/v1/lib/extras/scheduler/callrules/gen.go +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -1,3 +1,3 @@ package callrules -//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Call -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index db38dc8e..c3fa03d0 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -1,6 +1,6 @@ package eventrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 4a96815c..0e42d7dc 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -1,6 +1,6 @@ package eventrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go index b950626c..3d1dd0cc 100644 --- a/api/v1/lib/extras/scheduler/eventrules/gen.go +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -1,3 +1,3 @@ package eventrules -//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} From e920da67e968c7f3a626ab8f438fa9e23faa4bc9 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 03:23:31 +0000 Subject: [PATCH 50/67] rule generation: eliminate -return_type and -return_prototype in favor of -type --- api/v1/lib/extras/gen/gen.go | 70 ++++-- api/v1/lib/extras/gen/rules.go | 222 +++++++++--------- .../callrules/callrules_generated.go | 2 +- .../callrules/callrules_generated_test.go | 2 +- api/v1/lib/extras/scheduler/callrules/gen.go | 2 +- 5 files changed, 160 insertions(+), 138 deletions(-) diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go index cace882c..2f2ebc47 100644 --- a/api/v1/lib/extras/gen/gen.go +++ b/api/v1/lib/extras/gen/gen.go @@ -22,12 +22,10 @@ type ( TypeMap map[string]Type Config struct { - Package string - Imports []string - ReturnType string - ReturnPrototype string - Args string // arguments that we were invoked with - Types TypeMap + Package string + Imports []string + Args string // arguments that we were invoked with + Types TypeMap } ) @@ -57,8 +55,20 @@ func (tm *TypeMap) Set(s string) error { t := (*tm)[tok[0]] t.Notation, t.Spec = tok[0], tok[1] + if t.Notation == "" { + return fmt.Errorf("type notation in %q may not be an empty string", s) + } + + if t.Spec == "" { + return fmt.Errorf("type specification in %q may not be an empty string", s) + } + if len(tok) == 3 { t.Prototype = tok[2] + + if t.Prototype == "" { + return fmt.Errorf("prototype specification in %q may not be an empty string", s) + } } (*tm)[tok[0]] = t @@ -72,37 +82,40 @@ func (tm *TypeMap) String() string { return fmt.Sprintf("%#v", *tm) } -func (c *Config) ReturnVar(names ...string) string { - if c.ReturnType == "" || len(names) == 0 { +func (c *Config) Var(notation string, names ...string) string { + t := c.Type(notation) + if t == "" || len(names) == 0 { return "" } - return "var " + strings.Join(names, ",") + " " + c.ReturnType + return "var " + strings.Join(names, ",") + " " + t } -func (c *Config) ReturnArg(name string) string { - if c.ReturnType == "" { +func (c *Config) Arg(notation, name string) string { + t := c.Type(notation) + if t == "" { return "" } if name == "" { - return c.ReturnType + return t } if strings.HasSuffix(name, ",") { - return strings.TrimSpace(name[:len(name)-1]+" "+c.ReturnType) + ", " + return strings.TrimSpace(name[:len(name)-1]+" "+t) + ", " } - return name + " " + c.ReturnType + return name + " " + t } -func (c *Config) ReturnRef(name string) string { - if c.ReturnType == "" || name == "" { - return "" +func (c *Config) Ref(notation, name string) (string, error) { + t := c.Type(notation) + if t == "" || name == "" { + return "", nil } if strings.HasSuffix(name, ",") { if len(name) < 2 { - panic("expected ref name before comma") + return "", errors.New("expected ref name before comma") } - return name[:len(name)-1] + ", " + return name[:len(name)-1] + ", ", nil } - return name + return name, nil } func (c *Config) RequireType(notation string) (string, error) { @@ -113,6 +126,18 @@ func (c *Config) RequireType(notation string) (string, error) { return "", nil } +func (c *Config) RequirePrototype(notation string) (string, error) { + t, ok := c.Types[notation] + if !ok { + // needed for optional types: don't require the prototype if the optional type is not defined + return "", nil + } + if t.Prototype == "" { + return "", fmt.Errorf("prototype for type %q is required but not specified", notation) + } + return "", nil +} + func (c *Config) Type(notation string) string { t, ok := c.Types[notation] if !ok { @@ -131,8 +156,6 @@ func (c *Config) Prototype(notation string) string { func (c *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&c.Package, "package", c.Package, "destination package") - fs.StringVar(&c.ReturnType, "return_type", c.ReturnType, "golang type of a return arg") - fs.StringVar(&c.ReturnPrototype, "return_prototype", c.ReturnPrototype, "golang expression of a return obj prototype") fs.Var(c, "import", "packages to import") fs.Var(&c.Types, "type", "auxilliary type mappings in {notation}:{type-spec}:{prototype-expr} format") } @@ -175,9 +198,6 @@ func Run(src, test *template.Template, args ...string) { if c.Package == "" { c.Package = "foo" } - if c.ReturnType != "" && c.ReturnPrototype == "" { - panic(errors.New("return_prototype is required when return_type is set")) - } if output == "" { output = defaultOutput diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 5e8d4eca..8838c473 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -26,6 +26,8 @@ import ( ) {{.RequireType "E" -}} +{{.RequirePrototype "E" -}} +{{.RequirePrototype "Z" -}} type ( evaler interface { // Eval executes a filter, rule, or decorator function; if the returned event is nil then @@ -34,16 +36,16 @@ type ( // If changes to the event object are needed, the suggested approach is to make a copy, // modify the copy, and pass the copy to the chain. // Eval implementations SHOULD be safe to execute concurrently. - Eval(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) + Eval(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) } // Rule is the functional adaptation of evaler. // A nil Rule is valid: it is Eval'd as a noop. - Rule func(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) + Rule func(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, // no additional rules are processed. - Chain func(context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) + Chain func(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) // Rules is a list of rules to be processed, in order. Rules []Rule @@ -59,23 +61,23 @@ var ( _ = evaler(Rules{}) // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return ctx, e, {{.ReturnRef "z," -}} err + chainIdentity = func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ctx, e, {{.Ref "Z" "z," -}} err } ) // Eval is a convenience func that processes a nil Rule as a noop. -func (r Rule) Eval(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { +func (r Rule) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { if r != nil { - return r(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } - return ch(ctx, e, {{.ReturnRef "z," -}} err) + return ch(ctx, e, {{.Ref "Z" "z," -}} err) } // Eval is a Rule func that processes the set of all Rules. If there are no rules in the // set then control is simply passed to the Chain. -func (rs Rules) Eval(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return ch(rs.Chain()(ctx, e, {{.ReturnRef "z," -}} err)) +func (rs Rules) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(rs.Chain()(ctx, e, {{.Ref "Z" "z," -}} err)) } // Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.Type "E"}}, error) @@ -84,8 +86,8 @@ func (rs Rules) Chain() Chain { if len(rs) == 0 { return chainIdentity } - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return rs[0].Eval(ctx, e, {{.ReturnRef "z," -}} err, rs[1:].Chain()) + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return rs[0].Eval(ctx, e, {{.Ref "Z" "z," -}} err, rs[1:].Chain()) } } @@ -192,16 +194,16 @@ func (r Rule) Once() Rule { return nil } var once sync.Once - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { ruleInvoked := false once.Do(func() { - ctx, e, {{.ReturnRef "z," -}} err = r(ctx, e, {{.ReturnRef "z," -}} err, ch) + ctx, e, {{.Ref "Z" "z," -}} err = r(ctx, e, {{.Ref "Z" "z," -}} err, ch) ruleInvoked = true }) if !ruleInvoked { - ctx, e, {{.ReturnRef "z," -}} err = ch(ctx, e, {{.ReturnRef "z," -}} err) + ctx, e, {{.Ref "Z" "z," -}} err = ch(ctx, e, {{.Ref "Z" "z," -}} err) } - return ctx, e, {{.ReturnRef "z," -}} err + return ctx, e, {{.Ref "Z" "z," -}} err } } @@ -211,17 +213,17 @@ func (r Rule) Poll(p <-chan struct{}) Rule { if p == nil || r == nil { return nil } - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { select { case <-p: // do something // TODO(jdef): optimization: if we detect the chan is closed, affect a state change // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) case <-ctx.Done(): - return ctx, e, {{.ReturnRef "z," -}} Error2(err, ctx.Err()) + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) default: - return ch(ctx, e, {{.ReturnRef "z," -}} err) + return ch(ctx, e, {{.Ref "Z" "z," -}} err) } } } @@ -247,11 +249,11 @@ func (r Rule) EveryN(nthTime int) Rule { return false } ) - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { if forward() { - return r(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } - return ch(ctx, e, {{.ReturnRef "z," -}} err) + return ch(ctx, e, {{.Ref "Z" "z," -}} err) } } @@ -260,17 +262,17 @@ func Drop() Rule { return Rule(nil).ThenDrop() } -// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) tuple as-is. +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) tuple as-is. func (r Rule) ThenDrop() Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, _ Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, chainIdentity) + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, _ Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, chainIdentity) } } // Fail returns a Rule that injects the given error. func Fail(injected error) Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, injected)) + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, injected)) } } @@ -282,11 +284,11 @@ func DropOnError() Rule { // DropOnError decorates a rule by pre-checking the error state: if the error state != nil then // the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. func (r Rule) DropOnError() Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { if err != nil { - return ctx, e, {{.ReturnRef "z," -}} err + return ctx, e, {{.Ref "Z" "z," -}} err } - return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) } } @@ -302,12 +304,12 @@ func DropOnSuccess() Rule { } func (r Rule) DropOnSuccess() Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { if err == nil { // bypass remainder of chain - return ctx, e, {{.ReturnRef "z," -}} err + return ctx, e, {{.Ref "Z" "z," -}} err } - return r.Eval(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) } } @@ -334,39 +336,39 @@ import ( func prototype() {{.Type "E"}} { return {{.Prototype "E"}} } func counter(i *int) Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { *i++ - return ch(ctx, e, {{.ReturnRef "z," -}} err) + return ch(ctx, e, {{.Ref "Z" "z," -}} err) } } func tracer(r Rule, name string, t *testing.T) Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { t.Log("executing", name) - return r(ctx, e, {{.ReturnRef "z," -}} err, ch) + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } } func returnError(re error) Rule { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { - return ch(ctx, e, {{.ReturnRef "z," -}} Error2(err, re)) + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, re)) } } func chainCounter(i *int, ch Chain) Chain { - return func(ctx context.Context, e {{.Type "E"}}, {{.ReturnArg "z," -}} err error) (context.Context, {{.Type "E"}}, {{.ReturnArg "," -}} error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { *i++ - return ch(ctx, e, {{.ReturnRef "z," -}} err) + return ch(ctx, e, {{.Ref "Z" "z," -}} err) } } func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) -{{if .ReturnType}} - {{.ReturnVar "z0"}} +{{if .Type "Z"}} + {{.Var "Z" "z0"}} {{end}} - _, e, {{.ReturnRef "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.ReturnRef "z0," -}} nil, chainIdentity) + _, e, {{.Ref "Z" "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.Ref "Z" "z0," -}} nil, chainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -385,28 +387,28 @@ func TestRules(t *testing.T) { ctx = context.Background() ) - {{if .ReturnType -}} - {{.ReturnVar "z0"}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + {{.Var "Z" "z0"}} + var zp = {{.Prototype "Z"}} {{end -}} // multiple rules in Rules should execute, dropping nil rules along the way for _, tc := range []struct { e {{.Type "E"}} - {{if .ReturnType}} - {{- .ReturnArg "z "}} + {{if .Type "Z"}} + {{- .Arg "Z" "z "}} {{end -}} err error }{ - {nil, {{.ReturnRef "z0," -}} nil}, - {nil, {{.ReturnRef "z0," -}} a}, - {p, {{.ReturnRef "z0," -}} nil}, - {p, {{.ReturnRef "z0," -}} a}, -{{if .ReturnType}} - {nil, {{.ReturnRef "zp," -}} nil}, - {nil, {{.ReturnRef "zp," -}} a}, - {p, {{.ReturnRef "zp," -}} nil}, - {p, {{.ReturnRef "zp," -}} a}, + {nil, {{.Ref "Z" "z0," -}} nil}, + {nil, {{.Ref "Z" "z0," -}} a}, + {p, {{.Ref "Z" "z0," -}} nil}, + {p, {{.Ref "Z" "z0," -}} a}, +{{if .Type "Z"}} + {nil, {{.Ref "Z" "zp," -}} nil}, + {nil, {{.Ref "Z" "zp," -}} a}, + {p, {{.Ref "Z" "zp," -}} nil}, + {p, {{.Ref "Z" "zp," -}} a}, {{end}} } { var ( i int @@ -417,12 +419,12 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, {{.ReturnRef "zz," -}} err = rule(ctx, tc.e, {{.ReturnRef "tc.z," -}} tc.err, chainIdentity) + _, e, {{.Ref "Z" "zz," -}} err = rule(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, chainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != tc.z { t.Errorf("expected return object %q instead of %q", tc.z, zz) } @@ -434,12 +436,12 @@ func TestRules(t *testing.T) { t.Error("expected 2 rule executions instead of", i) } - // empty Rules should not change event, {{.ReturnRef "z," -}} err - _, e, {{.ReturnRef "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.ReturnRef "tc.z," -}} tc.err, chainIdentity) + // empty Rules should not change event, {{.Ref "Z" "z," -}} err + _, e, {{.Ref "Z" "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, chainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != tc.z { t.Errorf("expected return object %q instead of %q", tc.z, zz) } @@ -503,15 +505,15 @@ func TestAndThen(t *testing.T) { r2 = Rule(nil).AndThen(counter(&i)) a = errors.New("a") ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} for k, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -537,8 +539,8 @@ func TestOnFailure(t *testing.T) { r1 = counter(&i) r2 = Fail(a).OnFailure(counter(&i)) ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} for k, tc := range []struct { r Rule @@ -547,11 +549,11 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, {{.ReturnRef "zz," -}} err := tc.r(ctx, p, {{.ReturnRef "zp," -}} tc.initialError, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := tc.r(ctx, p, {{.Ref "Z" "zp," -}} tc.initialError, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -577,17 +579,17 @@ func TestDropOnError(t *testing.T) { r2 = counter(&i).DropOnError() a = errors.New("a") ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -602,11 +604,11 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, {{.ReturnRef "zz," -}} err := r2(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -627,17 +629,17 @@ func TestDropOnSuccess(t *testing.T) { r1 = counter(&i) r2 = counter(&i).DropOnSuccess() ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -653,11 +655,11 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, {{.ReturnRef "zz," -}} err := r2(ctx, p, {{.ReturnRef "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -673,11 +675,11 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, {{.ReturnRef "zz," -}} err = r3(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err = r3(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -702,16 +704,16 @@ func TestThenDrop(t *testing.T) { r1 = counter(&i) r2 = counter(&i).ThenDrop() ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} anErr, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -738,17 +740,17 @@ func TestDrop(t *testing.T) { r1 = counter(&i) r2 = Rules{Drop(), counter(&i)}.Eval ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} anErr, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -774,17 +776,17 @@ func TestIf(t *testing.T) { r1 = counter(&i).If(true).Eval r2 = counter(&i).If(false).Eval ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -809,17 +811,17 @@ func TestUnless(t *testing.T) { r1 = counter(&i).Unless(false).Eval r2 = counter(&i).Unless(true).Eval ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -844,16 +846,16 @@ func TestOnce(t *testing.T) { r1 = counter(&i).Once().Eval r2 = Rule(nil).Once().Eval ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } @@ -896,16 +898,16 @@ func TestPoll(t *testing.T) { r1 = counter(&i).Poll(tc.ch).Eval r2 = Rule(nil).Poll(tc.ch).Eval ) - {{if .ReturnType -}} - var zp = {{.ReturnPrototype}} + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} {{end -}} for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { - _, e, {{.ReturnRef "zz," -}} err := r(ctx, p, {{.ReturnRef "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } - {{if .ReturnType -}} + {{if .Type "Z" -}} if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 819bd443..ef779f63 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -1,6 +1,6 @@ package callrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 63b62c82..7268efa7 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -1,6 +1,6 @@ package callrules -// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} // GENERATED CODE FOLLOWS; DO NOT EDIT. import ( diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go index 2090d776..a146496a 100644 --- a/api/v1/lib/extras/scheduler/callrules/gen.go +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -1,3 +1,3 @@ package callrules -//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -return_type mesos.Response -return_prototype &mesos.ResponseWrapper{} +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} From 9051de04786ec63339dfd7046800cd4a9024fd8e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 04:14:52 +0000 Subject: [PATCH 51/67] gen: code generation for rule-handler and rule-metrics bindings --- api/v1/lib/extras/gen/rule_handlers.go | 68 +++++++++++++++++++ api/v1/lib/extras/gen/rule_metrics.go | 40 +++++++++++ api/v1/lib/extras/scheduler/eventrules/gen.go | 4 ++ .../{handlers.go => handlers_generated.go} | 22 ++---- .../{metrics.go => metrics_generated.go} | 4 ++ 5 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 api/v1/lib/extras/gen/rule_handlers.go create mode 100644 api/v1/lib/extras/gen/rule_metrics.go rename api/v1/lib/extras/scheduler/eventrules/{handlers.go => handlers_generated.go} (72%) rename api/v1/lib/extras/scheduler/eventrules/{metrics.go => metrics_generated.go} (74%) diff --git a/api/v1/lib/extras/gen/rule_handlers.go b/api/v1/lib/extras/gen/rule_handlers.go new file mode 100644 index 00000000..b5aff94a --- /dev/null +++ b/api/v1/lib/extras/gen/rule_handlers.go @@ -0,0 +1,68 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequireType "H" -}} +{{.RequireType "HF" -}} +// Handle generates a rule that executes the given {{.Type "H"}}. +func Handle(h {{.Type "H"}}) Rule { + if h == nil { + return nil + } + return func(ctx context.Context, e {{.Type "E"}}, err error, chain Chain) (context.Context, {{.Type "E"}}, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h {{.Type "HF"}}) Rule { + return Handle({{.Type "H"}}(h)) +} + +// Handle returns a Rule that invokes the receiver, then the given {{.Type "H"}} +func (r Rule) Handle(h {{.Type "H"}}) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h {{.Type "HF"}}) Rule { + return r.Handle({{.Type "H"}}(h)) +} + +// HandleEvent implements {{.Type "H"}} for Rule +func (r Rule) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { + if r == nil { + return nil + } + _, _, err = r(ctx, e, nil, chainIdentity) + return +} + +// HandleEvent implements {{.Type "H"}} for Rules +func (rs Rules) HandleEvent(ctx context.Context, e {{.Type "E"}}) error { + return Rule(rs.Eval).HandleEvent(ctx, e) +} +`)) diff --git a/api/v1/lib/extras/gen/rule_metrics.go b/api/v1/lib/extras/gen/rule_metrics.go new file mode 100644 index 00000000..830f9d0b --- /dev/null +++ b/api/v1/lib/extras/gen/rule_metrics.go @@ -0,0 +1,40 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}}{{/* TODO(jdef): should support an optional return arg for use w/ calls */ -}} +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, e {{.Type "E"}}, err error, ch Chain) (context.Context, {{.Type "E"}}, error) { + typename := strings.ToLower(e.GetType().String()) + harness(func() error { + ctx, e, err = ch(ctx, e, err) + return err + }, typename) + return ctx, e, err + } +} +`)) diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go index 3d1dd0cc..18169c36 100644 --- a/api/v1/lib/extras/scheduler/eventrules/gen.go +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -1,3 +1,7 @@ package eventrules //go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} + +//go:generate go run ../../gen/rule_handlers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/events -type E:*scheduler.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event -output metrics_generated.go diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go similarity index 72% rename from api/v1/lib/extras/scheduler/eventrules/handlers.go rename to api/v1/lib/extras/scheduler/eventrules/handlers_generated.go index 0190f99b..ce9ae252 100644 --- a/api/v1/lib/extras/scheduler/eventrules/handlers.go +++ b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go @@ -1,5 +1,8 @@ package eventrules +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/events -type E:*scheduler.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + import ( "context" @@ -7,7 +10,7 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" ) -// Handle generates a rule that executes the given handler. +// Handle generates a rule that executes the given events.Handler. func Handle(h events.Handler) Rule { if h == nil { return nil @@ -23,7 +26,7 @@ func HandleF(h events.HandlerFunc) Rule { return Handle(events.Handler(h)) } -// Handle returns a Rule that invokes the receiver, then the given Handler +// Handle returns a Rule that invokes the receiver, then the given events.Handler func (r Rule) Handle(h events.Handler) Rule { return Rules{r, Handle(h)}.Eval } @@ -46,18 +49,3 @@ func (r Rule) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { func (rs Rules) HandleEvent(ctx context.Context, e *scheduler.Event) error { return Rule(rs.Eval).HandleEvent(ctx, e) } - -/* -// Apply returns the result of a singleton rule set (the receiver) applied to the given event handler. -func (r Rule) Apply(h events.Handler) events.HandlerFunc { - if r == nil { - return h.HandleEvent - } - return r.Handle(h).HandleEvent -} - -// ApplyF is the functional equivalent of Apply -func (r Rule) ApplyF(h events.HandlerFunc) events.HandlerFunc { - return r.Apply(events.Handler(h)) -} -*/ diff --git a/api/v1/lib/extras/scheduler/eventrules/metrics.go b/api/v1/lib/extras/scheduler/eventrules/metrics_generated.go similarity index 74% rename from api/v1/lib/extras/scheduler/eventrules/metrics.go rename to api/v1/lib/extras/scheduler/eventrules/metrics_generated.go index 14df3a7a..c3d71df9 100644 --- a/api/v1/lib/extras/scheduler/eventrules/metrics.go +++ b/api/v1/lib/extras/scheduler/eventrules/metrics_generated.go @@ -1,10 +1,14 @@ package eventrules +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + import ( "context" "strings" "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) From 13141823893fac3c9f55e70690e69e801478af14 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 04:32:21 +0000 Subject: [PATCH 52/67] rules/metrics: code generation for caller metrics too --- api/v1/lib/extras/gen/rule_metrics.go | 8 +++--- api/v1/lib/extras/scheduler/callrules/gen.go | 2 ++ .../lib/extras/scheduler/callrules/metrics.go | 21 ---------------- .../scheduler/callrules/metrics_generated.go | 25 +++++++++++++++++++ 4 files changed, 31 insertions(+), 25 deletions(-) delete mode 100644 api/v1/lib/extras/scheduler/callrules/metrics.go create mode 100644 api/v1/lib/extras/scheduler/callrules/metrics_generated.go diff --git a/api/v1/lib/extras/gen/rule_metrics.go b/api/v1/lib/extras/gen/rule_metrics.go index 830f9d0b..430ff331 100644 --- a/api/v1/lib/extras/gen/rule_metrics.go +++ b/api/v1/lib/extras/gen/rule_metrics.go @@ -26,15 +26,15 @@ import ( {{end}} ) -{{.RequireType "E" -}}{{/* TODO(jdef): should support an optional return arg for use w/ calls */ -}} +{{.RequireType "E" -}} func Metrics(harness metrics.Harness) Rule { - return func(ctx context.Context, e {{.Type "E"}}, err error, ch Chain) (context.Context, {{.Type "E"}}, error) { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { typename := strings.ToLower(e.GetType().String()) harness(func() error { - ctx, e, err = ch(ctx, e, err) + ctx, e, {{.Ref "Z" "z," -}} err = ch(ctx, e, {{.Ref "Z" "z," -}} err) return err }, typename) - return ctx, e, err + return ctx, e, {{.Ref "Z" "z," -}} err } } `)) diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go index a146496a..f4f57a04 100644 --- a/api/v1/lib/extras/scheduler/callrules/gen.go +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -1,3 +1,5 @@ package callrules //go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call -type Z:mesos.Response -output metrics_generated.go diff --git a/api/v1/lib/extras/scheduler/callrules/metrics.go b/api/v1/lib/extras/scheduler/callrules/metrics.go deleted file mode 100644 index 1adc1219..00000000 --- a/api/v1/lib/extras/scheduler/callrules/metrics.go +++ /dev/null @@ -1,21 +0,0 @@ -package callrules - -import ( - "context" - "strings" - - "github.com/mesos/mesos-go/api/v1/lib" - "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -func Metrics(harness metrics.Harness) Rule { - return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - typename := strings.ToLower(c.GetType().String()) - harness(func() error { - _, _, r, err = ch(ctx, c, r, err) - return err // need to count these - }, typename) - return ctx, c, r, err - } -} diff --git a/api/v1/lib/extras/scheduler/callrules/metrics_generated.go b/api/v1/lib/extras/scheduler/callrules/metrics_generated.go new file mode 100644 index 00000000..7f46acf7 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/metrics_generated.go @@ -0,0 +1,25 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call -type Z:mesos.Response -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + typename := strings.ToLower(e.GetType().String()) + harness(func() error { + ctx, e, z, err = ch(ctx, e, z, err) + return err + }, typename) + return ctx, e, z, err + } +} From 9aa2ccf9e726fd51e3586d8d3544808ff15916e6 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 04:35:47 +0000 Subject: [PATCH 53/67] callrules: partition specialized rules from generic ones in preparation for codegen --- .../lib/extras/scheduler/callrules/callers.go | 17 ------------- .../lib/extras/scheduler/callrules/helpers.go | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 api/v1/lib/extras/scheduler/callrules/helpers.go diff --git a/api/v1/lib/extras/scheduler/callrules/callers.go b/api/v1/lib/extras/scheduler/callrules/callers.go index cfb8ab9a..8e1aaa96 100644 --- a/api/v1/lib/extras/scheduler/callrules/callers.go +++ b/api/v1/lib/extras/scheduler/callrules/callers.go @@ -52,20 +52,3 @@ var ( _ = calls.Caller(Rule(nil)) _ = calls.Caller(Rules(nil)) ) - -// WithFrameworkID returns a Rule that injects a framework ID to outgoing calls, with the following exceptions: -// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) -// - calls are not modified when the detected framework ID is "" -func WithFrameworkID(frameworkID func() string) Rule { - return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - // never overwrite framework ID for subscribe calls; the scheduler must do that part - if c.GetType() != scheduler.Call_SUBSCRIBE { - if fid := frameworkID(); fid != "" { - c2 := *c - c2.FrameworkID = &mesos.FrameworkID{Value: fid} - c = &c2 - } - } - return ch(ctx, c, r, err) - } -} diff --git a/api/v1/lib/extras/scheduler/callrules/helpers.go b/api/v1/lib/extras/scheduler/callrules/helpers.go new file mode 100644 index 00000000..3826262e --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/helpers.go @@ -0,0 +1,25 @@ +package callrules + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// WithFrameworkID returns a Rule that injects a framework ID to outgoing calls, with the following exceptions: +// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) +// - calls are not modified when the detected framework ID is "" +func WithFrameworkID(frameworkID func() string) Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + // never overwrite framework ID for subscribe calls; the scheduler must do that part + if c.GetType() != scheduler.Call_SUBSCRIBE { + if fid := frameworkID(); fid != "" { + c2 := *c + c2.FrameworkID = &mesos.FrameworkID{Value: fid} + c = &c2 + } + } + return ch(ctx, c, r, err) + } +} From 4b08ba71ddea4b8a37d00e5eacb2ed39b6c113b2 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 04:48:45 +0000 Subject: [PATCH 54/67] scheduler/calls: Caller API is now code-generated --- Makefile | 1 + api/v1/lib/extras/gen/callers.go | 54 +++++++++++++++++++ .../calls/{caller.go => calls_generated.go} | 6 ++- api/v1/lib/scheduler/calls/gen.go | 3 ++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 api/v1/lib/extras/gen/callers.go rename api/v1/lib/scheduler/calls/{caller.go => calls_generated.go} (81%) create mode 100644 api/v1/lib/scheduler/calls/gen.go diff --git a/Makefile b/Makefile index a79627f6..340356f1 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ generate: go generate -x ./api/v1/lib/extras/scheduler/callrules go generate -x ./api/v1/lib/executor/events go generate -x ./api/v1/lib/scheduler/events + go generate -x ./api/v1/lib/scheduler/calls GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) diff --git a/api/v1/lib/extras/gen/callers.go b/api/v1/lib/extras/gen/callers.go new file mode 100644 index 00000000..72b8fb4b --- /dev/null +++ b/api/v1/lib/extras/gen/callers.go @@ -0,0 +1,54 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "C" -}} +type ( + // Caller is the public interface this framework scheduler's should consume + Caller interface { + // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. + Call(context.Context, {{.Type "C"}}) (mesos.Response, error) + } + + // CallerFunc is the functional adaptation of the Caller interface + CallerFunc func(context.Context, {{.Type "C"}}) (mesos.Response, error) +) + +// Call implements the Caller interface for CallerFunc +func (f CallerFunc) Call(ctx context.Context, c {{.Type "C"}}) (mesos.Response, error) { + return f(ctx, c) +} + +// CallNoData is a convenience func that executes the given Call using the provided Caller +// and always drops the response data. +func CallNoData(ctx context.Context, caller Caller, call {{.Type "C"}}) error { + resp, err := caller.Call(ctx, call) + if resp != nil { + resp.Close() + } + return err +} +`)) diff --git a/api/v1/lib/scheduler/calls/caller.go b/api/v1/lib/scheduler/calls/calls_generated.go similarity index 81% rename from api/v1/lib/scheduler/calls/caller.go rename to api/v1/lib/scheduler/calls/calls_generated.go index 0538476a..9645905f 100644 --- a/api/v1/lib/scheduler/calls/caller.go +++ b/api/v1/lib/scheduler/calls/calls_generated.go @@ -1,14 +1,18 @@ package calls +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type C:*scheduler.Call +// GENERATED CODE FOLLOWS; DO NOT EDIT. + import ( "context" "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) -// Caller is the public interface this framework scheduler's should consume type ( + // Caller is the public interface this framework scheduler's should consume Caller interface { // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. Call(context.Context, *scheduler.Call) (mesos.Response, error) diff --git a/api/v1/lib/scheduler/calls/gen.go b/api/v1/lib/scheduler/calls/gen.go new file mode 100644 index 00000000..1be50f95 --- /dev/null +++ b/api/v1/lib/scheduler/calls/gen.go @@ -0,0 +1,3 @@ +package calls + +//go:generate go run ../../extras/gen/callers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type C:*scheduler.Call From 447808478ff524ac980cb85032310c70ca1ca6b7 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 04:58:35 +0000 Subject: [PATCH 55/67] callrules: generate Caller/Rule API integration --- api/v1/lib/extras/gen/rule_callers.go | 75 +++++++++++++++++++ .../{callers.go => callers_generated.go} | 4 + api/v1/lib/extras/scheduler/callrules/gen.go | 2 + 3 files changed, 81 insertions(+) create mode 100644 api/v1/lib/extras/gen/rule_callers.go rename api/v1/lib/extras/scheduler/callrules/{callers.go => callers_generated.go} (84%) diff --git a/api/v1/lib/extras/gen/rule_callers.go b/api/v1/lib/extras/gen/rule_callers.go new file mode 100644 index 00000000..22156a94 --- /dev/null +++ b/api/v1/lib/extras/gen/rule_callers.go @@ -0,0 +1,75 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequireType "C" -}} +{{.RequireType "CF" -}} +// Call returns a Rule that invokes the given Caller +func Call(caller {{.Type "C"}}) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c {{.Type "E"}}, _ mesos.Response, _ error, ch Chain) (context.Context, {{.Type "E"}}, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf {{.Type "CF"}}) Rule { + return Call({{.Type "C"}}(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller {{.Type "C"}}) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf {{.Type "CF"}}) Rule { + return r.Caller({{.Type "C"}}(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c {{.Type "E"}}) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c {{.Type "E"}}) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = {{.Type "C"}}(Rule(nil)) + _ = {{.Type "C"}}(Rules(nil)) +) +`)) diff --git a/api/v1/lib/extras/scheduler/callrules/callers.go b/api/v1/lib/extras/scheduler/callrules/callers_generated.go similarity index 84% rename from api/v1/lib/extras/scheduler/callrules/callers.go rename to api/v1/lib/extras/scheduler/callrules/callers_generated.go index 8e1aaa96..8343507c 100644 --- a/api/v1/lib/extras/scheduler/callrules/callers.go +++ b/api/v1/lib/extras/scheduler/callrules/callers_generated.go @@ -1,9 +1,13 @@ package callrules +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/calls -type E:*scheduler.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + import ( "context" "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" ) diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go index f4f57a04..d63d35b4 100644 --- a/api/v1/lib/extras/scheduler/callrules/gen.go +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -2,4 +2,6 @@ package callrules //go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +//go:generate go run ../../gen/rule_callers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/calls -type E:*scheduler.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go + //go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call -type Z:mesos.Response -output metrics_generated.go From 32d66e14aff7fdcc8258f315582915304a005956 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 05:07:48 +0000 Subject: [PATCH 56/67] executor/calls: code-generated Caller API --- Makefile | 7 ++-- api/v1/lib/executor/calls/calls_generated.go | 38 ++++++++++++++++++++ api/v1/lib/executor/calls/gen.go | 3 ++ 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 api/v1/lib/executor/calls/calls_generated.go create mode 100644 api/v1/lib/executor/calls/gen.go diff --git a/Makefile b/Makefile index 340356f1..bde288ed 100644 --- a/Makefile +++ b/Makefile @@ -78,12 +78,9 @@ sync: (cd ${CMD_VENDOR}; govendor sync) .PHONY: generate +generate: GENERATE_PACKAGES = ./api/v1/lib/extras/scheduler/eventrules ./api/v1/lib/extras/scheduler/callrules ./api/v1/lib/executor/events ./api/v1/lib/executor/calls ./api/v1/lib/scheduler/events ./api/v1/lib/scheduler/calls generate: - go generate -x ./api/v1/lib/extras/scheduler/eventrules - go generate -x ./api/v1/lib/extras/scheduler/callrules - go generate -x ./api/v1/lib/executor/events - go generate -x ./api/v1/lib/scheduler/events - go generate -x ./api/v1/lib/scheduler/calls + go generate -x ${GENERATE_PACKAGES} GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) diff --git a/api/v1/lib/executor/calls/calls_generated.go b/api/v1/lib/executor/calls/calls_generated.go new file mode 100644 index 00000000..e4e1b01d --- /dev/null +++ b/api/v1/lib/executor/calls/calls_generated.go @@ -0,0 +1,38 @@ +package calls + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type C:*executor.Call +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + // Caller is the public interface this framework scheduler's should consume + Caller interface { + // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. + Call(context.Context, *executor.Call) (mesos.Response, error) + } + + // CallerFunc is the functional adaptation of the Caller interface + CallerFunc func(context.Context, *executor.Call) (mesos.Response, error) +) + +// Call implements the Caller interface for CallerFunc +func (f CallerFunc) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + return f(ctx, c) +} + +// CallNoData is a convenience func that executes the given Call using the provided Caller +// and always drops the response data. +func CallNoData(ctx context.Context, caller Caller, call *executor.Call) error { + resp, err := caller.Call(ctx, call) + if resp != nil { + resp.Close() + } + return err +} diff --git a/api/v1/lib/executor/calls/gen.go b/api/v1/lib/executor/calls/gen.go new file mode 100644 index 00000000..b78ed2ea --- /dev/null +++ b/api/v1/lib/executor/calls/gen.go @@ -0,0 +1,3 @@ +package calls + +//go:generate go run ../../extras/gen/callers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type C:*executor.Call From 6ba98bd457df83f19d5794b6a3296c8ab7f97a25 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 15:25:03 +0000 Subject: [PATCH 57/67] extras/executor: add callrules and eventrules packages w/ generated APIs --- Makefile | 2 +- .../executor/callrules/callers_generated.go | 58 ++ .../executor/callrules/callrules_generated.go | 302 ++++++++++ .../callrules/callrules_generated_test.go | 554 ++++++++++++++++++ api/v1/lib/extras/executor/callrules/gen.go | 7 + .../executor/callrules/metrics_generated.go | 25 + .../eventrules/eventrules_generated.go | 301 ++++++++++ .../eventrules/eventrules_generated_test.go | 488 +++++++++++++++ api/v1/lib/extras/executor/eventrules/gen.go | 7 + .../executor/eventrules/handlers_generated.go | 51 ++ .../executor/eventrules/metrics_generated.go | 24 + 11 files changed, 1818 insertions(+), 1 deletion(-) create mode 100644 api/v1/lib/extras/executor/callrules/callers_generated.go create mode 100644 api/v1/lib/extras/executor/callrules/callrules_generated.go create mode 100644 api/v1/lib/extras/executor/callrules/callrules_generated_test.go create mode 100644 api/v1/lib/extras/executor/callrules/gen.go create mode 100644 api/v1/lib/extras/executor/callrules/metrics_generated.go create mode 100644 api/v1/lib/extras/executor/eventrules/eventrules_generated.go create mode 100644 api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go create mode 100644 api/v1/lib/extras/executor/eventrules/gen.go create mode 100644 api/v1/lib/extras/executor/eventrules/handlers_generated.go create mode 100644 api/v1/lib/extras/executor/eventrules/metrics_generated.go diff --git a/Makefile b/Makefile index bde288ed..eb1862ed 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ sync: (cd ${CMD_VENDOR}; govendor sync) .PHONY: generate -generate: GENERATE_PACKAGES = ./api/v1/lib/extras/scheduler/eventrules ./api/v1/lib/extras/scheduler/callrules ./api/v1/lib/executor/events ./api/v1/lib/executor/calls ./api/v1/lib/scheduler/events ./api/v1/lib/scheduler/calls +generate: GENERATE_PACKAGES = ./api/v1/lib/extras/executor/eventrules ./api/v1/lib/extras/executor/callrules ./api/v1/lib/extras/scheduler/eventrules ./api/v1/lib/extras/scheduler/callrules ./api/v1/lib/executor/events ./api/v1/lib/executor/calls ./api/v1/lib/scheduler/events ./api/v1/lib/scheduler/calls generate: go generate -x ${GENERATE_PACKAGES} diff --git a/api/v1/lib/extras/executor/callrules/callers_generated.go b/api/v1/lib/extras/executor/callrules/callers_generated.go new file mode 100644 index 00000000..bb0fe027 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callers_generated.go @@ -0,0 +1,58 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/calls -type E:*executor.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/executor" + "github.com/mesos/mesos-go/api/v1/lib/executor/calls" +) + +// Call returns a Rule that invokes the given Caller +func Call(caller calls.Caller) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c *executor.Call, _ mesos.Response, _ error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf calls.CallerFunc) Rule { + return Call(calls.Caller(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller calls.Caller) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf calls.CallerFunc) Rule { + return r.Caller(calls.Caller(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = calls.Caller(Rule(nil)) + _ = calls.Caller(Rules(nil)) +) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go new file mode 100644 index 00000000..73f28d17 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -0,0 +1,302 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *executor.Call, mesos.Response, error, Chain) (context.Context, *executor.Call, mesos.Response, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *executor.Call, mesos.Response, error, Chain) (context.Context, *executor.Call, mesos.Response, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *executor.Call, mesos.Response, error) (context.Context, *executor.Call, mesos.Response, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) + + // chainIdentity is a Chain that returns the arguments as its results. + chainIdentity = func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + return ctx, e, z, err + } +) + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if r != nil { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(rs.Chain()(ctx, e, z, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *executor.Call, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return chainIdentity + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, z, err = r(ctx, e, z, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, z, err = ch(ctx, e, z, err) + } + return ctx, e, z, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. +// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + if p == nil || r == nil { + return nil + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + select { + case <-p: + // do something + // TODO(jdef): optimization: if we detect the chan is closed, affect a state change + // whereby this select is no longer invoked (and always pass control to r). + return r(ctx, e, z, err, ch) + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return ch(ctx, e, z, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 || r == nil { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if forward() { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) + } +} + +// Drop aborts the Chain and returns the (context.Context, *executor.Call, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Call, mesos.Response, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, _ Chain) (context.Context, *executor.Call, mesos.Response, error) { + return r.Eval(ctx, e, z, err, chainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if err != nil { + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go new file mode 100644 index 00000000..99f34c32 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -0,0 +1,554 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func prototype() *executor.Call { return &executor.Call{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + t.Log("executing", name) + return r(ctx, e, z, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + var z0 mesos.Response + + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + var z0 mesos.Response + var zp = &mesos.ResponseWrapper{} + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *executor.Call + z mesos.Response + err error + }{ + {nil, z0, nil}, + {nil, z0, a}, + {p, z0, nil}, + {p, z0, a}, + + {nil, zp, nil}, + {nil, zp, a}, + {p, zp, nil}, + {p, zp, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, chainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, z, err + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, chainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + var zp = &mesos.ResponseWrapper{} + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestPoll(t *testing.T) { + var ( + ch1 <-chan struct{} // always nil + ch2 = make(chan struct{}) // non-nil, blocking + ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking + ch4 = make(chan struct{}) // non-nil, closed + ) + ch3 <- struct{}{} + close(ch4) + for ti, tc := range []struct { + ch <-chan struct{} + wantsRuleCount []int + }{ + {ch1, []int{0, 0, 0, 0}}, + {ch2, []int{0, 0, 0, 0}}, + {ch3, []int{1, 1, 1, 1}}, + {ch4, []int{1, 2, 2, 2}}, + } { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Poll(tc.ch).Eval + r2 = Rule(nil).Poll(tc.ch).Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 2; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := (k * 2) + x + 1; j != y { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", + ti, y, j) + } + } + } + } +} diff --git a/api/v1/lib/extras/executor/callrules/gen.go b/api/v1/lib/extras/executor/callrules/gen.go new file mode 100644 index 00000000..1b7dcac9 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/gen.go @@ -0,0 +1,7 @@ +package callrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} + +//go:generate go run ../../gen/rule_callers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/calls -type E:*executor.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call -type Z:mesos.Response -output metrics_generated.go diff --git a/api/v1/lib/extras/executor/callrules/metrics_generated.go b/api/v1/lib/extras/executor/callrules/metrics_generated.go new file mode 100644 index 00000000..c3c6231b --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/metrics_generated.go @@ -0,0 +1,25 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call -type Z:mesos.Response -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + typename := strings.ToLower(e.GetType().String()) + harness(func() error { + ctx, e, z, err = ch(ctx, e, z, err) + return err + }, typename) + return ctx, e, z, err + } +} diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go new file mode 100644 index 00000000..41ff7410 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -0,0 +1,301 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *executor.Event, error, Chain) (context.Context, *executor.Event, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *executor.Event, error, Chain) (context.Context, *executor.Event, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *executor.Event, error) (context.Context, *executor.Event, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) + + // chainIdentity is a Chain that returns the arguments as its results. + chainIdentity = func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + return ctx, e, err + } +) + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if r != nil { + return r(ctx, e, err, ch) + } + return ch(ctx, e, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(rs.Chain()(ctx, e, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *executor.Event, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return chainIdentity + } + return func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + return rs[0].Eval(ctx, e, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, err = r(ctx, e, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, err = ch(ctx, e, err) + } + return ctx, e, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. +// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + if p == nil || r == nil { + return nil + } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + select { + case <-p: + // do something + // TODO(jdef): optimization: if we detect the chan is closed, affect a state change + // whereby this select is no longer invoked (and always pass control to r). + return r(ctx, e, err, ch) + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return ch(ctx, e, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 || r == nil { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if forward() { + return r(ctx, e, err, ch) + } + return ch(ctx, e, err) + } +} + +// Drop aborts the Chain and returns the (context.Context, *executor.Event, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Event, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *executor.Event, err error, _ Chain) (context.Context, *executor.Event, error) { + return r.Eval(ctx, e, err, chainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(ctx, e, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if err != nil { + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go new file mode 100644 index 00000000..70fac3e1 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -0,0 +1,488 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func prototype() *executor.Event { return &executor.Event{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + t.Log("executing", name) + return r(ctx, e, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(ctx, e, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *executor.Event + err error + }{ + {nil, nil}, + {nil, a}, + {p, nil}, + {p, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, err + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestPoll(t *testing.T) { + var ( + ch1 <-chan struct{} // always nil + ch2 = make(chan struct{}) // non-nil, blocking + ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking + ch4 = make(chan struct{}) // non-nil, closed + ) + ch3 <- struct{}{} + close(ch4) + for ti, tc := range []struct { + ch <-chan struct{} + wantsRuleCount []int + }{ + {ch1, []int{0, 0, 0, 0}}, + {ch2, []int{0, 0, 0, 0}}, + {ch3, []int{1, 1, 1, 1}}, + {ch4, []int{1, 2, 2, 2}}, + } { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Poll(tc.ch).Eval + r2 = Rule(nil).Poll(tc.ch).Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 2; x++ { + _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != nil { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := (k * 2) + x + 1; j != y { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", + ti, y, j) + } + } + } + } +} diff --git a/api/v1/lib/extras/executor/eventrules/gen.go b/api/v1/lib/extras/executor/eventrules/gen.go new file mode 100644 index 00000000..ce43a6f8 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/gen.go @@ -0,0 +1,7 @@ +package eventrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} + +//go:generate go run ../../gen/rule_handlers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/events -type E:*executor.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event -output metrics_generated.go diff --git a/api/v1/lib/extras/executor/eventrules/handlers_generated.go b/api/v1/lib/extras/executor/eventrules/handlers_generated.go new file mode 100644 index 00000000..c9cf19be --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/handlers_generated.go @@ -0,0 +1,51 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/events -type E:*executor.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/executor" + "github.com/mesos/mesos-go/api/v1/lib/executor/events" +) + +// Handle generates a rule that executes the given events.Handler. +func Handle(h events.Handler) Rule { + if h == nil { + return nil + } + return func(ctx context.Context, e *executor.Event, err error, chain Chain) (context.Context, *executor.Event, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h events.HandlerFunc) Rule { + return Handle(events.Handler(h)) +} + +// Handle returns a Rule that invokes the receiver, then the given events.Handler +func (r Rule) Handle(h events.Handler) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h events.HandlerFunc) Rule { + return r.Handle(events.Handler(h)) +} + +// HandleEvent implements events.Handler for Rule +func (r Rule) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if r == nil { + return nil + } + _, _, err = r(ctx, e, nil, chainIdentity) + return +} + +// HandleEvent implements events.Handler for Rules +func (rs Rules) HandleEvent(ctx context.Context, e *executor.Event) error { + return Rule(rs.Eval).HandleEvent(ctx, e) +} diff --git a/api/v1/lib/extras/executor/eventrules/metrics_generated.go b/api/v1/lib/extras/executor/eventrules/metrics_generated.go new file mode 100644 index 00000000..2b531176 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/metrics_generated.go @@ -0,0 +1,24 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func Metrics(harness metrics.Harness) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + typename := strings.ToLower(e.GetType().String()) + harness(func() error { + ctx, e, err = ch(ctx, e, err) + return err + }, typename) + return ctx, e, err + } +} From 55e05e3ca6ed824bb20f00e3904b20ebeb892391 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 16:07:08 +0000 Subject: [PATCH 58/67] rules: rename Poll to RateLimit, add Overflow parameter --- .../executor/callrules/callrules_generated.go | 59 ++++++++++++++--- .../callrules/callrules_generated_test.go | 6 +- .../eventrules/eventrules_generated.go | 59 ++++++++++++++--- .../eventrules/eventrules_generated_test.go | 6 +- api/v1/lib/extras/gen/rules.go | 65 +++++++++++++++---- .../callrules/callrules_generated.go | 59 ++++++++++++++--- .../callrules/callrules_generated_test.go | 6 +- .../eventrules/eventrules_generated.go | 59 ++++++++++++++--- .../eventrules/eventrules_generated_test.go | 6 +- 9 files changed, 270 insertions(+), 55 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 73f28d17..1051202e 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -5,6 +5,7 @@ package callrules import ( "context" + "errors" "fmt" "sync" @@ -191,23 +192,65 @@ func (r Rule) Once() Rule { } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. -// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. -func (r Rule) Poll(p <-chan struct{}) Rule { +type Overflow int + +const ( + // OverflowDiscard aborts the rule chain and returns the current state + OverflowDiscard Overflow = iota + // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow + OverflowDiscardWithError + // OverflowBackpressure waits until the rule may execute, or the context is canceled. + OverflowBackpressure + // OverflowSkipRule skips over the decorated rule and continues processing the rule chain + OverflowSkipRule + // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipRuleWithError +) + +var ErrOverflow = errors.New("overflow: rate limit exceeded") + +// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// A nil chan will always skip the rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { if p == nil || r == nil { return nil } return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + checkTieBreaker := func() (context.Context, *executor.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } select { case <-p: - // do something - // TODO(jdef): optimization: if we detect the chan is closed, affect a state change - // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, z, err, ch) + return checkTieBreaker() case <-ctx.Done(): return ctx, e, z, Error2(err, ctx.Err()) default: - return ch(ctx, e, z, err) + // overflow + switch over { + case OverflowBackpressure: + select { + case <-p: + return checkTieBreaker() + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + } + case OverflowDiscardWithError: + return ctx, e, z, Error2(err, ErrOverflow) + case OverflowDiscard: + return ctx, e, z, err + case OverflowSkipRuleWithError: + return ch(ctx, e, z, Error2(err, ErrOverflow)) + case OverflowSkipRule: + return ch(ctx, e, z, err) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } } } } diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index 99f34c32..e805a050 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -502,7 +502,7 @@ func TestOnce(t *testing.T) { } } -func TestPoll(t *testing.T) { +func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil ch2 = make(chan struct{}) // non-nil, blocking @@ -524,8 +524,8 @@ func TestPoll(t *testing.T) { i, j int p = prototype() ctx = context.Background() - r1 = counter(&i).Poll(tc.ch).Eval - r2 = Rule(nil).Poll(tc.ch).Eval + r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval + r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 41ff7410..577426bb 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -5,6 +5,7 @@ package eventrules import ( "context" + "errors" "fmt" "sync" @@ -190,23 +191,65 @@ func (r Rule) Once() Rule { } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. -// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. -func (r Rule) Poll(p <-chan struct{}) Rule { +type Overflow int + +const ( + // OverflowDiscard aborts the rule chain and returns the current state + OverflowDiscard Overflow = iota + // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow + OverflowDiscardWithError + // OverflowBackpressure waits until the rule may execute, or the context is canceled. + OverflowBackpressure + // OverflowSkipRule skips over the decorated rule and continues processing the rule chain + OverflowSkipRule + // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipRuleWithError +) + +var ErrOverflow = errors.New("overflow: rate limit exceeded") + +// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// A nil chan will always skip the rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { if p == nil || r == nil { return nil } return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + checkTieBreaker := func() (context.Context, *executor.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } select { case <-p: - // do something - // TODO(jdef): optimization: if we detect the chan is closed, affect a state change - // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, err, ch) + return checkTieBreaker() case <-ctx.Done(): return ctx, e, Error2(err, ctx.Err()) default: - return ch(ctx, e, err) + // overflow + switch over { + case OverflowBackpressure: + select { + case <-p: + return checkTieBreaker() + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + } + case OverflowDiscardWithError: + return ctx, e, Error2(err, ErrOverflow) + case OverflowDiscard: + return ctx, e, err + case OverflowSkipRuleWithError: + return ch(ctx, e, Error2(err, ErrOverflow)) + case OverflowSkipRule: + return ch(ctx, e, err) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } } } } diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 70fac3e1..0008b9df 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -440,7 +440,7 @@ func TestOnce(t *testing.T) { } } -func TestPoll(t *testing.T) { +func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil ch2 = make(chan struct{}) // non-nil, blocking @@ -462,8 +462,8 @@ func TestPoll(t *testing.T) { i, j int p = prototype() ctx = context.Background() - r1 = counter(&i).Poll(tc.ch).Eval - r2 = Rule(nil).Poll(tc.ch).Eval + r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval + r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 8838c473..b0f992d8 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -18,6 +18,7 @@ var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} import ( "context" + "errors" "fmt" "sync" {{range .Imports}} @@ -207,23 +208,65 @@ func (r Rule) Once() Rule { } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. -// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. -func (r Rule) Poll(p <-chan struct{}) Rule { +type Overflow int + +const ( + // OverflowDiscard aborts the rule chain and returns the current state + OverflowDiscard Overflow = iota + // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow + OverflowDiscardWithError + // OverflowBackpressure waits until the rule may execute, or the context is canceled. + OverflowBackpressure + // OverflowSkipRule skips over the decorated rule and continues processing the rule chain + OverflowSkipRule + // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipRuleWithError +) + +var ErrOverflow = errors.New("overflow: rate limit exceeded") + +// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// A nil chan will always skip the rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { if p == nil || r == nil { return nil } return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + checkTieBreaker := func() (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + select { + case <-ctx.Done(): + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) + default: + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + } select { case <-p: - // do something - // TODO(jdef): optimization: if we detect the chan is closed, affect a state change - // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + return checkTieBreaker() case <-ctx.Done(): return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) default: - return ch(ctx, e, {{.Ref "Z" "z," -}} err) + // overflow + switch over { + case OverflowBackpressure: + select { + case <-p: + return checkTieBreaker() + case <-ctx.Done(): + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) + } + case OverflowDiscardWithError: + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow) + case OverflowDiscard: + return ctx, e, {{.Ref "Z" "z," -}} err + case OverflowSkipRuleWithError: + return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow)) + case OverflowSkipRule: + return ch(ctx, e, {{.Ref "Z" "z," -}} err) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } } } } @@ -873,7 +916,7 @@ func TestOnce(t *testing.T) { } } -func TestPoll(t *testing.T) { +func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil ch2 = make(chan struct{}) // non-nil, blocking @@ -895,8 +938,8 @@ func TestPoll(t *testing.T) { i, j int p = prototype() ctx = context.Background() - r1 = counter(&i).Poll(tc.ch).Eval - r2 = Rule(nil).Poll(tc.ch).Eval + r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval + r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval ) {{if .Type "Z" -}} var zp = {{.Prototype "Z"}} diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index ef779f63..2d0ee589 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -5,6 +5,7 @@ package callrules import ( "context" + "errors" "fmt" "sync" @@ -191,23 +192,65 @@ func (r Rule) Once() Rule { } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. -// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. -func (r Rule) Poll(p <-chan struct{}) Rule { +type Overflow int + +const ( + // OverflowDiscard aborts the rule chain and returns the current state + OverflowDiscard Overflow = iota + // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow + OverflowDiscardWithError + // OverflowBackpressure waits until the rule may execute, or the context is canceled. + OverflowBackpressure + // OverflowSkipRule skips over the decorated rule and continues processing the rule chain + OverflowSkipRule + // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipRuleWithError +) + +var ErrOverflow = errors.New("overflow: rate limit exceeded") + +// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// A nil chan will always skip the rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { if p == nil || r == nil { return nil } return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + checkTieBreaker := func() (context.Context, *scheduler.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } select { case <-p: - // do something - // TODO(jdef): optimization: if we detect the chan is closed, affect a state change - // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, z, err, ch) + return checkTieBreaker() case <-ctx.Done(): return ctx, e, z, Error2(err, ctx.Err()) default: - return ch(ctx, e, z, err) + // overflow + switch over { + case OverflowBackpressure: + select { + case <-p: + return checkTieBreaker() + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + } + case OverflowDiscardWithError: + return ctx, e, z, Error2(err, ErrOverflow) + case OverflowDiscard: + return ctx, e, z, err + case OverflowSkipRuleWithError: + return ch(ctx, e, z, Error2(err, ErrOverflow)) + case OverflowSkipRule: + return ch(ctx, e, z, err) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } } } } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 7268efa7..6adaa9ca 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -502,7 +502,7 @@ func TestOnce(t *testing.T) { } } -func TestPoll(t *testing.T) { +func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil ch2 = make(chan struct{}) // non-nil, blocking @@ -524,8 +524,8 @@ func TestPoll(t *testing.T) { i, j int p = prototype() ctx = context.Background() - r1 = counter(&i).Poll(tc.ch).Eval - r2 = Rule(nil).Poll(tc.ch).Eval + r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval + r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index c3fa03d0..81cc7114 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -5,6 +5,7 @@ package eventrules import ( "context" + "errors" "fmt" "sync" @@ -190,23 +191,65 @@ func (r Rule) Once() Rule { } } -// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it skips the rule. -// A nil chan will always skip the rule. May be useful, for example, when rate-limiting logged events. -func (r Rule) Poll(p <-chan struct{}) Rule { +type Overflow int + +const ( + // OverflowDiscard aborts the rule chain and returns the current state + OverflowDiscard Overflow = iota + // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow + OverflowDiscardWithError + // OverflowBackpressure waits until the rule may execute, or the context is canceled. + OverflowBackpressure + // OverflowSkipRule skips over the decorated rule and continues processing the rule chain + OverflowSkipRule + // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipRuleWithError +) + +var ErrOverflow = errors.New("overflow: rate limit exceeded") + +// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// A nil chan will always skip the rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { if p == nil || r == nil { return nil } return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + checkTieBreaker := func() (context.Context, *scheduler.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } select { case <-p: - // do something - // TODO(jdef): optimization: if we detect the chan is closed, affect a state change - // whereby this select is no longer invoked (and always pass control to r). - return r(ctx, e, err, ch) + return checkTieBreaker() case <-ctx.Done(): return ctx, e, Error2(err, ctx.Err()) default: - return ch(ctx, e, err) + // overflow + switch over { + case OverflowBackpressure: + select { + case <-p: + return checkTieBreaker() + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + } + case OverflowDiscardWithError: + return ctx, e, Error2(err, ErrOverflow) + case OverflowDiscard: + return ctx, e, err + case OverflowSkipRuleWithError: + return ch(ctx, e, Error2(err, ErrOverflow)) + case OverflowSkipRule: + return ch(ctx, e, err) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } } } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 0e42d7dc..3f88ff86 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -440,7 +440,7 @@ func TestOnce(t *testing.T) { } } -func TestPoll(t *testing.T) { +func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil ch2 = make(chan struct{}) // non-nil, blocking @@ -462,8 +462,8 @@ func TestPoll(t *testing.T) { i, j int p = prototype() ctx = context.Background() - r1 = counter(&i).Poll(tc.ch).Eval - r2 = Rule(nil).Poll(tc.ch).Eval + r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval + r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval ) for k, r := range []Rule{r1, r2} { for x := 0; x < 2; x++ { From 5ddd39e079a598f6541ffdcd0dd6283f4602e7db Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 5 Jun 2017 16:08:19 +0000 Subject: [PATCH 59/67] msh: removed redundant line --- api/v1/cmd/msh/msh.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go index fb390759..45e46045 100644 --- a/api/v1/cmd/msh/msh.go +++ b/api/v1/cmd/msh/msh.go @@ -134,7 +134,6 @@ func buildClient() calls.Caller { func buildEventHandler(caller calls.Caller) events.Handler { logger := controller.LogEvents() return controller.LiftErrors().Handle(events.Handlers{ - scheduler.Event_FAILURE: logger, scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)}, scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), From a944cc11ef6a6310acd2c65991571080dca04bc7 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 9 Jun 2017 12:32:45 +0000 Subject: [PATCH 60/67] rules: additional RateLimit unit tests and bugfix w/ respect to Overflow --- .../executor/callrules/callrules_generated.go | 16 ++-- .../callrules/callrules_generated_test.go | 74 ++++++++++----- .../eventrules/eventrules_generated.go | 16 ++-- .../eventrules/eventrules_generated_test.go | 74 ++++++++++----- api/v1/lib/extras/gen/rules.go | 90 +++++++++++++------ .../callrules/callrules_generated.go | 16 ++-- .../callrules/callrules_generated_test.go | 74 ++++++++++----- .../eventrules/eventrules_generated.go | 16 ++-- .../eventrules/eventrules_generated_test.go | 74 ++++++++++----- 9 files changed, 310 insertions(+), 140 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 1051202e..14ad0eb4 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -201,19 +201,19 @@ const ( OverflowDiscardWithError // OverflowBackpressure waits until the rule may execute, or the context is canceled. OverflowBackpressure - // OverflowSkipRule skips over the decorated rule and continues processing the rule chain - OverflowSkipRule - // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipRuleWithError + // OverflowSkip skips over the decorated rule and continues processing the rule chain + OverflowSkip + // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipWithError ) var ErrOverflow = errors.New("overflow: rate limit exceeded") // RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// A nil chan will always skip the rule. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { - if p == nil || r == nil { + if r == nil { return nil } return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { @@ -244,9 +244,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return ctx, e, z, Error2(err, ErrOverflow) case OverflowDiscard: return ctx, e, z, err - case OverflowSkipRuleWithError: + case OverflowSkipWithError: return ch(ctx, e, z, Error2(err, ErrOverflow)) - case OverflowSkipRule: + case OverflowSkip: return ch(ctx, e, z, err) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index e805a050..0e128c93 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -503,50 +503,84 @@ func TestOnce(t *testing.T) { } func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } var ( - ch1 <-chan struct{} // always nil - ch2 = make(chan struct{}) // non-nil, blocking - ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking - ch4 = make(chan struct{}) // non-nil, closed + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + ch4 = make(chan struct{}) // non-nil, closed + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() ) - ch3 <- struct{}{} close(ch4) + // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { - ch <-chan struct{} - wantsRuleCount []int + ctx context.Context + ch <-chan struct{} + over Overflow + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int }{ - {ch1, []int{0, 0, 0, 0}}, - {ch2, []int{0, 0, 0, 0}}, - {ch3, []int{1, 1, 1, 1}}, - {ch4, []int{1, 2, 2, 2}}, + {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int p = prototype() - ctx = context.Background() - r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval - r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval + r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { + // execute each rule twice for x := 0; x < 2; x++ { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } - if err != nil { - t.Errorf("test case %d failed: unexpected error %v", ti, err) + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) } if y := tc.wantsRuleCount[k*2+x]; i != y { t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", ti, k, x, y, i) } - if y := (k * 2) + x + 1; j != y { - t.Errorf("test case %d failed: expected chain count of %d instead of %d", - ti, y, j) + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) } } } diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 577426bb..d60c8bd9 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -200,19 +200,19 @@ const ( OverflowDiscardWithError // OverflowBackpressure waits until the rule may execute, or the context is canceled. OverflowBackpressure - // OverflowSkipRule skips over the decorated rule and continues processing the rule chain - OverflowSkipRule - // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipRuleWithError + // OverflowSkip skips over the decorated rule and continues processing the rule chain + OverflowSkip + // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipWithError ) var ErrOverflow = errors.New("overflow: rate limit exceeded") // RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// A nil chan will always skip the rule. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { - if p == nil || r == nil { + if r == nil { return nil } return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { @@ -243,9 +243,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return ctx, e, Error2(err, ErrOverflow) case OverflowDiscard: return ctx, e, err - case OverflowSkipRuleWithError: + case OverflowSkipWithError: return ch(ctx, e, Error2(err, ErrOverflow)) - case OverflowSkipRule: + case OverflowSkip: return ch(ctx, e, err) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 0008b9df..7dc5c131 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -441,46 +441,80 @@ func TestOnce(t *testing.T) { } func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } var ( - ch1 <-chan struct{} // always nil - ch2 = make(chan struct{}) // non-nil, blocking - ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking - ch4 = make(chan struct{}) // non-nil, closed + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + ch4 = make(chan struct{}) // non-nil, closed + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() ) - ch3 <- struct{}{} close(ch4) + // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { - ch <-chan struct{} - wantsRuleCount []int + ctx context.Context + ch <-chan struct{} + over Overflow + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int }{ - {ch1, []int{0, 0, 0, 0}}, - {ch2, []int{0, 0, 0, 0}}, - {ch3, []int{1, 1, 1, 1}}, - {ch4, []int{1, 2, 2, 2}}, + {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int p = prototype() - ctx = context.Background() - r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval - r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval + r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain ) for k, r := range []Rule{r1, r2} { + // execute each rule twice for x := 0; x < 2; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } - if err != nil { - t.Errorf("test case %d failed: unexpected error %v", ti, err) + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) } if y := tc.wantsRuleCount[k*2+x]; i != y { t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", ti, k, x, y, i) } - if y := (k * 2) + x + 1; j != y { - t.Errorf("test case %d failed: expected chain count of %d instead of %d", - ti, y, j) + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) } } } diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index b0f992d8..0c4fbb99 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -217,19 +217,19 @@ const ( OverflowDiscardWithError // OverflowBackpressure waits until the rule may execute, or the context is canceled. OverflowBackpressure - // OverflowSkipRule skips over the decorated rule and continues processing the rule chain - OverflowSkipRule - // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipRuleWithError + // OverflowSkip skips over the decorated rule and continues processing the rule chain + OverflowSkip + // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipWithError ) var ErrOverflow = errors.New("overflow: rate limit exceeded") // RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// A nil chan will always skip the rule. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { - if p == nil || r == nil { + if r == nil { return nil } return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { @@ -260,9 +260,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow) case OverflowDiscard: return ctx, e, {{.Ref "Z" "z," -}} err - case OverflowSkipRuleWithError: + case OverflowSkipWithError: return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow)) - case OverflowSkipRule: + case OverflowSkip: return ch(ctx, e, {{.Ref "Z" "z," -}} err) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) @@ -917,36 +917,70 @@ func TestOnce(t *testing.T) { } func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } var ( - ch1 <-chan struct{} // always nil - ch2 = make(chan struct{}) // non-nil, blocking - ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking - ch4 = make(chan struct{}) // non-nil, closed + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + ch4 = make(chan struct{}) // non-nil, closed + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() ) - ch3 <- struct{}{} close(ch4) + // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { - ch <-chan struct{} - wantsRuleCount []int + ctx context.Context + ch <-chan struct{} + over Overflow + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int }{ - {ch1, []int{0, 0, 0, 0}}, - {ch2, []int{0, 0, 0, 0}}, - {ch3, []int{1, 1, 1, 1}}, - {ch4, []int{1, 2, 2, 2}}, + {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int p = prototype() - ctx = context.Background() - r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval - r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval + r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain ) {{if .Type "Z" -}} var zp = {{.Prototype "Z"}} {{end -}} for k, r := range []Rule{r1, r2} { + // execute each rule twice for x := 0; x < 2; x++ { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -955,16 +989,16 @@ func TestRateLimit(t *testing.T) { t.Errorf("expected return object %q instead of %q", zp, zz) } {{end -}} - if err != nil { - t.Errorf("test case %d failed: unexpected error %v", ti, err) + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) } if y := tc.wantsRuleCount[k*2+x]; i != y { t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", ti, k, x, y, i) } - if y := (k * 2) + x + 1; j != y { - t.Errorf("test case %d failed: expected chain count of %d instead of %d", - ti, y, j) + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) } } } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 2d0ee589..204273ff 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -201,19 +201,19 @@ const ( OverflowDiscardWithError // OverflowBackpressure waits until the rule may execute, or the context is canceled. OverflowBackpressure - // OverflowSkipRule skips over the decorated rule and continues processing the rule chain - OverflowSkipRule - // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipRuleWithError + // OverflowSkip skips over the decorated rule and continues processing the rule chain + OverflowSkip + // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipWithError ) var ErrOverflow = errors.New("overflow: rate limit exceeded") // RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// A nil chan will always skip the rule. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { - if p == nil || r == nil { + if r == nil { return nil } return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { @@ -244,9 +244,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return ctx, e, z, Error2(err, ErrOverflow) case OverflowDiscard: return ctx, e, z, err - case OverflowSkipRuleWithError: + case OverflowSkipWithError: return ch(ctx, e, z, Error2(err, ErrOverflow)) - case OverflowSkipRule: + case OverflowSkip: return ch(ctx, e, z, err) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 6adaa9ca..5928b2c7 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -503,50 +503,84 @@ func TestOnce(t *testing.T) { } func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } var ( - ch1 <-chan struct{} // always nil - ch2 = make(chan struct{}) // non-nil, blocking - ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking - ch4 = make(chan struct{}) // non-nil, closed + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + ch4 = make(chan struct{}) // non-nil, closed + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() ) - ch3 <- struct{}{} close(ch4) + // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { - ch <-chan struct{} - wantsRuleCount []int + ctx context.Context + ch <-chan struct{} + over Overflow + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int }{ - {ch1, []int{0, 0, 0, 0}}, - {ch2, []int{0, 0, 0, 0}}, - {ch3, []int{1, 1, 1, 1}}, - {ch4, []int{1, 2, 2, 2}}, + {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int p = prototype() - ctx = context.Background() - r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval - r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval + r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { + // execute each rule twice for x := 0; x < 2; x++ { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } if zz != zp { t.Errorf("expected return object %q instead of %q", zp, zz) } - if err != nil { - t.Errorf("test case %d failed: unexpected error %v", ti, err) + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) } if y := tc.wantsRuleCount[k*2+x]; i != y { t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", ti, k, x, y, i) } - if y := (k * 2) + x + 1; j != y { - t.Errorf("test case %d failed: expected chain count of %d instead of %d", - ti, y, j) + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) } } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 81cc7114..c713c52f 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -200,19 +200,19 @@ const ( OverflowDiscardWithError // OverflowBackpressure waits until the rule may execute, or the context is canceled. OverflowBackpressure - // OverflowSkipRule skips over the decorated rule and continues processing the rule chain - OverflowSkipRule - // OverflowSkipRuleWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipRuleWithError + // OverflowSkip skips over the decorated rule and continues processing the rule chain + OverflowSkip + // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain + OverflowSkipWithError ) var ErrOverflow = errors.New("overflow: rate limit exceeded") // RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// A nil chan will always skip the rule. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { - if p == nil || r == nil { + if r == nil { return nil } return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { @@ -243,9 +243,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return ctx, e, Error2(err, ErrOverflow) case OverflowDiscard: return ctx, e, err - case OverflowSkipRuleWithError: + case OverflowSkipWithError: return ch(ctx, e, Error2(err, ErrOverflow)) - case OverflowSkipRule: + case OverflowSkip: return ch(ctx, e, err) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 3f88ff86..a6643289 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -441,46 +441,80 @@ func TestOnce(t *testing.T) { } func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } var ( - ch1 <-chan struct{} // always nil - ch2 = make(chan struct{}) // non-nil, blocking - ch3 = make(chan struct{}, 1) // non-nil, non-blocking then blocking - ch4 = make(chan struct{}) // non-nil, closed + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + ch4 = make(chan struct{}) // non-nil, closed + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() ) - ch3 <- struct{}{} close(ch4) + // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { - ch <-chan struct{} - wantsRuleCount []int + ctx context.Context + ch <-chan struct{} + over Overflow + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int }{ - {ch1, []int{0, 0, 0, 0}}, - {ch2, []int{0, 0, 0, 0}}, - {ch3, []int{1, 1, 1, 1}}, - {ch4, []int{1, 2, 2, 2}}, + {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int p = prototype() - ctx = context.Background() - r1 = counter(&i).RateLimit(tc.ch, OverflowSkipRule).Eval - r2 = Rule(nil).RateLimit(tc.ch, OverflowSkipRule).Eval + r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain ) for k, r := range []Rule{r1, r2} { + // execute each rule twice for x := 0; x < 2; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } - if err != nil { - t.Errorf("test case %d failed: unexpected error %v", ti, err) + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) } if y := tc.wantsRuleCount[k*2+x]; i != y { t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", ti, k, x, y, i) } - if y := (k * 2) + x + 1; j != y { - t.Errorf("test case %d failed: expected chain count of %d instead of %d", - ti, y, j) + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) } } } From 289fdd0d3666759135bf2753d8fdd55cb5b2e23b Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 9 Jun 2017 13:42:31 +0000 Subject: [PATCH 61/67] rules: separate UnlessDone from RateLimit, refactor tests --- .../executor/callrules/callrules_generated.go | 37 ++++--- .../callrules/callrules_generated_test.go | 63 +++++++++-- .../eventrules/eventrules_generated.go | 37 ++++--- .../eventrules/eventrules_generated_test.go | 59 ++++++++-- api/v1/lib/extras/gen/rules.go | 104 +++++++++++++----- .../callrules/callrules_generated.go | 37 ++++--- .../callrules/callrules_generated_test.go | 63 +++++++++-- .../eventrules/eventrules_generated.go | 37 ++++--- .../eventrules/eventrules_generated_test.go | 59 ++++++++-- 9 files changed, 361 insertions(+), 135 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 14ad0eb4..3b6e9d2c 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -192,6 +192,23 @@ func (r Rule) Once() Rule { } } +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } +} + type Overflow int const ( @@ -217,29 +234,14 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return nil } return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - checkTieBreaker := func() (context.Context, *executor.Call, mesos.Response, error) { - select { - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) - default: - return r(ctx, e, z, err, ch) - } - } select { case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) + // continue default: // overflow switch over { case OverflowBackpressure: - select { - case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) - } + <-p case OverflowDiscardWithError: return ctx, e, z, Error2(err, ErrOverflow) case OverflowDiscard: @@ -252,6 +254,7 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } + return r(ctx, e, z, err, ch) } } diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index 0e128c93..b7064496 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -167,6 +167,56 @@ func TestError2(t *testing.T) { } } +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + func TestAndThen(t *testing.T) { var ( i, j int @@ -514,14 +564,8 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() - fin = func() context.Context { - c, cancel := context.WithCancel(context.Background()) - cancel() - return c - }() ) close(ch4) - // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { ctx context.Context ch <-chan struct{} @@ -531,28 +575,27 @@ func TestRateLimit(t *testing.T) { wantsChainCount []int }{ {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + // TODO(jdef): test OverflowBackpressure (blocking) + {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index d60c8bd9..3b1af3f6 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -191,6 +191,23 @@ func (r Rule) Once() Rule { } } +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } +} + type Overflow int const ( @@ -216,29 +233,14 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return nil } return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - checkTieBreaker := func() (context.Context, *executor.Event, error) { - select { - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) - default: - return r(ctx, e, err, ch) - } - } select { case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) + // continue default: // overflow switch over { case OverflowBackpressure: - select { - case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) - } + <-p case OverflowDiscardWithError: return ctx, e, Error2(err, ErrOverflow) case OverflowDiscard: @@ -251,6 +253,7 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } + return r(ctx, e, err, ch) } } diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 7dc5c131..2add12ff 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -150,6 +150,52 @@ func TestError2(t *testing.T) { } } +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + func TestAndThen(t *testing.T) { var ( i, j int @@ -452,14 +498,8 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() - fin = func() context.Context { - c, cancel := context.WithCancel(context.Background()) - cancel() - return c - }() ) close(ch4) - // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { ctx context.Context ch <-chan struct{} @@ -469,28 +509,27 @@ func TestRateLimit(t *testing.T) { wantsChainCount []int }{ {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + // TODO(jdef): test OverflowBackpressure (blocking) + {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 0c4fbb99..0d16327d 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -208,6 +208,23 @@ func (r Rule) Once() Rule { } } +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + select { + case <-ctx.Done(): + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) + default: + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + } +} + type Overflow int const ( @@ -233,29 +250,14 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return nil } return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - checkTieBreaker := func() (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - select { - case <-ctx.Done(): - return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) - default: - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) - } - } select { case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) + // continue default: // overflow switch over { case OverflowBackpressure: - select { - case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) - } + <-p case OverflowDiscardWithError: return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow) case OverflowDiscard: @@ -268,6 +270,7 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } } @@ -539,6 +542,60 @@ func TestError2(t *testing.T) { } } +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + {{end -}} + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + func TestAndThen(t *testing.T) { var ( i, j int @@ -928,14 +985,8 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() - fin = func() context.Context { - c, cancel := context.WithCancel(context.Background()) - cancel() - return c - }() ) close(ch4) - // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { ctx context.Context ch <-chan struct{} @@ -945,28 +996,27 @@ func TestRateLimit(t *testing.T) { wantsChainCount []int }{ {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + // TODO(jdef): test OverflowBackpressure (blocking) + {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 204273ff..023ac7ae 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -192,6 +192,23 @@ func (r Rule) Once() Rule { } } +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } +} + type Overflow int const ( @@ -217,29 +234,14 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return nil } return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - checkTieBreaker := func() (context.Context, *scheduler.Call, mesos.Response, error) { - select { - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) - default: - return r(ctx, e, z, err, ch) - } - } select { case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) + // continue default: // overflow switch over { case OverflowBackpressure: - select { - case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, z, Error2(err, ctx.Err()) - } + <-p case OverflowDiscardWithError: return ctx, e, z, Error2(err, ErrOverflow) case OverflowDiscard: @@ -252,6 +254,7 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } + return r(ctx, e, z, err, ch) } } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 5928b2c7..a884de19 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -167,6 +167,56 @@ func TestError2(t *testing.T) { } } +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + func TestAndThen(t *testing.T) { var ( i, j int @@ -514,14 +564,8 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() - fin = func() context.Context { - c, cancel := context.WithCancel(context.Background()) - cancel() - return c - }() ) close(ch4) - // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { ctx context.Context ch <-chan struct{} @@ -531,28 +575,27 @@ func TestRateLimit(t *testing.T) { wantsChainCount []int }{ {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + // TODO(jdef): test OverflowBackpressure (blocking) + {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index c713c52f..6b613386 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -191,6 +191,23 @@ func (r Rule) Once() Rule { } } +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } +} + type Overflow int const ( @@ -216,29 +233,14 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { return nil } return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - checkTieBreaker := func() (context.Context, *scheduler.Event, error) { - select { - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) - default: - return r(ctx, e, err, ch) - } - } select { case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) + // continue default: // overflow switch over { case OverflowBackpressure: - select { - case <-p: - return checkTieBreaker() - case <-ctx.Done(): - return ctx, e, Error2(err, ctx.Err()) - } + <-p case OverflowDiscardWithError: return ctx, e, Error2(err, ErrOverflow) case OverflowDiscard: @@ -251,6 +253,7 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } + return r(ctx, e, err, ch) } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index a6643289..d2334e36 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -150,6 +150,52 @@ func TestError2(t *testing.T) { } } +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + func TestAndThen(t *testing.T) { var ( i, j int @@ -452,14 +498,8 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() - fin = func() context.Context { - c, cancel := context.WithCancel(context.Background()) - cancel() - return c - }() ) close(ch4) - // TODO(jdef): unit test for OverflowBackpressure for ti, tc := range []struct { ctx context.Context ch <-chan struct{} @@ -469,28 +509,27 @@ func TestRateLimit(t *testing.T) { wantsChainCount []int }{ {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkip, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {fin, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscard, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {fin, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + // TODO(jdef): test OverflowBackpressure (blocking) + {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int From 38fa54ff86ace7fe1f6d13443c7aa393d454e4f5 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 10 Jun 2017 19:06:26 +0000 Subject: [PATCH 62/67] rules: make ChainIdentity public; simplify Overflow and RateLimit, more unit testing --- .../executor/callrules/callers_generated.go | 2 +- .../executor/callrules/callrules_generated.go | 103 +++++--- .../callrules/callrules_generated_test.go | 123 +++++---- .../eventrules/eventrules_generated.go | 103 +++++--- .../eventrules/eventrules_generated_test.go | 118 +++++---- .../executor/eventrules/handlers_generated.go | 2 +- api/v1/lib/extras/gen/gen.go | 3 +- api/v1/lib/extras/gen/rule_callers.go | 2 +- api/v1/lib/extras/gen/rule_handlers.go | 2 +- api/v1/lib/extras/gen/rules.go | 234 +++++++++++------- .../scheduler/callrules/callers_generated.go | 2 +- .../callrules/callrules_generated.go | 103 +++++--- .../callrules/callrules_generated_test.go | 123 +++++---- .../eventrules/eventrules_generated.go | 103 +++++--- .../eventrules/eventrules_generated_test.go | 118 +++++---- .../eventrules/handlers_generated.go | 2 +- 16 files changed, 715 insertions(+), 428 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callers_generated.go b/api/v1/lib/extras/executor/callrules/callers_generated.go index bb0fe027..dd03eb0f 100644 --- a/api/v1/lib/extras/executor/callrules/callers_generated.go +++ b/api/v1/lib/extras/executor/callrules/callers_generated.go @@ -43,7 +43,7 @@ func (r Rule) Call(ctx context.Context, c *executor.Call) (mesos.Response, error if r == nil { return nil, nil } - _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) return resp, err } diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 3b6e9d2c..dfd11fe7 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -5,7 +5,6 @@ package callrules import ( "context" - "errors" "fmt" "sync" @@ -44,13 +43,13 @@ type ( var ( _ = evaler(Rule(nil)) _ = evaler(Rules{}) - - // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { - return ctx, e, z, err - } ) +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + return ctx, e, z, err +} + // Eval is a convenience func that processes a nil Rule as a noop. func (r Rule) Eval(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { if r != nil { @@ -69,7 +68,7 @@ func (rs Rules) Eval(ctx context.Context, e *executor.Call, z mesos.Response, er // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { - return chainIdentity + return ChainIdentity } return func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) @@ -212,44 +211,65 @@ func (r Rule) UnlessDone() Rule { type Overflow int const ( - // OverflowDiscard aborts the rule chain and returns the current state - OverflowDiscard Overflow = iota - // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow - OverflowDiscardWithError - // OverflowBackpressure waits until the rule may execute, or the context is canceled. - OverflowBackpressure - // OverflowSkip skips over the decorated rule and continues processing the rule chain - OverflowSkip - // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipWithError + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise ) -var ErrOverflow = errors.New("overflow: rate limit exceeded") - -// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. -func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + if r != nil && p == nil && over == OverflowWait { + panic("deadlock detected: reads from token chan will permanently block rule processing") + } + return rateLimit(r, acquireFunc(p), over, otherwise) +} + +// acquireFunc wraps a signal chan with a func that can be used with rateLimit. +// should only be called by RateLimit (because it implements deadlock detection). +func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { + if tokenCh == nil { + // always false/blocked: acquire never succeeds + return func(block bool) bool { + if block { + panic("deadlock detected: block should never be true when the token chan is nil") + } + return false + } + } + return func(block bool) bool { + if block { + <-tokenCh + return true + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// rateLimit is more easily unit tested than RateLimit. +func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - select { - case <-p: - // continue - default: - // overflow + if !acquire(false) { + // non-blocking acquire failed, check overflow policy switch over { - case OverflowBackpressure: - <-p - case OverflowDiscardWithError: - return ctx, e, z, Error2(err, ErrOverflow) - case OverflowDiscard: - return ctx, e, z, err - case OverflowSkipWithError: - return ch(ctx, e, z, Error2(err, ErrOverflow)) - case OverflowSkip: - return ch(ctx, e, z, err) + case OverflowWait: + _ = acquire(true) // block until there's a signal + case OverflowOtherwise: + return otherwise.Eval(ctx, e, z, err, ch) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } @@ -259,8 +279,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). -func (r Rule) EveryN(nthTime int) Rule { +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } @@ -283,7 +304,7 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(ctx, e, z, err, ch) } - return ch(ctx, e, z, err) + return otherwise.Eval(ctx, e, z, err, ch) } } @@ -295,7 +316,7 @@ func Drop() Rule { // ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Call, mesos.Response, error) tuple as-is. func (r Rule) ThenDrop() Rule { return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, _ Chain) (context.Context, *executor.Call, mesos.Response, error) { - return r.Eval(ctx, e, z, err, chainIdentity) + return r.Eval(ctx, e, z, err, ChainIdentity) } } diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index b7064496..0b5a58f2 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -42,13 +42,19 @@ func chainCounter(i *int, ch Chain) Chain { } } +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *executor.Call, _ mesos.Response, _ error) (context.Context, *executor.Call, mesos.Response, error) { + panic(x) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) var z0 mesos.Response - _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, chainIdentity) + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, ChainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -94,7 +100,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, chainIdentity) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, ChainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -110,7 +116,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, z, err - _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, chainIdentity) + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, ChainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -193,7 +199,7 @@ func TestUnlessDone(t *testing.T) { r2 = r1.UnlessDone() ) for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -228,7 +234,7 @@ func TestAndThen(t *testing.T) { ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -264,7 +270,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -296,7 +302,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -313,7 +319,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -340,7 +346,7 @@ func TestDropOnSuccess(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -358,7 +364,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -376,7 +382,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -406,7 +412,7 @@ func TestThenDrop(t *testing.T) { var zp = &mesos.ResponseWrapper{} // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -439,7 +445,7 @@ func TestDrop(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -471,7 +477,7 @@ func TestIf(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -502,7 +508,7 @@ func TestUnless(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -532,7 +538,7 @@ func TestOnce(t *testing.T) { var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -564,50 +570,56 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() + p = prototype() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() ) + var zp = &mesos.ResponseWrapper{} close(ch4) for ti, tc := range []struct { ctx context.Context ch <-chan struct{} over Overflow + otherwise Rule wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit wantsRuleCount []int wantsChainCount []int }{ - {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - // TODO(jdef): test OverflowBackpressure (blocking) - {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int - p = prototype() - r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval - r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain ) - var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { // execute each rule twice for x := 0; x < 2; x++ { - _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -628,4 +640,29 @@ func TestRateLimit(t *testing.T) { } } } + // test blocking capability via rateLimit + blocked := false + r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } } diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 3b1af3f6..35a1e3c5 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -5,7 +5,6 @@ package eventrules import ( "context" - "errors" "fmt" "sync" @@ -43,13 +42,13 @@ type ( var ( _ = evaler(Rule(nil)) _ = evaler(Rules{}) - - // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { - return ctx, e, err - } ) +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + return ctx, e, err +} + // Eval is a convenience func that processes a nil Rule as a noop. func (r Rule) Eval(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { if r != nil { @@ -68,7 +67,7 @@ func (rs Rules) Eval(ctx context.Context, e *executor.Event, err error, ch Chain // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { - return chainIdentity + return ChainIdentity } return func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { return rs[0].Eval(ctx, e, err, rs[1:].Chain()) @@ -211,44 +210,65 @@ func (r Rule) UnlessDone() Rule { type Overflow int const ( - // OverflowDiscard aborts the rule chain and returns the current state - OverflowDiscard Overflow = iota - // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow - OverflowDiscardWithError - // OverflowBackpressure waits until the rule may execute, or the context is canceled. - OverflowBackpressure - // OverflowSkip skips over the decorated rule and continues processing the rule chain - OverflowSkip - // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipWithError + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise ) -var ErrOverflow = errors.New("overflow: rate limit exceeded") - -// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. -func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + if r != nil && p == nil && over == OverflowWait { + panic("deadlock detected: reads from token chan will permanently block rule processing") + } + return rateLimit(r, acquireFunc(p), over, otherwise) +} + +// acquireFunc wraps a signal chan with a func that can be used with rateLimit. +// should only be called by RateLimit (because it implements deadlock detection). +func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { + if tokenCh == nil { + // always false/blocked: acquire never succeeds + return func(block bool) bool { + if block { + panic("deadlock detected: block should never be true when the token chan is nil") + } + return false + } + } + return func(block bool) bool { + if block { + <-tokenCh + return true + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// rateLimit is more easily unit tested than RateLimit. +func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - select { - case <-p: - // continue - default: - // overflow + if !acquire(false) { + // non-blocking acquire failed, check overflow policy switch over { - case OverflowBackpressure: - <-p - case OverflowDiscardWithError: - return ctx, e, Error2(err, ErrOverflow) - case OverflowDiscard: - return ctx, e, err - case OverflowSkipWithError: - return ch(ctx, e, Error2(err, ErrOverflow)) - case OverflowSkip: - return ch(ctx, e, err) + case OverflowWait: + _ = acquire(true) // block until there's a signal + case OverflowOtherwise: + return otherwise.Eval(ctx, e, err, ch) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } @@ -258,8 +278,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). -func (r Rule) EveryN(nthTime int) Rule { +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } @@ -282,7 +303,7 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(ctx, e, err, ch) } - return ch(ctx, e, err) + return otherwise.Eval(ctx, e, err, ch) } } @@ -294,7 +315,7 @@ func Drop() Rule { // ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Event, error) tuple as-is. func (r Rule) ThenDrop() Rule { return func(ctx context.Context, e *executor.Event, err error, _ Chain) (context.Context, *executor.Event, error) { - return r.Eval(ctx, e, err, chainIdentity) + return r.Eval(ctx, e, err, ChainIdentity) } } diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 2add12ff..deb951fe 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -41,11 +41,17 @@ func chainCounter(i *int, ch Chain) Chain { } } +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *executor.Event, _ error) (context.Context, *executor.Event, error) { + panic(x) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) - _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, ChainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -83,7 +89,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) + _, e, err = rule(ctx, tc.e, tc.err, ChainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -96,7 +102,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, err - _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, ChainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -175,7 +181,7 @@ func TestUnlessDone(t *testing.T) { r2 = r1.UnlessDone() ) for k, r := range []Rule{r1, r2} { - _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -206,7 +212,7 @@ func TestAndThen(t *testing.T) { a = errors.New("a") ) for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -238,7 +244,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -266,7 +272,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -280,7 +286,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -303,7 +309,7 @@ func TestDropOnSuccess(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -318,7 +324,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -333,7 +339,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err = r3(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -359,7 +365,7 @@ func TestThenDrop(t *testing.T) { ) // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -388,7 +394,7 @@ func TestDrop(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -416,7 +422,7 @@ func TestIf(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -443,7 +449,7 @@ func TestUnless(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -469,7 +475,7 @@ func TestOnce(t *testing.T) { ) for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -498,49 +504,55 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() + p = prototype() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() ) close(ch4) for ti, tc := range []struct { ctx context.Context ch <-chan struct{} over Overflow + otherwise Rule wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit wantsRuleCount []int wantsChainCount []int }{ - {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - // TODO(jdef): test OverflowBackpressure (blocking) - {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int - p = prototype() - r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval - r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain ) for k, r := range []Rule{r1, r2} { // execute each rule twice for x := 0; x < 2; x++ { - _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -558,4 +570,26 @@ func TestRateLimit(t *testing.T) { } } } + // test blocking capability via rateLimit + blocked := false + r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, err := r(ctx, p, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } } diff --git a/api/v1/lib/extras/executor/eventrules/handlers_generated.go b/api/v1/lib/extras/executor/eventrules/handlers_generated.go index c9cf19be..1cf7ee8c 100644 --- a/api/v1/lib/extras/executor/eventrules/handlers_generated.go +++ b/api/v1/lib/extras/executor/eventrules/handlers_generated.go @@ -41,7 +41,7 @@ func (r Rule) HandleEvent(ctx context.Context, e *executor.Event) (err error) { if r == nil { return nil } - _, _, err = r(ctx, e, nil, chainIdentity) + _, _, err = r(ctx, e, nil, ChainIdentity) return } diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go index 2f2ebc47..fe88a517 100644 --- a/api/v1/lib/extras/gen/gen.go +++ b/api/v1/lib/extras/gen/gen.go @@ -24,7 +24,7 @@ type ( Config struct { Package string Imports []string - Args string // arguments that we were invoked with + Args string // arguments that we were invoked with; TODO(jdef) rename this to Flags? Types TypeMap } ) @@ -37,6 +37,7 @@ func (c *Config) String() string { } func (c *Config) Set(s string) error { + // TODO(jdef) this is gnarly, *Config shouldn't actually implement Set or String; define a new type for imports c.Imports = append(c.Imports, s) return nil } diff --git a/api/v1/lib/extras/gen/rule_callers.go b/api/v1/lib/extras/gen/rule_callers.go index 22156a94..e637f864 100644 --- a/api/v1/lib/extras/gen/rule_callers.go +++ b/api/v1/lib/extras/gen/rule_callers.go @@ -59,7 +59,7 @@ func (r Rule) Call(ctx context.Context, c {{.Type "E"}}) (mesos.Response, error) if r == nil { return nil, nil } - _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) return resp, err } diff --git a/api/v1/lib/extras/gen/rule_handlers.go b/api/v1/lib/extras/gen/rule_handlers.go index b5aff94a..23592364 100644 --- a/api/v1/lib/extras/gen/rule_handlers.go +++ b/api/v1/lib/extras/gen/rule_handlers.go @@ -57,7 +57,7 @@ func (r Rule) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { if r == nil { return nil } - _, _, err = r(ctx, e, nil, chainIdentity) + _, _, err = r(ctx, e, nil, ChainIdentity) return } diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 0d16327d..1a0735b0 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -18,7 +18,6 @@ var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} import ( "context" - "errors" "fmt" "sync" {{range .Imports}} @@ -28,7 +27,7 @@ import ( {{.RequireType "E" -}} {{.RequirePrototype "E" -}} -{{.RequirePrototype "Z" -}} +{{.RequirePrototype "Z" -}}{{/* Z is an optional type, but if it's specified then it needs a prototype */ -}} type ( evaler interface { // Eval executes a filter, rule, or decorator function; if the returned event is nil then @@ -60,13 +59,13 @@ type ( var ( _ = evaler(Rule(nil)) _ = evaler(Rules{}) - - // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - return ctx, e, {{.Ref "Z" "z," -}} err - } ) +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ctx, e, {{.Ref "Z" "z," -}} err +} + // Eval is a convenience func that processes a nil Rule as a noop. func (r Rule) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { if r != nil { @@ -85,7 +84,7 @@ func (rs Rules) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} e // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { - return chainIdentity + return ChainIdentity } return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { return rs[0].Eval(ctx, e, {{.Ref "Z" "z," -}} err, rs[1:].Chain()) @@ -228,44 +227,65 @@ func (r Rule) UnlessDone() Rule { type Overflow int const ( - // OverflowDiscard aborts the rule chain and returns the current state - OverflowDiscard Overflow = iota - // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow - OverflowDiscardWithError - // OverflowBackpressure waits until the rule may execute, or the context is canceled. - OverflowBackpressure - // OverflowSkip skips over the decorated rule and continues processing the rule chain - OverflowSkip - // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipWithError + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise ) -var ErrOverflow = errors.New("overflow: rate limit exceeded") - -// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. -func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + if r != nil && p == nil && over == OverflowWait { + panic("deadlock detected: reads from token chan will permanently block rule processing") + } + return rateLimit(r, acquireFunc(p), over, otherwise) +} + +// acquireFunc wraps a signal chan with a func that can be used with rateLimit. +// should only be called by RateLimit (because it implements deadlock detection). +func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { + if tokenCh == nil { + // always false/blocked: acquire never succeeds + return func(block bool) bool { + if block { + panic("deadlock detected: block should never be true when the token chan is nil") + } + return false + } + } + return func(block bool) bool { + if block { + <-tokenCh + return true + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// rateLimit is more easily unit tested than RateLimit. +func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - select { - case <-p: - // continue - default: - // overflow + if !acquire(false) { + // non-blocking acquire failed, check overflow policy switch over { - case OverflowBackpressure: - <-p - case OverflowDiscardWithError: - return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow) - case OverflowDiscard: - return ctx, e, {{.Ref "Z" "z," -}} err - case OverflowSkipWithError: - return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, ErrOverflow)) - case OverflowSkip: - return ch(ctx, e, {{.Ref "Z" "z," -}} err) + case OverflowWait: + _ = acquire(true) // block until there's a signal + case OverflowOtherwise: + return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } @@ -275,8 +295,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). -func (r Rule) EveryN(nthTime int) Rule { +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } @@ -299,7 +320,7 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } - return ch(ctx, e, {{.Ref "Z" "z," -}} err) + return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) } } @@ -311,7 +332,7 @@ func Drop() Rule { // ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) tuple as-is. func (r Rule) ThenDrop() Rule { return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, _ Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, chainIdentity) + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ChainIdentity) } } @@ -408,13 +429,19 @@ func chainCounter(i *int, ch Chain) Chain { } } +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ {{.Type "E"}}, {{.Arg "Z" "_," -}} _ error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + panic(x) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) {{if .Type "Z"}} {{.Var "Z" "z0"}} {{end}} - _, e, {{.Ref "Z" "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.Ref "Z" "z0," -}} nil, chainIdentity) + _, e, {{.Ref "Z" "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.Ref "Z" "z0," -}} nil, ChainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -465,7 +492,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, {{.Ref "Z" "zz," -}} err = rule(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, chainIdentity) + _, e, {{.Ref "Z" "zz," -}} err = rule(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, ChainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -483,7 +510,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, {{.Ref "Z" "z," -}} err - _, e, {{.Ref "Z" "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, chainIdentity) + _, e, {{.Ref "Z" "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, ChainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -570,7 +597,7 @@ func TestUnlessDone(t *testing.T) { r2 = r1.UnlessDone() ) for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -609,7 +636,7 @@ func TestAndThen(t *testing.T) { var zp = {{.Prototype "Z"}} {{end -}} for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -649,7 +676,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, {{.Ref "Z" "zz," -}} err := tc.r(ctx, p, {{.Ref "Z" "zp," -}} tc.initialError, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := tc.r(ctx, p, {{.Ref "Z" "zp," -}} tc.initialError, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -685,7 +712,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -704,7 +731,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -735,7 +762,7 @@ func TestDropOnSuccess(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -755,7 +782,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -775,7 +802,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, {{.Ref "Z" "zz," -}} err = r3(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err = r3(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -809,7 +836,7 @@ func TestThenDrop(t *testing.T) { {{end -}} // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -846,7 +873,7 @@ func TestDrop(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -882,7 +909,7 @@ func TestIf(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -917,7 +944,7 @@ func TestUnless(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -951,7 +978,7 @@ func TestOnce(t *testing.T) { {{end -}} for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -985,52 +1012,58 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() + p = prototype() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} close(ch4) for ti, tc := range []struct { ctx context.Context ch <-chan struct{} over Overflow + otherwise Rule wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit wantsRuleCount []int wantsChainCount []int }{ - {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - // TODO(jdef): test OverflowBackpressure (blocking) - {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int - p = prototype() - r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval - r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain ) - {{if .Type "Z" -}} - var zp = {{.Prototype "Z"}} - {{end -}} for k, r := range []Rule{r1, r2} { // execute each rule twice for x := 0; x < 2; x++ { - _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, chainIdentity)) + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -1053,5 +1086,32 @@ func TestRateLimit(t *testing.T) { } } } + // test blocking capability via rateLimit + blocked := false + r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } } `)) diff --git a/api/v1/lib/extras/scheduler/callrules/callers_generated.go b/api/v1/lib/extras/scheduler/callrules/callers_generated.go index 8343507c..f016e948 100644 --- a/api/v1/lib/extras/scheduler/callrules/callers_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callers_generated.go @@ -43,7 +43,7 @@ func (r Rule) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, erro if r == nil { return nil, nil } - _, _, resp, err := r(ctx, c, nil, nil, chainIdentity) + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) return resp, err } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 023ac7ae..23a8978d 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -5,7 +5,6 @@ package callrules import ( "context" - "errors" "fmt" "sync" @@ -44,13 +43,13 @@ type ( var ( _ = evaler(Rule(nil)) _ = evaler(Rules{}) - - // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { - return ctx, e, z, err - } ) +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + return ctx, e, z, err +} + // Eval is a convenience func that processes a nil Rule as a noop. func (r Rule) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { if r != nil { @@ -69,7 +68,7 @@ func (rs Rules) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, e // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { - return chainIdentity + return ChainIdentity } return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) @@ -212,44 +211,65 @@ func (r Rule) UnlessDone() Rule { type Overflow int const ( - // OverflowDiscard aborts the rule chain and returns the current state - OverflowDiscard Overflow = iota - // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow - OverflowDiscardWithError - // OverflowBackpressure waits until the rule may execute, or the context is canceled. - OverflowBackpressure - // OverflowSkip skips over the decorated rule and continues processing the rule chain - OverflowSkip - // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipWithError + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise ) -var ErrOverflow = errors.New("overflow: rate limit exceeded") - -// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. -func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + if r != nil && p == nil && over == OverflowWait { + panic("deadlock detected: reads from token chan will permanently block rule processing") + } + return rateLimit(r, acquireFunc(p), over, otherwise) +} + +// acquireFunc wraps a signal chan with a func that can be used with rateLimit. +// should only be called by RateLimit (because it implements deadlock detection). +func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { + if tokenCh == nil { + // always false/blocked: acquire never succeeds + return func(block bool) bool { + if block { + panic("deadlock detected: block should never be true when the token chan is nil") + } + return false + } + } + return func(block bool) bool { + if block { + <-tokenCh + return true + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// rateLimit is more easily unit tested than RateLimit. +func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - select { - case <-p: - // continue - default: - // overflow + if !acquire(false) { + // non-blocking acquire failed, check overflow policy switch over { - case OverflowBackpressure: - <-p - case OverflowDiscardWithError: - return ctx, e, z, Error2(err, ErrOverflow) - case OverflowDiscard: - return ctx, e, z, err - case OverflowSkipWithError: - return ch(ctx, e, z, Error2(err, ErrOverflow)) - case OverflowSkip: - return ch(ctx, e, z, err) + case OverflowWait: + _ = acquire(true) // block until there's a signal + case OverflowOtherwise: + return otherwise.Eval(ctx, e, z, err, ch) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } @@ -259,8 +279,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). -func (r Rule) EveryN(nthTime int) Rule { +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } @@ -283,7 +304,7 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(ctx, e, z, err, ch) } - return ch(ctx, e, z, err) + return otherwise.Eval(ctx, e, z, err, ch) } } @@ -295,7 +316,7 @@ func Drop() Rule { // ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Call, mesos.Response, error) tuple as-is. func (r Rule) ThenDrop() Rule { return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, _ Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - return r.Eval(ctx, e, z, err, chainIdentity) + return r.Eval(ctx, e, z, err, ChainIdentity) } } diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index a884de19..c3a5df15 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -42,13 +42,19 @@ func chainCounter(i *int, ch Chain) Chain { } } +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *scheduler.Call, _ mesos.Response, _ error) (context.Context, *scheduler.Call, mesos.Response, error) { + panic(x) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) var z0 mesos.Response - _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, chainIdentity) + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, ChainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -94,7 +100,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, chainIdentity) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, ChainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -110,7 +116,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, z, err - _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, chainIdentity) + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, ChainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -193,7 +199,7 @@ func TestUnlessDone(t *testing.T) { r2 = r1.UnlessDone() ) for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -228,7 +234,7 @@ func TestAndThen(t *testing.T) { ) var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -264,7 +270,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -296,7 +302,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -313,7 +319,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -340,7 +346,7 @@ func TestDropOnSuccess(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -358,7 +364,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, chainIdentity)) + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -376,7 +382,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -406,7 +412,7 @@ func TestThenDrop(t *testing.T) { var zp = &mesos.ResponseWrapper{} // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -439,7 +445,7 @@ func TestDrop(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -471,7 +477,7 @@ func TestIf(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -502,7 +508,7 @@ func TestUnless(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -532,7 +538,7 @@ func TestOnce(t *testing.T) { var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -564,50 +570,56 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() + p = prototype() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() ) + var zp = &mesos.ResponseWrapper{} close(ch4) for ti, tc := range []struct { ctx context.Context ch <-chan struct{} over Overflow + otherwise Rule wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit wantsRuleCount []int wantsChainCount []int }{ - {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - // TODO(jdef): test OverflowBackpressure (blocking) - {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int - p = prototype() - r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval - r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain ) - var zp = &mesos.ResponseWrapper{} for k, r := range []Rule{r1, r2} { // execute each rule twice for x := 0; x < 2; x++ { - _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, chainIdentity)) + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -628,4 +640,29 @@ func TestRateLimit(t *testing.T) { } } } + // test blocking capability via rateLimit + blocked := false + r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 6b613386..6a4d7194 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -5,7 +5,6 @@ package eventrules import ( "context" - "errors" "fmt" "sync" @@ -43,13 +42,13 @@ type ( var ( _ = evaler(Rule(nil)) _ = evaler(Rules{}) - - // chainIdentity is a Chain that returns the arguments as its results. - chainIdentity = func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { - return ctx, e, err - } ) +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + return ctx, e, err +} + // Eval is a convenience func that processes a nil Rule as a noop. func (r Rule) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { if r != nil { @@ -68,7 +67,7 @@ func (rs Rules) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chai // from Rule to Rule. Chain is safe to invoke concurrently. func (rs Rules) Chain() Chain { if len(rs) == 0 { - return chainIdentity + return ChainIdentity } return func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { return rs[0].Eval(ctx, e, err, rs[1:].Chain()) @@ -211,44 +210,65 @@ func (r Rule) UnlessDone() Rule { type Overflow int const ( - // OverflowDiscard aborts the rule chain and returns the current state - OverflowDiscard Overflow = iota - // OverflowDiscardWithError aborts the rule chain and returns the current state merged with ErrOverflow - OverflowDiscardWithError - // OverflowBackpressure waits until the rule may execute, or the context is canceled. - OverflowBackpressure - // OverflowSkip skips over the decorated rule and continues processing the rule chain - OverflowSkip - // OverflowSkipWithError skips over the decorated rule and merges ErrOverflow upon executing the chain - OverflowSkipWithError + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise ) -var ErrOverflow = errors.New("overflow: rate limit exceeded") - -// RateLimit invokes the receiving Rule if the chan is readable (may be closed), otherwise it handles the "overflow" +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will always trigger an overflow. -func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + if r != nil && p == nil && over == OverflowWait { + panic("deadlock detected: reads from token chan will permanently block rule processing") + } + return rateLimit(r, acquireFunc(p), over, otherwise) +} + +// acquireFunc wraps a signal chan with a func that can be used with rateLimit. +// should only be called by RateLimit (because it implements deadlock detection). +func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { + if tokenCh == nil { + // always false/blocked: acquire never succeeds + return func(block bool) bool { + if block { + panic("deadlock detected: block should never be true when the token chan is nil") + } + return false + } + } + return func(block bool) bool { + if block { + <-tokenCh + return true + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// rateLimit is more easily unit tested than RateLimit. +func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - select { - case <-p: - // continue - default: - // overflow + if !acquire(false) { + // non-blocking acquire failed, check overflow policy switch over { - case OverflowBackpressure: - <-p - case OverflowDiscardWithError: - return ctx, e, Error2(err, ErrOverflow) - case OverflowDiscard: - return ctx, e, err - case OverflowSkipWithError: - return ch(ctx, e, Error2(err, ErrOverflow)) - case OverflowSkip: - return ch(ctx, e, err) + case OverflowWait: + _ = acquire(true) // block until there's a signal + case OverflowOtherwise: + return otherwise.Eval(ctx, e, err, ch) default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } @@ -258,8 +278,9 @@ func (r Rule) RateLimit(p <-chan struct{}, over Overflow) Rule { } // EveryN invokes the receiving rule beginning with the first event seen and then every n'th -// time after that. If nthTime is less then 2 then this call is a noop (the receiver is returned). -func (r Rule) EveryN(nthTime int) Rule { +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } @@ -282,7 +303,7 @@ func (r Rule) EveryN(nthTime int) Rule { if forward() { return r(ctx, e, err, ch) } - return ch(ctx, e, err) + return otherwise.Eval(ctx, e, err, ch) } } @@ -294,7 +315,7 @@ func Drop() Rule { // ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Event, error) tuple as-is. func (r Rule) ThenDrop() Rule { return func(ctx context.Context, e *scheduler.Event, err error, _ Chain) (context.Context, *scheduler.Event, error) { - return r.Eval(ctx, e, err, chainIdentity) + return r.Eval(ctx, e, err, ChainIdentity) } } diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index d2334e36..9c4e4165 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -41,11 +41,17 @@ func chainCounter(i *int, ch Chain) Chain { } } +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *scheduler.Event, _ error) (context.Context, *scheduler.Event, error) { + panic(x) + } +} + func TestChainIdentity(t *testing.T) { var i int counterRule := counter(&i) - _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, chainIdentity) + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, ChainIdentity) if e != nil { t.Error("expected nil event instead of", e) } @@ -83,7 +89,7 @@ func TestRules(t *testing.T) { tracer(counter(&i), "counter2", t), nil, ) - _, e, err = rule(ctx, tc.e, tc.err, chainIdentity) + _, e, err = rule(ctx, tc.e, tc.err, ChainIdentity) ) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) @@ -96,7 +102,7 @@ func TestRules(t *testing.T) { } // empty Rules should not change event, err - _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, chainIdentity) + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, ChainIdentity) if e != tc.e { t.Errorf("expected prototype event %q instead of %q", tc.e, e) } @@ -175,7 +181,7 @@ func TestUnlessDone(t *testing.T) { r2 = r1.UnlessDone() ) for k, r := range []Rule{r1, r2} { - _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -206,7 +212,7 @@ func TestAndThen(t *testing.T) { a = errors.New("a") ) for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -238,7 +244,7 @@ func TestOnFailure(t *testing.T) { {r1, a}, {r2, nil}, } { - _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, chainIdentity)) + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -266,7 +272,7 @@ func TestDropOnError(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -280,7 +286,7 @@ func TestDropOnError(t *testing.T) { t.Errorf("expected chain count of 1 instead of %d", j) } } - _, e, err := r2(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -303,7 +309,7 @@ func TestDropOnSuccess(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for _, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -318,7 +324,7 @@ func TestDropOnSuccess(t *testing.T) { } } a := errors.New("a") - _, e, err := r2(ctx, p, a, chainCounter(&j, chainIdentity)) + _, e, err := r2(ctx, p, a, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -333,7 +339,7 @@ func TestDropOnSuccess(t *testing.T) { } r3 := Rules{DropOnSuccess(), r1}.Eval - _, e, err = r3(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err = r3(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -359,7 +365,7 @@ func TestThenDrop(t *testing.T) { ) // r1 and r2 should execute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -388,7 +394,7 @@ func TestDrop(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, anErr, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -416,7 +422,7 @@ func TestIf(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -443,7 +449,7 @@ func TestUnless(t *testing.T) { // r1 should execute the counter rule // r2 should NOT exexute the counter rule for k, r := range []Rule{r1, r2} { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -469,7 +475,7 @@ func TestOnce(t *testing.T) { ) for k, r := range []Rule{r1, r2} { for x := 0; x < 5; x++ { - _, e, err := r(ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("expected event %q instead of %q", p, e) } @@ -498,49 +504,55 @@ func TestRateLimit(t *testing.T) { ch2 = make(chan struct{}) // non-nil, blocking ch4 = make(chan struct{}) // non-nil, closed ctx = context.Background() + p = prototype() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() ) close(ch4) for ti, tc := range []struct { ctx context.Context ch <-chan struct{} over Overflow + otherwise Rule wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit wantsRuleCount []int wantsChainCount []int }{ - {ctx, ch1, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, ch2, OverflowSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, - {ctx, o(), OverflowSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, - {ctx, ch4, OverflowSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - {ctx, ch1, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, ch2, OverflowDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, - {ctx, o(), OverflowDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, - {ctx, ch4, OverflowDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, - - // TODO(jdef): test OverflowBackpressure (blocking) - {ctx, ch4, OverflowBackpressure, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( i, j int - p = prototype() - r1 = counter(&i).RateLimit(tc.ch, tc.over).Eval - r2 = Rule(nil).RateLimit(tc.ch, tc.over).Eval // a nil rule still invokes the chain + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain ) for k, r := range []Rule{r1, r2} { // execute each rule twice for x := 0; x < 2; x++ { - _, e, err := r(tc.ctx, p, nil, chainCounter(&j, chainIdentity)) + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) if e != p { t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) } @@ -558,4 +570,26 @@ func TestRateLimit(t *testing.T) { } } } + // test blocking capability via rateLimit + blocked := false + r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, err := r(ctx, p, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } } diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go index ce9ae252..3b682cbf 100644 --- a/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go @@ -41,7 +41,7 @@ func (r Rule) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { if r == nil { return nil } - _, _, err = r(ctx, e, nil, chainIdentity) + _, _, err = r(ctx, e, nil, ChainIdentity) return } From ce52cb0717be2f19abe7aff0fa84558c0190fd4a Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 11 Jun 2017 22:07:07 +0000 Subject: [PATCH 63/67] controller: abort Run control loop if re-registration chan is closed --- api/v1/lib/extras/scheduler/controller/controller.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index ee296d83..b20efa8a 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -58,8 +58,9 @@ func WithSubscriptionTerminated(handler func(error)) Option { } // WithRegistrationTokens limits the rate at which a framework (re)registers with Mesos. -// The chan should either be non-blocking, or should yield a struct{} in order to allow the -// framework registration process to continue. May be nil. +// A non-nil chan should yield a struct{} in order to allow the framework registration process to continue. +// When nil, there is no backoff delay between re-subscription attempts. +// A closed chan disables re-registration and terminates the Run control loop. func WithRegistrationTokens(registrationTokens <-chan struct{}) Option { return func(c *Config) Option { old := c.registrationTokens @@ -105,8 +106,11 @@ func Run(ctx context.Context, framework *mesos.FrameworkInfo, caller calls.Calle } if config.registrationTokens != nil { select { - case <-config.registrationTokens: - // continue + case _, ok := <-config.registrationTokens: + if !ok { + // re-registration canceled, exit Run loop + return + } case <-ctx.Done(): return ctx.Err() } From 3cb466f870b3ab7db7eaaeeca1b40abf8fd039ca Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 11 Jun 2017 22:08:10 +0000 Subject: [PATCH 64/67] backoff: close burst notifier chan upon notifier termination --- api/v1/lib/backoff/backoff.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/v1/lib/backoff/backoff.go b/api/v1/lib/backoff/backoff.go index 347b6ce5..3993829a 100644 --- a/api/v1/lib/backoff/backoff.go +++ b/api/v1/lib/backoff/backoff.go @@ -27,6 +27,7 @@ func BurstNotifier(burst int, minWait, maxWait time.Duration, until <-chan struc // listen for tokens emitted by child buckets and forward them to the tokens chan tokens := make(chan struct{}) go func() { + defer close(tokens) for { i, _, _ := reflect.Select(cases) if i == burst { From 15c85507d763f02f71732d130665efc66e83bb3f Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 11 Jun 2017 22:42:07 +0000 Subject: [PATCH 65/67] rules: RateLimit should not block if/when context is Done --- .../executor/callrules/callrules_generated.go | 94 +++++++++++------- .../callrules/callrules_generated_test.go | 2 +- .../eventrules/eventrules_generated.go | 94 +++++++++++------- .../eventrules/eventrules_generated_test.go | 2 +- api/v1/lib/extras/gen/rules.go | 96 ++++++++++++------- .../callrules/callrules_generated.go | 94 +++++++++++------- .../callrules/callrules_generated_test.go | 2 +- .../eventrules/eventrules_generated.go | 94 +++++++++++------- .../eventrules/eventrules_generated_test.go | 2 +- 9 files changed, 300 insertions(+), 180 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index dfd11fe7..8677b417 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -217,33 +217,41 @@ const ( OverflowOtherwise ) -// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { if r != nil && p == nil && over == OverflowWait { panic("deadlock detected: reads from token chan will permanently block rule processing") } - return rateLimit(r, acquireFunc(p), over, otherwise) + return limit(r, acquireChan(p), over, otherwise) } -// acquireFunc wraps a signal chan with a func that can be used with rateLimit. -// should only be called by RateLimit (because it implements deadlock detection). -func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if tokenCh == nil { - // always false/blocked: acquire never succeeds - return func(block bool) bool { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { if block { - panic("deadlock detected: block should never be true when the token chan is nil") + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } } return false } } - return func(block bool) bool { + return func(ctx context.Context, block bool) bool { if block { - <-tokenCh - return true + select { + case <-tokenCh: + return true + case <-ctx.Done(): + return false + } } select { case <-tokenCh: @@ -254,30 +262,37 @@ func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { } } -// rateLimit is more easily unit tested than RateLimit. -func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } if acquire == nil { panic("acquire func is not allowed to be nil") } - return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - if !acquire(false) { - // non-blocking acquire failed, check overflow policy - switch over { - case OverflowWait: - _ = acquire(true) // block until there's a signal - case OverflowOtherwise: + switch over { + case OverflowWait: + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + _ = acquire(ctx, true) // block until there's a signal + return r(ctx, e, z, err, ch) + } + case OverflowOtherwise: + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if !acquire(ctx, false) { return otherwise.Eval(ctx, e, z, err, ch) - default: - panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return r(ctx, e, z, err, ch) } - return r(ctx, e, z, err, ch) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } +/* TODO(jdef) not sure that this is very useful, leaving out for now... + // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. @@ -285,29 +300,38 @@ func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { var ( i = 1 // begin with the first event seen m sync.Mutex - forward = func() bool { + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: m.Lock() i-- - if i == 0 { + if i <= 0 { i = nthTime - m.Unlock() - return true + result = true } m.Unlock() - return false - } - ) - return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - if forward() { - return r(ctx, e, z, err, ch) } - return otherwise.Eval(ctx, e, z, err, ch) + return } } +*/ + // Drop aborts the Chain and returns the (context.Context, *executor.Call, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index 0b5a58f2..46a4319b 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -642,7 +642,7 @@ func TestRateLimit(t *testing.T) { } // test blocking capability via rateLimit blocked := false - r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) if e != p { t.Errorf("expected event %q instead of %q", p, e) diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 35a1e3c5..517deef8 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -216,33 +216,41 @@ const ( OverflowOtherwise ) -// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { if r != nil && p == nil && over == OverflowWait { panic("deadlock detected: reads from token chan will permanently block rule processing") } - return rateLimit(r, acquireFunc(p), over, otherwise) + return limit(r, acquireChan(p), over, otherwise) } -// acquireFunc wraps a signal chan with a func that can be used with rateLimit. -// should only be called by RateLimit (because it implements deadlock detection). -func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if tokenCh == nil { - // always false/blocked: acquire never succeeds - return func(block bool) bool { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { if block { - panic("deadlock detected: block should never be true when the token chan is nil") + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } } return false } } - return func(block bool) bool { + return func(ctx context.Context, block bool) bool { if block { - <-tokenCh - return true + select { + case <-tokenCh: + return true + case <-ctx.Done(): + return false + } } select { case <-tokenCh: @@ -253,30 +261,37 @@ func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { } } -// rateLimit is more easily unit tested than RateLimit. -func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } if acquire == nil { panic("acquire func is not allowed to be nil") } - return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - if !acquire(false) { - // non-blocking acquire failed, check overflow policy - switch over { - case OverflowWait: - _ = acquire(true) // block until there's a signal - case OverflowOtherwise: + switch over { + case OverflowWait: + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + _ = acquire(ctx, true) // block until there's a signal + return r(ctx, e, err, ch) + } + case OverflowOtherwise: + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if !acquire(ctx, false) { return otherwise.Eval(ctx, e, err, ch) - default: - panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return r(ctx, e, err, ch) } - return r(ctx, e, err, ch) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } +/* TODO(jdef) not sure that this is very useful, leaving out for now... + // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. @@ -284,29 +299,38 @@ func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { var ( i = 1 // begin with the first event seen m sync.Mutex - forward = func() bool { + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: m.Lock() i-- - if i == 0 { + if i <= 0 { i = nthTime - m.Unlock() - return true + result = true } m.Unlock() - return false - } - ) - return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - if forward() { - return r(ctx, e, err, ch) } - return otherwise.Eval(ctx, e, err, ch) + return } } +*/ + // Drop aborts the Chain and returns the (context.Context, *executor.Event, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index deb951fe..7185e51c 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -572,7 +572,7 @@ func TestRateLimit(t *testing.T) { } // test blocking capability via rateLimit blocked := false - r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) _, e, err := r(ctx, p, nil, ChainIdentity) if e != p { t.Errorf("expected event %q instead of %q", p, e) diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 1a0735b0..bf4bc567 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -233,33 +233,41 @@ const ( OverflowOtherwise ) -// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { if r != nil && p == nil && over == OverflowWait { panic("deadlock detected: reads from token chan will permanently block rule processing") } - return rateLimit(r, acquireFunc(p), over, otherwise) + return limit(r, acquireChan(p), over, otherwise) } -// acquireFunc wraps a signal chan with a func that can be used with rateLimit. -// should only be called by RateLimit (because it implements deadlock detection). -func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if tokenCh == nil { - // always false/blocked: acquire never succeeds - return func(block bool) bool { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { if block { - panic("deadlock detected: block should never be true when the token chan is nil") + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } } return false } } - return func(block bool) bool { + return func(ctx context.Context, block bool) bool { if block { - <-tokenCh - return true + select { + case <-tokenCh: + return true + case <-ctx.Done(): + return false + } } select { case <-tokenCh: @@ -270,30 +278,37 @@ func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { } } -// rateLimit is more easily unit tested than RateLimit. -func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } if acquire == nil { panic("acquire func is not allowed to be nil") } - return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - if !acquire(false) { - // non-blocking acquire failed, check overflow policy - switch over { - case OverflowWait: - _ = acquire(true) // block until there's a signal - case OverflowOtherwise: + switch over { + case OverflowWait: + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + _ = acquire(ctx, true) // block until there's a signal + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + case OverflowOtherwise: + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if !acquire(ctx, false) { return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) - default: - panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } +/* TODO(jdef) not sure that this is very useful, leaving out for now... + // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. @@ -301,29 +316,38 @@ func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { var ( i = 1 // begin with the first event seen m sync.Mutex - forward = func() bool { + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: m.Lock() i-- - if i == 0 { + if i <= 0 { i = nthTime - m.Unlock() - return true + result = true } m.Unlock() - return false - } - ) - return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - if forward() { - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) } - return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) + return } } +*/ + // Drop aborts the Chain and returns the (context.Context, {{.Type "E"}}, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() @@ -1088,7 +1112,7 @@ func TestRateLimit(t *testing.T) { } // test blocking capability via rateLimit blocked := false - r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, ChainIdentity) if e != p { t.Errorf("expected event %q instead of %q", p, e) diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 23a8978d..fbd04d29 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -217,33 +217,41 @@ const ( OverflowOtherwise ) -// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { if r != nil && p == nil && over == OverflowWait { panic("deadlock detected: reads from token chan will permanently block rule processing") } - return rateLimit(r, acquireFunc(p), over, otherwise) + return limit(r, acquireChan(p), over, otherwise) } -// acquireFunc wraps a signal chan with a func that can be used with rateLimit. -// should only be called by RateLimit (because it implements deadlock detection). -func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if tokenCh == nil { - // always false/blocked: acquire never succeeds - return func(block bool) bool { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { if block { - panic("deadlock detected: block should never be true when the token chan is nil") + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } } return false } } - return func(block bool) bool { + return func(ctx context.Context, block bool) bool { if block { - <-tokenCh - return true + select { + case <-tokenCh: + return true + case <-ctx.Done(): + return false + } } select { case <-tokenCh: @@ -254,30 +262,37 @@ func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { } } -// rateLimit is more easily unit tested than RateLimit. -func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } if acquire == nil { panic("acquire func is not allowed to be nil") } - return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - if !acquire(false) { - // non-blocking acquire failed, check overflow policy - switch over { - case OverflowWait: - _ = acquire(true) // block until there's a signal - case OverflowOtherwise: + switch over { + case OverflowWait: + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + _ = acquire(ctx, true) // block until there's a signal + return r(ctx, e, z, err, ch) + } + case OverflowOtherwise: + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if !acquire(ctx, false) { return otherwise.Eval(ctx, e, z, err, ch) - default: - panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return r(ctx, e, z, err, ch) } - return r(ctx, e, z, err, ch) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } +/* TODO(jdef) not sure that this is very useful, leaving out for now... + // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. @@ -285,29 +300,38 @@ func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { var ( i = 1 // begin with the first event seen m sync.Mutex - forward = func() bool { + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: m.Lock() i-- - if i == 0 { + if i <= 0 { i = nthTime - m.Unlock() - return true + result = true } m.Unlock() - return false - } - ) - return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - if forward() { - return r(ctx, e, z, err, ch) } - return otherwise.Eval(ctx, e, z, err, ch) + return } } +*/ + // Drop aborts the Chain and returns the (context.Context, *scheduler.Call, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index c3a5df15..d9281cbf 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -642,7 +642,7 @@ func TestRateLimit(t *testing.T) { } // test blocking capability via rateLimit blocked := false - r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) if e != p { t.Errorf("expected event %q instead of %q", p, e) diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 6a4d7194..3fb4771a 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -216,33 +216,41 @@ const ( OverflowOtherwise ) -// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (may be closed), otherwise proceeds +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. -// Returns nil (noop) if the receiver is nil, otherwise a nil chan will trigger an overflow. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { if r != nil && p == nil && over == OverflowWait { panic("deadlock detected: reads from token chan will permanently block rule processing") } - return rateLimit(r, acquireFunc(p), over, otherwise) + return limit(r, acquireChan(p), over, otherwise) } -// acquireFunc wraps a signal chan with a func that can be used with rateLimit. -// should only be called by RateLimit (because it implements deadlock detection). -func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if tokenCh == nil { - // always false/blocked: acquire never succeeds - return func(block bool) bool { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { if block { - panic("deadlock detected: block should never be true when the token chan is nil") + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } } return false } } - return func(block bool) bool { + return func(ctx context.Context, block bool) bool { if block { - <-tokenCh - return true + select { + case <-tokenCh: + return true + case <-ctx.Done(): + return false + } } select { case <-tokenCh: @@ -253,30 +261,37 @@ func acquireFunc(tokenCh <-chan struct{}) func(bool) bool { } } -// rateLimit is more easily unit tested than RateLimit. -func rateLimit(r Rule, acquire func(block bool) bool, over Overflow, otherwise Rule) Rule { +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { if r == nil { return nil } if acquire == nil { panic("acquire func is not allowed to be nil") } - return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - if !acquire(false) { - // non-blocking acquire failed, check overflow policy - switch over { - case OverflowWait: - _ = acquire(true) // block until there's a signal - case OverflowOtherwise: + switch over { + case OverflowWait: + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + _ = acquire(ctx, true) // block until there's a signal + return r(ctx, e, err, ch) + } + case OverflowOtherwise: + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if !acquire(ctx, false) { return otherwise.Eval(ctx, e, err, ch) - default: - panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return r(ctx, e, err, ch) } - return r(ctx, e, err, ch) + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } } +/* TODO(jdef) not sure that this is very useful, leaving out for now... + // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. @@ -284,29 +299,38 @@ func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { var ( i = 1 // begin with the first event seen m sync.Mutex - forward = func() bool { + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: m.Lock() i-- - if i == 0 { + if i <= 0 { i = nthTime - m.Unlock() - return true + result = true } m.Unlock() - return false - } - ) - return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - if forward() { - return r(ctx, e, err, ch) } - return otherwise.Eval(ctx, e, err, ch) + return } } +*/ + // Drop aborts the Chain and returns the (context.Context, *scheduler.Event, error) tuple as-is. func Drop() Rule { return Rule(nil).ThenDrop() diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 9c4e4165..50da6e1c 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -572,7 +572,7 @@ func TestRateLimit(t *testing.T) { } // test blocking capability via rateLimit blocked := false - r := rateLimit(Rule(nil).Eval, func(b bool) bool { blocked = b; return false }, OverflowWait, nil) + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) _, e, err := r(ctx, p, nil, ChainIdentity) if e != p { t.Errorf("expected event %q instead of %q", p, e) From d7478ca211a7f4914e1d4423bdea746a5a95b37e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 11 Jun 2017 23:21:08 +0000 Subject: [PATCH 66/67] rules: unit test and fix for cancelled context r/ RateLimit --- .../executor/callrules/callrules_generated.go | 34 ++++----- .../callrules/callrules_generated_test.go | 35 +++++++++- .../eventrules/eventrules_generated.go | 34 ++++----- .../eventrules/eventrules_generated_test.go | 35 +++++++++- api/v1/lib/extras/gen/rules.go | 69 ++++++++++++++----- .../callrules/callrules_generated.go | 34 ++++----- .../callrules/callrules_generated_test.go | 35 +++++++++- .../eventrules/eventrules_generated.go | 34 ++++----- .../eventrules/eventrules_generated_test.go | 35 +++++++++- 9 files changed, 255 insertions(+), 90 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 8677b417..5b419dc4 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -221,10 +221,8 @@ const ( // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. // Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { - if r != nil && p == nil && over == OverflowWait { - panic("deadlock detected: reads from token chan will permanently block rule processing") - } return limit(r, acquireChan(p), over, otherwise) } @@ -248,10 +246,15 @@ func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if block { select { case <-tokenCh: - return true + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } case <-ctx.Done(): - return false } + return false } select { case <-tokenCh: @@ -273,22 +276,20 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl if acquire == nil { panic("acquire func is not allowed to be nil") } + blocking := false switch over { - case OverflowWait: - return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - _ = acquire(ctx, true) // block until there's a signal - return r(ctx, e, z, err, ch) - } case OverflowOtherwise: - return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { - if !acquire(ctx, false) { - return otherwise.Eval(ctx, e, z, err, ch) - } - return r(ctx, e, z, err, ch) - } + case OverflowWait: + blocking = true default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, z, err, ch) + } + return r(ctx, e, z, err, ch) + } } /* TODO(jdef) not sure that this is very useful, leaving out for now... @@ -296,6 +297,7 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index 46a4319b..5092caf8 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -568,9 +568,15 @@ func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil, blocking ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() ch4 = make(chan struct{}) // non-nil, closed - ctx = context.Background() p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() errOverflow = errors.New("overflow") otherwiseSkip = Rule(nil) @@ -581,6 +587,7 @@ func TestRateLimit(t *testing.T) { var zp = &mesos.ResponseWrapper{} close(ch4) for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 ctx context.Context ch <-chan struct{} over Overflow @@ -594,21 +601,45 @@ func TestRateLimit(t *testing.T) { {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( @@ -660,7 +691,7 @@ func TestRateLimit(t *testing.T) { didPanic := false func() { defer func() { didPanic = recover() != nil }() - Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, zp, nil, ChainIdentity) }() if !didPanic { t.Error("expected panic because we configured a rule to deadlock") diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 517deef8..0c3f34ff 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -220,10 +220,8 @@ const ( // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. // Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { - if r != nil && p == nil && over == OverflowWait { - panic("deadlock detected: reads from token chan will permanently block rule processing") - } return limit(r, acquireChan(p), over, otherwise) } @@ -247,10 +245,15 @@ func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if block { select { case <-tokenCh: - return true + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } case <-ctx.Done(): - return false } + return false } select { case <-tokenCh: @@ -272,22 +275,20 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl if acquire == nil { panic("acquire func is not allowed to be nil") } + blocking := false switch over { - case OverflowWait: - return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - _ = acquire(ctx, true) // block until there's a signal - return r(ctx, e, err, ch) - } case OverflowOtherwise: - return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { - if !acquire(ctx, false) { - return otherwise.Eval(ctx, e, err, ch) - } - return r(ctx, e, err, ch) - } + case OverflowWait: + blocking = true default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, err, ch) + } + return r(ctx, e, err, ch) + } } /* TODO(jdef) not sure that this is very useful, leaving out for now... @@ -295,6 +296,7 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 7185e51c..6b559aaa 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -502,9 +502,15 @@ func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil, blocking ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() ch4 = make(chan struct{}) // non-nil, closed - ctx = context.Background() p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() errOverflow = errors.New("overflow") otherwiseSkip = Rule(nil) @@ -514,6 +520,7 @@ func TestRateLimit(t *testing.T) { ) close(ch4) for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 ctx context.Context ch <-chan struct{} over Overflow @@ -527,21 +534,45 @@ func TestRateLimit(t *testing.T) { {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( @@ -587,7 +618,7 @@ func TestRateLimit(t *testing.T) { didPanic := false func() { defer func() { didPanic = recover() != nil }() - Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, nil, ChainIdentity) }() if !didPanic { t.Error("expected panic because we configured a rule to deadlock") diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index bf4bc567..3d2f12a4 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -237,10 +237,8 @@ const ( // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. // Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { - if r != nil && p == nil && over == OverflowWait { - panic("deadlock detected: reads from token chan will permanently block rule processing") - } return limit(r, acquireChan(p), over, otherwise) } @@ -264,10 +262,15 @@ func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if block { select { case <-tokenCh: - return true + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } case <-ctx.Done(): - return false } + return false } select { case <-tokenCh: @@ -289,22 +292,20 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl if acquire == nil { panic("acquire func is not allowed to be nil") } + blocking := false switch over { - case OverflowWait: - return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - _ = acquire(ctx, true) // block until there's a signal - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) - } case OverflowOtherwise: - return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { - if !acquire(ctx, false) { - return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) - } - return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) - } + case OverflowWait: + blocking = true default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } } /* TODO(jdef) not sure that this is very useful, leaving out for now... @@ -312,6 +313,7 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r @@ -1034,9 +1036,15 @@ func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil, blocking ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() ch4 = make(chan struct{}) // non-nil, closed - ctx = context.Background() p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() errOverflow = errors.New("overflow") otherwiseSkip = Rule(nil) @@ -1049,6 +1057,7 @@ func TestRateLimit(t *testing.T) { {{end -}} close(ch4) for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 ctx context.Context ch <-chan struct{} over Overflow @@ -1062,21 +1071,45 @@ func TestRateLimit(t *testing.T) { {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( @@ -1132,7 +1165,7 @@ func TestRateLimit(t *testing.T) { didPanic := false func() { defer func() { didPanic = recover() != nil }() - Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, {{.Ref "Z" "zp," -}} nil, ChainIdentity) }() if !didPanic { t.Error("expected panic because we configured a rule to deadlock") diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index fbd04d29..5a2cdc02 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -221,10 +221,8 @@ const ( // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. // Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { - if r != nil && p == nil && over == OverflowWait { - panic("deadlock detected: reads from token chan will permanently block rule processing") - } return limit(r, acquireChan(p), over, otherwise) } @@ -248,10 +246,15 @@ func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if block { select { case <-tokenCh: - return true + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } case <-ctx.Done(): - return false } + return false } select { case <-tokenCh: @@ -273,22 +276,20 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl if acquire == nil { panic("acquire func is not allowed to be nil") } + blocking := false switch over { - case OverflowWait: - return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - _ = acquire(ctx, true) // block until there's a signal - return r(ctx, e, z, err, ch) - } case OverflowOtherwise: - return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { - if !acquire(ctx, false) { - return otherwise.Eval(ctx, e, z, err, ch) - } - return r(ctx, e, z, err, ch) - } + case OverflowWait: + blocking = true default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, z, err, ch) + } + return r(ctx, e, z, err, ch) + } } /* TODO(jdef) not sure that this is very useful, leaving out for now... @@ -296,6 +297,7 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index d9281cbf..84608ba5 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -568,9 +568,15 @@ func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil, blocking ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() ch4 = make(chan struct{}) // non-nil, closed - ctx = context.Background() p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() errOverflow = errors.New("overflow") otherwiseSkip = Rule(nil) @@ -581,6 +587,7 @@ func TestRateLimit(t *testing.T) { var zp = &mesos.ResponseWrapper{} close(ch4) for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 ctx context.Context ch <-chan struct{} over Overflow @@ -594,21 +601,45 @@ func TestRateLimit(t *testing.T) { {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( @@ -660,7 +691,7 @@ func TestRateLimit(t *testing.T) { didPanic := false func() { defer func() { didPanic = recover() != nil }() - Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, zp, nil, ChainIdentity) }() if !didPanic { t.Error("expected panic because we configured a rule to deadlock") diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 3fb4771a..606bb54d 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -220,10 +220,8 @@ const ( // according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. // Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. // Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { - if r != nil && p == nil && over == OverflowWait { - panic("deadlock detected: reads from token chan will permanently block rule processing") - } return limit(r, acquireChan(p), over, otherwise) } @@ -247,10 +245,15 @@ func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { if block { select { case <-tokenCh: - return true + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } case <-ctx.Done(): - return false } + return false } select { case <-tokenCh: @@ -272,22 +275,20 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl if acquire == nil { panic("acquire func is not allowed to be nil") } + blocking := false switch over { - case OverflowWait: - return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - _ = acquire(ctx, true) // block until there's a signal - return r(ctx, e, err, ch) - } case OverflowOtherwise: - return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { - if !acquire(ctx, false) { - return otherwise.Eval(ctx, e, err, ch) - } - return r(ctx, e, err, ch) - } + case OverflowWait: + blocking = true default: panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) } + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, err, ch) + } + return r(ctx, e, err, ch) + } } /* TODO(jdef) not sure that this is very useful, leaving out for now... @@ -295,6 +296,7 @@ func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overfl // EveryN invokes the receiving rule beginning with the first event seen and then every n'th // time after that. If nthTime is less then 2 then the receiver is returned, undecorated. // The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { if nthTime < 2 || r == nil { return r diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 50da6e1c..826a9a6a 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -502,9 +502,15 @@ func TestRateLimit(t *testing.T) { var ( ch1 <-chan struct{} // always nil, blocking ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() ch4 = make(chan struct{}) // non-nil, closed - ctx = context.Background() p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() errOverflow = errors.New("overflow") otherwiseSkip = Rule(nil) @@ -514,6 +520,7 @@ func TestRateLimit(t *testing.T) { ) close(ch4) for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 ctx context.Context ch <-chan struct{} over Overflow @@ -527,21 +534,45 @@ func TestRateLimit(t *testing.T) { {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, } { var ( @@ -587,7 +618,7 @@ func TestRateLimit(t *testing.T) { didPanic := false func() { defer func() { didPanic = recover() != nil }() - Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil) + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, nil, ChainIdentity) }() if !didPanic { t.Error("expected panic because we configured a rule to deadlock") From 9a4f274f0d4229a8237903aa685d11a1d31441ec Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 12 Jun 2017 13:12:35 +0000 Subject: [PATCH 67/67] rules: unit tests for ErrorList.Error() --- .../executor/callrules/callrules_generated.go | 15 +++++--- .../callrules/callrules_generated_test.go | 20 +++++++++++ .../eventrules/eventrules_generated.go | 15 +++++--- .../eventrules/eventrules_generated_test.go | 20 +++++++++++ api/v1/lib/extras/gen/rules.go | 35 ++++++++++++++++--- .../callrules/callrules_generated.go | 15 +++++--- .../callrules/callrules_generated_test.go | 20 +++++++++++ .../eventrules/eventrules_generated.go | 15 +++++--- .../eventrules/eventrules_generated_test.go | 20 +++++++++++ 9 files changed, 150 insertions(+), 25 deletions(-) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go index 5b419dc4..aa4b0ed3 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -78,11 +78,15 @@ func (rs Rules) Chain() Chain { // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. func Concat(rs ...Rule) Rule { return Rules(rs).Eval } +const ( + MsgNoErrors = "no errors" +) + // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { switch len(es) { case 0: - return "no errors" + return MsgNoErrors case 1: return es[0].Error() default: @@ -113,13 +117,14 @@ func Error2(a, b error) error { // Err reduces an empty or singleton error list func (es ErrorList) Err() error { - if len(es) == 0 { + switch len(es) { + case 0: return nil - } - if len(es) == 1 { + case 1: return es[0] + default: + return es } - return es } // IsErrorList returns true if err is a non-nil error list diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go index 5092caf8..39ef4e4b 100644 --- a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -129,6 +129,26 @@ func TestRules(t *testing.T) { } } +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go index 0c3f34ff..327d85fe 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -77,11 +77,15 @@ func (rs Rules) Chain() Chain { // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. func Concat(rs ...Rule) Rule { return Rules(rs).Eval } +const ( + MsgNoErrors = "no errors" +) + // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { switch len(es) { case 0: - return "no errors" + return MsgNoErrors case 1: return es[0].Error() default: @@ -112,13 +116,14 @@ func Error2(a, b error) error { // Err reduces an empty or singleton error list func (es ErrorList) Err() error { - if len(es) == 0 { + switch len(es) { + case 0: return nil - } - if len(es) == 1 { + case 1: return es[0] + default: + return es } - return es } // IsErrorList returns true if err is a non-nil error list diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go index 6b559aaa..3d874d9b 100644 --- a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -112,6 +112,26 @@ func TestRules(t *testing.T) { } } +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go index 3d2f12a4..942f62a8 100644 --- a/api/v1/lib/extras/gen/rules.go +++ b/api/v1/lib/extras/gen/rules.go @@ -94,11 +94,15 @@ func (rs Rules) Chain() Chain { // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. func Concat(rs ...Rule) Rule { return Rules(rs).Eval } +const ( + MsgNoErrors = "no errors" +) + // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { switch len(es) { case 0: - return "no errors" + return MsgNoErrors case 1: return es[0].Error() default: @@ -129,13 +133,14 @@ func Error2(a, b error) error { // Err reduces an empty or singleton error list func (es ErrorList) Err() error { - if len(es) == 0 { + switch len(es) { + case 0: return nil - } - if len(es) == 1 { + case 1: return es[0] + default: + return es } - return es } // IsErrorList returns true if err is a non-nil error list @@ -551,6 +556,26 @@ func TestRules(t *testing.T) { } } +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go index 5a2cdc02..b50d5070 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -78,11 +78,15 @@ func (rs Rules) Chain() Chain { // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. func Concat(rs ...Rule) Rule { return Rules(rs).Eval } +const ( + MsgNoErrors = "no errors" +) + // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { switch len(es) { case 0: - return "no errors" + return MsgNoErrors case 1: return es[0].Error() default: @@ -113,13 +117,14 @@ func Error2(a, b error) error { // Err reduces an empty or singleton error list func (es ErrorList) Err() error { - if len(es) == 0 { + switch len(es) { + case 0: return nil - } - if len(es) == 1 { + case 1: return es[0] + default: + return es } - return es } // IsErrorList returns true if err is a non-nil error list diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go index 84608ba5..2bcf3450 100644 --- a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -129,6 +129,26 @@ func TestRules(t *testing.T) { } } +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a") diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go index 606bb54d..08f8c794 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -77,11 +77,15 @@ func (rs Rules) Chain() Chain { // It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. func Concat(rs ...Rule) Rule { return Rules(rs).Eval } +const ( + MsgNoErrors = "no errors" +) + // Error implements error; returns the message of the first error in the list. func (es ErrorList) Error() string { switch len(es) { case 0: - return "no errors" + return MsgNoErrors case 1: return es[0].Error() default: @@ -112,13 +116,14 @@ func Error2(a, b error) error { // Err reduces an empty or singleton error list func (es ErrorList) Err() error { - if len(es) == 0 { + switch len(es) { + case 0: return nil - } - if len(es) == 1 { + case 1: return es[0] + default: + return es } - return es } // IsErrorList returns true if err is a non-nil error list diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go index 826a9a6a..8b17f3ff 100644 --- a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -112,6 +112,26 @@ func TestRules(t *testing.T) { } } +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + func TestError2(t *testing.T) { var ( a = errors.New("a")