diff --git a/.travis.yml b/.travis.yml index b2876e1e..44fc65e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ language: go git: submodules: false go: - - 1.6.x - 1.7.x - 1.8 before_install: @@ -17,4 +16,4 @@ before_install: install: - make install script: - - rm -rf $HOME/gopath/pkg && make coverage && $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=_output/coverage.out + - rm -rf $HOME/gopath/pkg && make coverage && $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=_output/coverage.out -ignore=$((tr '\n' , | sed -e 's/$,//g') < <(find api/v1/cmd -type f -name '*.go'|grep -v -e /vendor/ ; ls api/v1/lib{,/scheduler,/executor}/*.pb{,_ffjson}.go ; find api/v0 -type d)) diff --git a/Makefile b/Makefile index 66629486..686b1088 100644 --- a/Makefile +++ b/Makefile @@ -33,15 +33,18 @@ test-verbose: TEST_FLAGS += -v test-verbose: test .PHONY: coverage $(COVERAGE_TARGETS) -coverage: TEST_FLAGS = -v -cover -race +coverage: REPORT=_output/coverage.out +coverage: COVER_PACKAGE = $(shell go list ${API_PKG}/...|egrep -v 'vendor|cmd'|tr '\n' ','|sed -e 's/,$$//') +coverage: TEST_FLAGS = -v -cover -coverpkg=$(COVER_PACKAGE) coverage: $(COVERAGE_TARGETS) - cat _output/*.cover | sed -e '2,$$ s/^mode:.*$$//' -e '/^$$/d' >_output/coverage.out -$(COVERAGE_TARGETS): - mkdir -p _output && go test ./$(@:%.cover=%) $(TEST_FLAGS) -coverprofile=_output/$(subst /,___,$@) + echo "mode: set" >$(REPORT) && cat _output/*.cover | grep -v mode: | sort -r | \ + awk '{if($$1 != last) {print $$0;last=$$1}}' >> $(REPORT) +$(COVERAGE_TARGETS): %.cover : + mkdir -p _output && go test ./$* $(TEST_FLAGS) -coverprofile=_output/$(subst /,___,$@) .PHONY: vet vet: - go $@ $(PACKAGES) + go $@ $(PACKAGES) $(BINARIES) .PHONY: codecs codecs: protobufs ffjson @@ -77,6 +80,11 @@ sync: (cd ${API_VENDOR}; govendor sync) (cd ${CMD_VENDOR}; govendor sync) +.PHONY: generate +generate: GENERATE_PACKAGES = ./api/v1/lib/extras/executor/eventrules ./api/v1/lib/extras/executor/callrules ./api/v1/lib/extras/scheduler/eventrules ./api/v1/lib/extras/scheduler/callrules ./api/v1/lib/executor/events ./api/v1/lib/executor/calls ./api/v1/lib/scheduler/events ./api/v1/lib/scheduler/calls +generate: + go generate -x ${GENERATE_PACKAGES} + GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) UID ?= $(shell id -u $$USER) diff --git a/README.md b/README.md index dc1a20b4..e0758c55 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Go bindings for Apache Mesos -Very early version of a pure Go language bindings for Apache Mesos. +Pure Go language bindings for Apache Mesos, under development. As with other pure implementations, mesos-go uses the HTTP wire protocol to communicate directly with a running Mesos master and its slave instances. One of the objectives of this project is to provide an idiomatic Go API that makes it super easy to create Mesos frameworks using Go. @@ -12,6 +12,8 @@ One of the objectives of this project is to provide an idiomatic Go API that mak New projects should use the Mesos v1 API bindings, located in `api/v1`. Unless otherwise indicated, the remainder of this README describes the Mesos v1 API implementation. +Please **vendor** this library to avoid unpleasant surprises via `go get ...`. + The Mesos v0 API version of the bindings, located in `api/v0`, are more mature but will not see any major development besides critical compatibility and bug fixes. ### Compatibility @@ -24,7 +26,7 @@ The Mesos v0 API version of the bindings, located in `api/v0`, are more mature b - Modular design for easy readability/extensibility ### Pre-Requisites -- Go 1.6 or higher +- Go 1.7 or higher - A standard and working Go workspace setup - Apache Mesos 1.0 or newer diff --git a/api/v1/cmd/example-executor/main.go b/api/v1/cmd/example-executor/main.go index c4f5ad4b..6ed236f3 100644 --- a/api/v1/cmd/example-executor/main.go +++ b/api/v1/cmd/example-executor/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "io" "log" @@ -139,54 +140,58 @@ func unacknowledgedUpdates(state *internalState) executor.CallOpt { } func eventLoop(state *internalState, decoder encoding.Decoder, h events.Handler) (err error) { + ctx := context.TODO() for err == nil && !state.shouldQuit { // housekeeping sendFailedTasks(state) var e executor.Event if err = decoder.Decode(&e); err == nil { - err = h.HandleEvent(&e) + err = h.HandleEvent(ctx, &e) } } return err } func buildEventHandler(state *internalState) events.Handler { - return events.NewMux( - events.Handle(executor.Event_SUBSCRIBED, events.HandlerFunc(func(e *executor.Event) error { + return events.HandlerFuncs{ + executor.Event_SUBSCRIBED: func(_ context.Context, e *executor.Event) error { log.Println("SUBSCRIBED") state.framework = e.Subscribed.FrameworkInfo state.executor = e.Subscribed.ExecutorInfo state.agent = e.Subscribed.AgentInfo return nil - })), - events.Handle(executor.Event_LAUNCH, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_LAUNCH: func(_ context.Context, e *executor.Event) error { launch(state, e.Launch.Task) return nil - })), - events.Handle(executor.Event_KILL, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_KILL: func(_ context.Context, e *executor.Event) error { log.Println("warning: KILL not implemented") return nil - })), - events.Handle(executor.Event_ACKNOWLEDGED, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_ACKNOWLEDGED: func(_ context.Context, e *executor.Event) error { delete(state.unackedTasks, e.Acknowledged.TaskID) delete(state.unackedUpdates, string(e.Acknowledged.UUID)) return nil - })), - events.Handle(executor.Event_MESSAGE, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_MESSAGE: func(_ context.Context, e *executor.Event) error { log.Printf("MESSAGE: received %d bytes of message data", len(e.Message.Data)) return nil - })), - events.Handle(executor.Event_SHUTDOWN, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_SHUTDOWN: func(_ context.Context, e *executor.Event) error { log.Println("SHUTDOWN received") state.shouldQuit = true return nil - })), - events.Handle(executor.Event_ERROR, events.HandlerFunc(func(e *executor.Event) error { + }, + executor.Event_ERROR: func(_ context.Context, e *executor.Event) error { log.Println("ERROR received") return errMustAbort - })), - ) + }, + }.Otherwise(func(_ context.Context, e *executor.Event) error { + log.Fatal("unexpected event", e) + return nil + }) } func sendFailedTasks(state *internalState) { diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 1ecb3f28..d4b29926 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -1,8 +1,8 @@ package app import ( + "context" "errors" - "fmt" "io" "log" "strconv" @@ -11,7 +11,12 @@ import ( "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/resourcefilters" "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" @@ -30,10 +35,9 @@ func (err StateError) Error() string { return string(err) } func Run(cfg Config) error { log.Printf("scheduler running with configuration: %+v", cfg) - shutdown := make(chan struct{}) - defer close(shutdown) + ctx, cancel := context.WithCancel(context.Background()) - state, err := newInternalState(cfg) + state, err := newInternalState(cfg, cancel) if err != nil { return err } @@ -41,111 +45,81 @@ func Run(cfg Config) error { // TODO(jdef) how to track/handle timeout errors that occur for SUBSCRIBE calls? we should // probably tolerate X number of subsequent subscribe failures before bailing. we'll need // to track the lastCallAttempted along with subsequentSubscribeTimeouts. - err = controller.New().Run(buildControllerConfig(state, shutdown)) + + fidStore := store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) + + state.cli = callrules.Concat( + callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)), + logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}), + callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), + ).Caller(state.cli) + + err = controller.Run( + ctx, + buildFrameworkInfo(state.config), + state.cli, + controller.WithEventHandler(buildEventHandler(state, fidStore)), + controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)), + controller.WithRegistrationTokens( + backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, ctx.Done()), + ), + controller.WithSubscriptionTerminated(func(err error) { + if err != nil { + if err != io.EOF { + log.Println(err) + } + if _, ok := err.(StateError); ok { + state.shutdown() + } + return + } + log.Println("disconnected") + }), + ) if state.err != nil { err = state.err } return err } -func buildControllerConfig(state *internalState, shutdown <-chan struct{}) controller.Config { - var ( - frameworkIDStore = NewInMemoryIDStore() - controlContext = &controller.ContextAdapter{ - DoneFunc: state.isDone, - FrameworkIDFunc: func() string { return frameworkIDStore.Get() }, - ErrorFunc: func(err error) { - if err != nil { - if err != io.EOF { - log.Println(err) - } - if _, ok := err.(StateError); ok { - state.markDone() - } - return - } - log.Println("disconnected") - }, - } - ) +// buildEventHandler generates and returns a handler to process events received from the subscription. +func buildEventHandler(state *internalState, fidStore store.Singleton) events.Handler { + // disable brief logs when verbose logs are enabled (there's no sense logging twice!) + logger := controller.LogEvents(nil).Unless(state.config.verbose) + return eventrules.Concat( + logAllEvents().If(state.config.verbose), + eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), + controller.LiftErrors().DropOnError(), + ).Handle(events.Handlers{ + scheduler.Event_FAILURE: logger.HandleF(failure), + scheduler.Event_OFFERS: trackOffersReceived(state).HandleF(resourceOffers(state)), + scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), + scheduler.Event_SUBSCRIBED: eventrules.Concat( + logger, + controller.TrackSubscription(fidStore, state.config.failoverTimeout), + ), + }.Otherwise(logger.HandleEvent)) +} - state.cli = calls.Decorators{ - callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), - logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}), - // automatically set the frameworkID for all outgoing calls - calls.SubscribedCaller(frameworkIDStore.Get), - }.Apply(state.cli) - - return controller.Config{ - Context: controlContext, - Framework: buildFrameworkInfo(state.config), - Caller: state.cli, - RegistrationTokens: backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), - - Handler: events.Decorators{ - eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), - events.Decorator(logAllEvents).If(state.config.verbose), - }.Apply(buildEventHandler(state, frameworkIDStore)), +func trackOffersReceived(state *internalState) eventrules.Rule { + return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) { + if err == nil { + state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers())) + } + return chain(ctx, e, err) } } -// buildEventHandler generates and returns a handler to process events received from the subscription. -func buildEventHandler(state *internalState, frameworkIDStore IDStore) events.Handler { - // TODO(jdef) would be nice to merge this ack handler with the status update handler below; need to - // figure out appropriate error propagation among chained handlers. - ack := events.AcknowledgeUpdates(func() calls.Caller { return state.cli }) - return events.NewMux( - events.DefaultHandler(events.HandlerFunc(controller.DefaultHandler)), - events.MapFuncs(map[scheduler.Event_Type]events.HandlerFunc{ - scheduler.Event_FAILURE: func(e *scheduler.Event) error { - log.Println("received a FAILURE event") - f := e.GetFailure() - failure(f.ExecutorID, f.AgentID, f.Status) - return nil - }, - scheduler.Event_OFFERS: func(e *scheduler.Event) error { - if state.config.verbose { - log.Println("received an OFFERS event") - } - offers := e.GetOffers().GetOffers() - state.metricsAPI.offersReceived.Int(len(offers)) - resourceOffers(state, offers) - return nil - }, - scheduler.Event_UPDATE: func(e *scheduler.Event) error { - if err := ack.HandleEvent(e); err != nil { - log.Printf("failed to ack status update for task: %+v", err) - // TODO(jdef) we don't return the error because that would cause the subscription - // to terminate; is that the right thing to do? - } - statusUpdate(state, e.GetUpdate().GetStatus()) - return nil - }, - scheduler.Event_SUBSCRIBED: func(e *scheduler.Event) error { - log.Println("received a SUBSCRIBED event") - frameworkID := e.GetSubscribed().GetFrameworkID().GetValue() - // order of `if` statements are important: tread carefully w/ respect to future changes - if frameworkID == "" { - // sanity check, should **never** happen - return StateError("mesos gave us an empty frameworkID") - } - if state.frameworkID != "" && state.frameworkID != frameworkID && state.config.checkpoint { - return StateError(fmt.Sprintf( - "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", - state.config.failoverTimeout)) - } - if state.frameworkID != frameworkID { - state.frameworkID = frameworkID - frameworkIDStore.Set(frameworkID) - log.Println("FrameworkID", frameworkID) - } - return nil - }, - }), +func failure(_ context.Context, e *scheduler.Event) error { + var ( + f = e.GetFailure() + eid, aid, stat = f.ExecutorID, f.AgentID, f.Status ) -} - -func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { if eid != nil { // executor failed.. msg := "executor '" + eid.Value + "' terminated" @@ -160,123 +134,135 @@ func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { // agent failed.. log.Println("agent '" + aid.Value + "' terminated") } + return nil } -func resourceOffers(state *internalState, offers []mesos.Offer) { - callOption := calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) - tasksLaunchedThisCycle := 0 - offersDeclined := 0 - for i := range offers { +func resourceOffers(state *internalState) events.HandlerFunc { + return func(ctx context.Context, e *scheduler.Event) error { var ( - remaining = mesos.Resources(offers[i].Resources) - tasks = []mesos.TaskInfo{} + offers = e.GetOffers().GetOffers() + callOption = calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) + tasksLaunchedThisCycle = 0 + offersDeclined = 0 ) + for i := range offers { + var ( + remaining = mesos.Resources(offers[i].Resources) + tasks = []mesos.TaskInfo{} + ) - if state.config.verbose { - log.Println("received offer id '" + offers[i].ID.Value + "' with resources " + remaining.String()) - } + if state.config.verbose { + log.Println("received offer id '" + offers[i].ID.Value + + "' with resources " + remaining.String()) + } - var wantsExecutorResources mesos.Resources - if len(offers[i].ExecutorIDs) == 0 { - wantsExecutorResources = mesos.Resources(state.executor.Resources) - } + var wantsExecutorResources mesos.Resources + if len(offers[i].ExecutorIDs) == 0 { + wantsExecutorResources = mesos.Resources(state.executor.Resources) + } - flattened := remaining.Flatten() + flattened := remaining.Flatten() - // avoid the expense of computing these if we can... - if state.config.summaryMetrics && state.config.resourceTypeMetrics { - for name, restype := range flattened.Types() { - if restype == mesos.SCALAR { - sum := flattened.SumScalars(mesos.NamedResources(name)) - state.metricsAPI.offeredResources(sum.GetValue(), name) + // avoid the expense of computing these if we can... + if state.config.summaryMetrics && state.config.resourceTypeMetrics { + for name, restype := range resources.Types(flattened...) { + if restype == mesos.SCALAR { + sum := resources.SumScalars(resourcefilters.Named(name), flattened...) + state.metricsAPI.offeredResources(sum.GetValue(), name) + } } } - } - taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...) - for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) { - state.tasksLaunched++ - taskID := state.tasksLaunched + taskWantsResources := state.wantsTaskResources.Plus(wantsExecutorResources...) + for state.tasksLaunched < state.totalTasks && flattened.ContainsAll(taskWantsResources) { + state.tasksLaunched++ + taskID := state.tasksLaunched - if state.config.verbose { - log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value) - } + if state.config.verbose { + log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value) + } - task := mesos.TaskInfo{ - TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}, - AgentID: offers[i].AgentID, - Executor: state.executor, - Resources: remaining.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign())), - } - task.Name = "Task " + task.TaskID.Value + task := mesos.TaskInfo{ + TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}, + AgentID: offers[i].AgentID, + Executor: state.executor, + Resources: resources.Find(state.wantsTaskResources.Flatten(mesos.RoleName(state.role).Assign()), remaining...), + } + task.Name = "Task " + task.TaskID.Value - remaining.Subtract(task.Resources...) - tasks = append(tasks, task) + remaining.Subtract(task.Resources...) + tasks = append(tasks, task) - flattened = remaining.Flatten() - } + flattened = remaining.Flatten() + } - // build Accept call to launch all of the tasks we've assembled - accept := calls.Accept( - calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID), - ).With(callOption) + // build Accept call to launch all of the tasks we've assembled + accept := calls.Accept( + calls.OfferOperations{calls.OpLaunch(tasks...)}.WithOffers(offers[i].ID), + ).With(callOption) - // send Accept call to mesos - err := calls.CallNoData(state.cli, accept) - if err != nil { - log.Printf("failed to launch tasks: %+v", err) - } else { - if n := len(tasks); n > 0 { - tasksLaunchedThisCycle += n + // send Accept call to mesos + err := calls.CallNoData(ctx, state.cli, accept) + if err != nil { + log.Printf("failed to launch tasks: %+v", err) } else { - offersDeclined++ + if n := len(tasks); n > 0 { + tasksLaunchedThisCycle += n + } else { + offersDeclined++ + } } } - } - state.metricsAPI.offersDeclined.Int(offersDeclined) - state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle) - if state.config.summaryMetrics { - state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) + state.metricsAPI.offersDeclined.Int(offersDeclined) + state.metricsAPI.tasksLaunched.Int(tasksLaunchedThisCycle) + if state.config.summaryMetrics { + state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) + } + return nil } } -func statusUpdate(state *internalState, s mesos.TaskStatus) { - if state.config.verbose { - msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() - if m := s.GetMessage(); m != "" { - msg += " with message '" + m + "'" +func statusUpdate(state *internalState) events.HandlerFunc { + return func(ctx context.Context, e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() + if state.config.verbose { + msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() + if m := s.GetMessage(); m != "" { + msg += " with message '" + m + "'" + } + log.Println(msg) } - log.Println(msg) - } - switch st := s.GetState(); st { - case mesos.TASK_FINISHED: - state.tasksFinished++ - state.metricsAPI.tasksFinished() + switch st := s.GetState(); st { + case mesos.TASK_FINISHED: + state.tasksFinished++ + state.metricsAPI.tasksFinished() - if state.tasksFinished == state.totalTasks { - log.Println("mission accomplished, terminating") - state.markDone() - } else { - tryReviveOffers(state) - } + if state.tasksFinished == state.totalTasks { + log.Println("mission accomplished, terminating") + state.shutdown() + } else { + tryReviveOffers(ctx, state) + } - case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: - state.err = errors.New("Exiting because task " + s.GetTaskID().Value + - " is in an unexpected state " + st.String() + - " with reason " + s.GetReason().String() + - " from source " + s.GetSource().String() + - " with message '" + s.GetMessage() + "'") - state.markDone() + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + state.err = errors.New("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") + state.shutdown() + } + return nil } } -func tryReviveOffers(state *internalState) { +func tryReviveOffers(ctx context.Context, state *internalState) { // limit the rate at which we request offer revival select { case <-state.reviveTokens: // not done yet, revive offers! - err := calls.CallNoData(state.cli, calls.Revive()) + err := calls.CallNoData(ctx, state.cli, calls.Revive()) if err != nil { log.Printf("failed to revive offers: %+v", err) return @@ -287,41 +273,39 @@ func tryReviveOffers(state *internalState) { } // logAllEvents logs every observed event; this is somewhat expensive to do -func logAllEvents(h events.Handler) events.Handler { - return events.HandlerFunc(func(e *scheduler.Event) error { +func logAllEvents() eventrules.Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch eventrules.Chain) (context.Context, *scheduler.Event, error) { log.Printf("%+v\n", *e) - return h.HandleEvent(e) - }) + return ch(ctx, e, err) + } } // eventMetrics logs metrics for every processed API event -func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) events.Decorator { +func eventMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) eventrules.Rule { timed := metricsAPI.eventReceivedLatency if !timingMetrics { timed = nil } harness := xmetrics.NewHarness(metricsAPI.eventReceivedCount, metricsAPI.eventErrorCount, timed, clock) - return events.Metrics(harness) + return eventrules.Metrics(harness, nil) } // callMetrics logs metrics for every outgoing Mesos call -func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) calls.Decorator { +func callMetrics(metricsAPI *metricsAPI, clock func() time.Time, timingMetrics bool) callrules.Rule { timed := metricsAPI.callLatency if !timingMetrics { timed = nil } harness := xmetrics.NewHarness(metricsAPI.callCount, metricsAPI.callErrorCount, timed, clock) - return calls.CallerMetrics(harness) + return callrules.Metrics(harness, nil) } // logCalls logs a specific message string when a particular call-type is observed -func logCalls(messages map[scheduler.Call_Type]string) calls.Decorator { - return func(caller calls.Caller) calls.Caller { - return calls.CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { - if message, ok := messages[c.GetType()]; ok { - log.Println(message) - } - return caller.Call(c) - }) +func logCalls(messages map[scheduler.Call_Type]string) callrules.Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch callrules.Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if message, ok := messages[c.GetType()]; ok { + log.Println(message) + } + return ch(ctx, c, r, err) } } diff --git a/api/v1/cmd/example-scheduler/app/config.go b/api/v1/cmd/example-scheduler/app/config.go index ecd7cbb9..9727dff4 100644 --- a/api/v1/cmd/example-scheduler/app/config.go +++ b/api/v1/cmd/example-scheduler/app/config.go @@ -49,8 +49,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.Var(&cfg.codec, "codec", "Codec to encode/decode scheduler API communications [protobuf, json]") fs.StringVar(&cfg.url, "url", cfg.url, "Mesos scheduler API URL") fs.DurationVar(&cfg.timeout, "timeout", cfg.timeout, "Mesos scheduler API connection timeout") - fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout") - fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable framework checkpointing") + fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout (recover from scheduler failure)") + fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable agent checkpointing for framework tasks (recover from agent failure)") fs.StringVar(&cfg.principal, "principal", cfg.principal, "Framework principal with which to authenticate") fs.StringVar(&cfg.hostname, "hostname", cfg.hostname, "Framework hostname that is advertised to the master") fs.Var(&cfg.labels, "label", "Framework label, may be specified multiple times") diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index fa484271..c3675a41 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -8,12 +8,12 @@ import ( "math/rand" "net/http" "os" - "sync" "time" proto "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" + "github.com/mesos/mesos-go/api/v1/lib/extras/resources" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" @@ -87,8 +87,8 @@ func prepareExecutorInfo( func buildWantsTaskResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.taskCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.taskMemory).Resource, + resources.NewCPUs(config.taskCPU).Resource, + resources.NewMemory(config.taskMemory).Resource, ) log.Println("wants-task-resources = " + r.String()) return @@ -96,8 +96,8 @@ func buildWantsTaskResources(config Config) (r mesos.Resources) { func buildWantsExecutorResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.execCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.execMemory).Resource, + resources.NewCPUs(config.execCPU).Resource, + resources.NewMemory(config.execMemory).Resource, ) log.Println("wants-executor-resources = " + r.String()) return @@ -117,13 +117,6 @@ func buildHTTPSched(cfg Config, creds credentials) calls.Caller { httpcli.Do(httpcli.With( authConfigOpt, httpcli.Timeout(cfg.timeout), - httpcli.Transport(func(t *http.Transport) { - // all calls should be ack'd by the server within this interval. - // TODO(jdef) it probably doesn't make sense if this value is larger - // than cfg.timeout. - t.ResponseHeaderTimeout = 15 * time.Second - t.MaxIdleConnsPerHost = 2 // don't depend on go's default - }), )), ) if cfg.compression { @@ -190,7 +183,7 @@ func loadCredentials(userConfig credentials) (result credentials, err error) { return } -func newInternalState(cfg Config) (*internalState, error) { +func newInternalState(cfg Config, shutdown func()) (*internalState, error) { metricsAPI := initMetrics(cfg) executorInfo, err := prepareExecutorInfo( cfg.executor, @@ -216,31 +209,15 @@ func newInternalState(cfg Config) (*internalState, error) { metricsAPI: metricsAPI, cli: buildHTTPSched(cfg, creds), random: rand.New(rand.NewSource(time.Now().Unix())), - done: make(chan struct{}), + shutdown: shutdown, } return state, nil } -func (state *internalState) markDone() { - state.doneOnce.Do(func() { - close(state.done) - }) -} - -func (state *internalState) isDone() bool { - select { - case <-state.done: - return true - default: - return false - } -} - type internalState struct { tasksLaunched int tasksFinished int totalTasks int - frameworkID string role string executor *mesos.ExecutorInfo cli calls.Caller @@ -249,7 +226,6 @@ type internalState struct { reviveTokens <-chan struct{} metricsAPI *metricsAPI err error - done chan struct{} - doneOnce sync.Once + shutdown func() random *rand.Rand } diff --git a/api/v1/cmd/example-scheduler/app/store.go b/api/v1/cmd/example-scheduler/app/store.go deleted file mode 100644 index bc7fa7a2..00000000 --- a/api/v1/cmd/example-scheduler/app/store.go +++ /dev/null @@ -1,43 +0,0 @@ -package app - -import "sync/atomic" - -// IDStore is a thread-safe abstraction to load and store a stringified ID. -type IDStore interface { - Get() string - Set(string) -} - -type IDStoreAdapter struct { - GetFunc func() string - SetFunc func(string) -} - -func (a IDStoreAdapter) Get() string { - if a.GetFunc != nil { - return a.GetFunc() - } - return "" -} - -func (a IDStoreAdapter) Set(s string) { - if a.SetFunc != nil { - a.SetFunc(s) - } -} - -func NewInMemoryIDStore() IDStore { - var frameworkID atomic.Value - return &IDStoreAdapter{ - GetFunc: func() string { - x := frameworkID.Load() - if x == nil { - return "" - } - return x.(string) - }, - SetFunc: func(s string) { - frameworkID.Store(s) - }, - } -} diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go new file mode 100644 index 00000000..0ee89b21 --- /dev/null +++ b/api/v1/cmd/msh/msh.go @@ -0,0 +1,222 @@ +// msh is a minimal mesos v1 scheduler; it executes a shell command on a mesos agent. +package main + +// Usage: msh {...command line args...} +// +// For example: +// msh -master 10.2.0.5:5050 -- ls -laF /tmp +// +// TODO: -gpu=1 to enable GPU_RESOURCES caps and request 1 gpu +// + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/offers" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/httpcli" + "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +const ( + RFC3339a = "20060102T150405Z0700" +) + +var ( + FrameworkName = "msh" + TaskName = "msh" + MesosMaster = "127.0.0.1:5050" + User = "root" + Role = mesos.RoleName("*") + CPUs = float64(0.010) + Memory = float64(64) + + fidStore store.Singleton + declineAndSuppress bool + refuseSeconds = calls.RefuseSeconds(5 * time.Second) + wantsResources mesos.Resources + taskPrototype mesos.TaskInfo +) + +func init() { + flag.StringVar(&FrameworkName, "framework_name", FrameworkName, "Name of the framework") + flag.StringVar(&TaskName, "task_name", TaskName, "Name of the msh task") + flag.StringVar(&MesosMaster, "master", MesosMaster, "IP:port of the mesos master") + flag.StringVar(&User, "user", User, "OS user that owns the launched task") + flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command") + flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command") + + fidStore = store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) < 1 { // msh by itself prints usage + flag.Usage() + os.Exit(1) + } + + wantsResources = mesos.Resources{ + resources.NewCPUs(CPUs).Resource, + resources.NewMemory(Memory).Resource, + } + taskPrototype = mesos.TaskInfo{ + Name: TaskName, + Command: &mesos.CommandInfo{ + Value: proto.String(args[0]), + Shell: proto.Bool(false), + }, + } + if len(args) > 1 { + taskPrototype.Command.Arguments = args[1:] + } + if err := run(); err != nil { + if exitErr, ok := err.(ExitError); ok { + if code := int(exitErr); code != 0 { + log.Println(exitErr) + os.Exit(code) + } + // else, code=0 indicates success, exit normally + } else { + panic(fmt.Sprintf("%#v", err)) + } + } +} + +func run() error { + var ( + ctx, cancel = context.WithCancel(context.Background()) + caller = callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)).Caller(buildClient()) + ) + + return controller.Run( + ctx, + &mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)}, + caller, + controller.WithEventHandler(buildEventHandler(caller)), + controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)), + controller.WithSubscriptionTerminated(func(err error) { + defer cancel() + if err == io.EOF { + log.Println("disconnected") + } + }), + ) +} + +func buildClient() calls.Caller { + return httpsched.NewCaller(httpcli.New( + httpcli.Endpoint(fmt.Sprintf("http://%s/api/v1/scheduler", MesosMaster)), + )) +} + +func buildEventHandler(caller calls.Caller) events.Handler { + logger := controller.LogEvents(nil) + return controller.LiftErrors().Handle(events.Handlers{ + scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)}, + scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), + scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), + }.Otherwise(logger.HandleEvent)) +} + +func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { + return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) { + if err != nil { + return chain(ctx, e, err) + } + if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress { + return chain(ctx, e, err) + } + off := offers.Slice(e.GetOffers().GetOffers()) + err = calls.CallNoData(ctx, caller, calls.Decline(off.IDs()...).With(refuseSeconds)) + if err == nil { + // we shouldn't have received offers, maybe the prior suppress call failed? + err = calls.CallNoData(ctx, caller, calls.Suppress()) + } + return ctx, e, err // drop + } +} + +func resourceOffers(caller calls.Caller) events.HandlerFunc { + return func(ctx context.Context, e *scheduler.Event) (err error) { + var ( + off = e.GetOffers().GetOffers() + index = offers.NewIndex(off, nil) + match = index.Find(offers.ContainsResources(wantsResources)) + ) + if match != nil { + task := taskPrototype + task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)} + task.AgentID = match.AgentID + task.Resources = resources.Find(wantsResources.Flatten(Role.Assign()), match.Resources...) + + err = calls.CallNoData(ctx, caller, calls.Accept( + calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), + )) + if err != nil { + return + } + + declineAndSuppress = true + } else { + log.Println("rejected insufficient offers") + } + // decline all but the possible match + delete(index, match.GetID()) + err = calls.CallNoData(ctx, caller, calls.Decline(index.IDs()...).With(refuseSeconds)) + if err != nil { + return + } + if declineAndSuppress { + err = calls.CallNoData(ctx, caller, calls.Suppress()) + } + return + } +} + +func statusUpdate(_ context.Context, e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() + switch st := s.GetState(); st { + case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING: + log.Printf("status update from agent %q: %v", s.GetAgentID().GetValue(), st) + if st != mesos.TASK_FINISHED { + return nil + } + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + log.Println("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") + return ExitError(3) + default: + log.Println("unexpected task state, aborting", st) + return ExitError(4) + } + return ExitError(0) // kind of ugly, but better than os.Exit(0) +} + +type ExitError int + +func (e ExitError) Error() string { return fmt.Sprintf("exit code %d", int(e)) } diff --git a/api/v1/lib/backoff/backoff.go b/api/v1/lib/backoff/backoff.go index 347b6ce5..3993829a 100644 --- a/api/v1/lib/backoff/backoff.go +++ b/api/v1/lib/backoff/backoff.go @@ -27,6 +27,7 @@ func BurstNotifier(burst int, minWait, maxWait time.Duration, until <-chan struc // listen for tokens emitted by child buckets and forward them to the tokens chan tokens := make(chan struct{}) go func() { + defer close(tokens) for { i, _, _ := reflect.Select(cases) if i == burst { diff --git a/api/v1/lib/builders.go b/api/v1/lib/builders.go deleted file mode 100644 index 01642b31..00000000 --- a/api/v1/lib/builders.go +++ /dev/null @@ -1,60 +0,0 @@ -package mesos - -type ( - // ResourceBuilder simplifies construction of Resource objects - ResourceBuilder struct{ *Resource } - // RangeBuilder simplifies construction of Range objects - RangeBuilder struct{ Ranges } -) - -func BuildRanges() RangeBuilder { - return RangeBuilder{Ranges: Ranges(nil)} -} - -// Span is a functional option for Ranges, defines the begin and end points of a -// continuous span within a range -func (rb RangeBuilder) Span(bp, ep uint64) RangeBuilder { - rb.Ranges = append(rb.Ranges, Value_Range{Begin: bp, End: ep}) - return rb -} - -func BuildResource() ResourceBuilder { - return ResourceBuilder{&Resource{}} -} -func (rb ResourceBuilder) Name(name string) ResourceBuilder { - rb.Resource.Name = name - return rb -} -func (rb ResourceBuilder) Role(role string) ResourceBuilder { - rb.Resource.Role = &role - return rb -} -func (rb ResourceBuilder) Scalar(x float64) ResourceBuilder { - rb.Resource.Type = SCALAR.Enum() - rb.Resource.Scalar = &Value_Scalar{Value: x} - return rb -} -func (rb ResourceBuilder) Set(x ...string) ResourceBuilder { - rb.Resource.Type = SET.Enum() - rb.Resource.Set = &Value_Set{Item: x} - return rb -} -func (rb ResourceBuilder) Ranges(rs Ranges) ResourceBuilder { - rb.Resource.Type = RANGES.Enum() - rb.Resource.Ranges = rb.Resource.Ranges.Add(&Value_Ranges{Range: rs}) - return rb -} -func (rb ResourceBuilder) Disk(persistenceID, containerPath string) ResourceBuilder { - rb.Resource.Disk = &Resource_DiskInfo{} - if containerPath != "" { - rb.Resource.Disk.Volume = &Volume{ContainerPath: containerPath} - } - if persistenceID != "" { - rb.Resource.Disk.Persistence = &Resource_DiskInfo_Persistence{ID: persistenceID} - } - return rb -} -func (rb ResourceBuilder) Revocable() ResourceBuilder { - rb.Resource.Revocable = &Resource_RevocableInfo{} - return rb -} diff --git a/api/v1/lib/encoding/codec.go b/api/v1/lib/encoding/codec.go index 1d4255b3..ebcd6ccd 100644 --- a/api/v1/lib/encoding/codec.go +++ b/api/v1/lib/encoding/codec.go @@ -48,7 +48,15 @@ type Codec struct { } // String implements the fmt.Stringer interface. -func (c *Codec) String() string { return c.Name } +func (c *Codec) String() string { + if c == nil { + return "" + } + return c.Name +} + +func (c *Codec) RequestContentType() string { return c.MediaTypes[0] } +func (c *Codec) ResponseContentType() string { return c.MediaTypes[1] } type ( // Marshaler composes the supported marshaling formats. diff --git a/api/v1/lib/encoding/framing/decoder.go b/api/v1/lib/encoding/framing/decoder.go index 863cde56..af663fc0 100644 --- a/api/v1/lib/encoding/framing/decoder.go +++ b/api/v1/lib/encoding/framing/decoder.go @@ -1,7 +1,7 @@ package framing import ( - "fmt" + "io" ) type ( @@ -9,58 +9,33 @@ type ( UnmarshalFunc func([]byte, interface{}) error // Decoder reads and decodes Protobuf messages from an io.Reader. - Decoder struct { - r Reader - buf []byte - uf UnmarshalFunc + Decoder interface { + // Decode reads the next encoded message from its input and stores it + // in the value pointed to by m. If m isn't a proto.Message, Decode will panic. + Decode(interface{}) error } -) - -// NewDecoder returns a new Decoder that reads from the given io.Reader. -// If r does not also implement StringReader, it will be wrapped in a bufio.Reader. -func NewDecoder(r Reader, uf UnmarshalFunc) *Decoder { - return &Decoder{r: r, buf: make([]byte, 4096), uf: uf} -} - -// MaxSize is the maximum decodable message size. -const MaxSize = 4 << 20 // 4MB -var ( - // ErrSize is returned by Decode calls when a message would exceed the maximum - // allowed size. - ErrSize = fmt.Errorf("proto: message exceeds %dMB", MaxSize>>20) + // DecoderFunc is the functional adaptation of Decoder + DecoderFunc func(interface{}) error ) -// Decode reads the next Protobuf-encoded message from its input and stores it -// in the value pointed to by m. If m isn't a proto.Message, Decode will panic. -func (d *Decoder) Decode(m interface{}) error { - var ( - buf = d.buf - readlen = 0 - ) - for { - eof, nr, err := d.r.ReadFrame(buf) - if err != nil { - return err - } - - readlen += nr - if readlen > MaxSize { - return ErrSize - } +func (f DecoderFunc) Decode(m interface{}) error { return f(m) } - if eof { - return d.uf(d.buf[:readlen], m) - } +var _ = Decoder(DecoderFunc(nil)) - if len(buf) == nr { - // readlen and len(d.buf) are the same here - newbuf := make([]byte, readlen+4096) - copy(newbuf, d.buf) - d.buf = newbuf - buf = d.buf[readlen:] - } else { - buf = buf[nr:] +// NewDecoder returns a new Decoder that reads from the given io.Reader. +// If r does not also implement StringReader, it will be wrapped in a bufio.Reader. +func NewDecoder(r Reader, uf UnmarshalFunc) DecoderFunc { + return func(m interface{}) error { + // Note: the buf returned by ReadFrame will change over time, it can't be sub-sliced + // and then those sub-slices retained. Examination of generated proto code seems to indicate + // that byte buffers are copied vs. referenced by sub-slice (gogo protoc). + frame, err := r.ReadFrame() + if err == nil || err == io.EOF { + if err2 := uf(frame, m); err2 != nil { + err = err2 + } } + return err } } diff --git a/api/v1/lib/encoding/framing/decoder_test.go b/api/v1/lib/encoding/framing/decoder_test.go new file mode 100644 index 00000000..496a394f --- /dev/null +++ b/api/v1/lib/encoding/framing/decoder_test.go @@ -0,0 +1,61 @@ +package framing + +import ( + "errors" + "fmt" + "io" + "reflect" + "testing" +) + +func TestNewDecoder(t *testing.T) { + var ( + byteCopy = UnmarshalFunc(func(b []byte, m interface{}) error { + if m == nil { + return errors.New("unmarshal target may not be nil") + } + v, ok := m.(*[]byte) + if !ok { + return fmt.Errorf("expected *[]byte instead of %T", m) + } + if v == nil { + return errors.New("target *[]byte may not be nil") + } + *v = append((*v)[:0], b...) + return nil + }) + singletonReader = func(b []byte) ReaderFunc { + eof := false + return func() ([]byte, error) { + if eof { + panic("reader should only be called once") + } + eof = true + return b, io.EOF + } + } + errorReader = func(err error) ReaderFunc { + return func() ([]byte, error) { return nil, err } + } + ) + for ti, tc := range []struct { + r Reader + wants []byte + wantsErr error + }{ + {errorReader(ErrorBadSize), nil, ErrorBadSize}, + {singletonReader(([]byte)("james")), ([]byte)("james"), io.EOF}, + } { + var ( + buf []byte + d = NewDecoder(tc.r, byteCopy) + err = d.Decode(&buf) + ) + if err != tc.wantsErr { + t.Errorf("test case %d failed: expected error %q instead of %q", ti, tc.wantsErr, err) + } + if !reflect.DeepEqual(buf, tc.wants) { + t.Errorf("test case %d failed: expected %#v instead of %#v", tc.wants, buf) + } + } +} diff --git a/api/v1/lib/encoding/framing/framing.go b/api/v1/lib/encoding/framing/framing.go index 60f67242..054cb24b 100644 --- a/api/v1/lib/encoding/framing/framing.go +++ b/api/v1/lib/encoding/framing/framing.go @@ -1,5 +1,25 @@ package framing -type Reader interface { - ReadFrame(buf []byte) (endOfFrame bool, n int, err error) -} +type Error string + +func (err Error) Error() string { return string(err) } + +const ( + ErrorUnderrun = Error("frame underrun, unexpected EOF") + ErrorBadSize = Error("bad frame size") + ErrorOversizedFrame = Error("oversized frame, max size exceeded") +) + +type ( + // Reader generates data frames from some source, returning io.EOF with the final frame. + Reader interface { + ReadFrame() (frame []byte, err error) + } + + // ReaderFunc is the functional adaptation of Reader + ReaderFunc func() ([]byte, error) +) + +func (f ReaderFunc) ReadFrame() ([]byte, error) { return f() } + +var _ = Reader(ReaderFunc(nil)) diff --git a/api/v1/lib/executor/calls/calls_generated.go b/api/v1/lib/executor/calls/calls_generated.go new file mode 100644 index 00000000..e4e1b01d --- /dev/null +++ b/api/v1/lib/executor/calls/calls_generated.go @@ -0,0 +1,38 @@ +package calls + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type C:*executor.Call +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + // Caller is the public interface this framework scheduler's should consume + Caller interface { + // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. + Call(context.Context, *executor.Call) (mesos.Response, error) + } + + // CallerFunc is the functional adaptation of the Caller interface + CallerFunc func(context.Context, *executor.Call) (mesos.Response, error) +) + +// Call implements the Caller interface for CallerFunc +func (f CallerFunc) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + return f(ctx, c) +} + +// CallNoData is a convenience func that executes the given Call using the provided Caller +// and always drops the response data. +func CallNoData(ctx context.Context, caller Caller, call *executor.Call) error { + resp, err := caller.Call(ctx, call) + if resp != nil { + resp.Close() + } + return err +} diff --git a/api/v1/lib/executor/calls/gen.go b/api/v1/lib/executor/calls/gen.go new file mode 100644 index 00000000..b78ed2ea --- /dev/null +++ b/api/v1/lib/executor/calls/gen.go @@ -0,0 +1,3 @@ +package calls + +//go:generate go run ../../extras/gen/callers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type C:*executor.Call diff --git a/api/v1/lib/executor/events/decorators.go b/api/v1/lib/executor/events/decorators.go deleted file mode 100644 index 4eb31eb5..00000000 --- a/api/v1/lib/executor/events/decorators.go +++ /dev/null @@ -1,66 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/executor" -) - -type ( - // Decorator functions typically modify behavior of the given delegate Handler. - Decorator func(Handler) Handler - - // Decorators aggregates Decorator functions - Decorators []Decorator -) - -var noopDecorator = Decorator(func(h Handler) Handler { return h }) - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// When returns a Decorator that evaluates the bool func every time the Handler is invoked. -// When f returns true, the Decorated Handler is invoked, otherwise the original Handler is. -func (d Decorator) When(f func() bool) Decorator { - if d == nil || f == nil { - return noopDecorator - } - return func(h Handler) Handler { - // generates a new decorator every time the Decorator func is invoked. - // probably OK for now. - decorated := d(h) - return HandlerFunc(func(e *executor.Event) (err error) { - if f() { - // let the decorated handler process this - err = decorated.HandleEvent(e) - } else { - err = h.HandleEvent(e) - } - return - }) - } -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Handler that is ultimately returned. -func (ds Decorators) Apply(h Handler) Handler { - for _, d := range ds { - h = d.Apply(h) - } - return h -} - -func (d Decorator) Apply(h Handler) Handler { - if d != nil { - h = d(h) - } - return h -} diff --git a/api/v1/lib/executor/events/events.go b/api/v1/lib/executor/events/events.go deleted file mode 100644 index 1e6d68c2..00000000 --- a/api/v1/lib/executor/events/events.go +++ /dev/null @@ -1,91 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/executor" -) - -type ( - // Handler is invoked upon the occurrence of some executor event that is generated - // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) - Handler interface { - HandleEvent(*executor.Event) error - } - - // HandlerFunc is a functional adaptation of the Handler interface - HandlerFunc func(*executor.Event) error - - // Mux maps event types to Handlers (only one Handler for each type). A "default" - // Handler implementation may be provided to handle cases in which there is no - // registered Handler for specific event type. - Mux struct { - handlers map[executor.Event_Type]Handler - defaultHandler Handler - } - - // Option is a functional configuration option that returns an "undo" option that - // reverts the change made by the option. - Option func(*Mux) Option -) - -// HandleEvents implements Handler for HandlerFunc -func (f HandlerFunc) HandleEvent(e *executor.Event) error { return f(e) } - -// NewMux generates and returns a new, empty Mux instance. -func NewMux(opts ...Option) *Mux { - m := &Mux{ - handlers: make(map[executor.Event_Type]Handler), - } - m.With(opts...) - return m -} - -// With applies the given options to the Mux and returns the result of invoking -// the last Option func. If no options are provided then a no-op Option is returned. -func (m *Mux) With(opts ...Option) Option { - var last Option // defaults to noop - last = Option(func(x *Mux) Option { return last }) - - for _, o := range opts { - if o != nil { - last = o(m) - } - } - return last -} - -// HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *executor.Event) (err error) { - h, found := m.handlers[e.GetType()] - if !found { - h = m.defaultHandler - } - if h != nil { - err = h.HandleEvent(e) - } - return -} - -// Handle returns an option that configures a Handler to handle a specific event type. -// If the specified Handler is nil then any currently registered Handler for the given -// event type is deleted upon application of the returned Option. -func Handle(et executor.Event_Type, eh Handler) Option { - return func(m *Mux) Option { - old := m.handlers[et] - if eh == nil { - delete(m.handlers, et) - } else { - m.handlers[et] = eh - } - return Handle(et, old) - } -} - -// DefaultHandler returns an option that configures the default handler that's invoked -// in cases where there is no Handler registered for specific event type. -func DefaultHandler(eh Handler) Option { - return func(m *Mux) Option { - old := m.defaultHandler - m.defaultHandler = eh - return DefaultHandler(old) - } -} diff --git a/api/v1/lib/executor/events/events_generated.go b/api/v1/lib/executor/events/events_generated.go new file mode 100644 index 00000000..3fa0d7a6 --- /dev/null +++ b/api/v1/lib/executor/events/events_generated.go @@ -0,0 +1,87 @@ +package events + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + // Handler is invoked upon the occurrence of some scheduler event that is generated + // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) + Handler interface { + HandleEvent(context.Context, *executor.Event) error + } + + // HandlerFunc is a functional adaptation of the Handler interface + HandlerFunc func(context.Context, *executor.Event) error + + // Handlers executes an event Handler according to the event's type + Handlers map[executor.Event_Type]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[executor.Event_Type]HandlerFunc +) + +// HandleEvent implements Handler for HandlerFunc +func (f HandlerFunc) HandleEvent(ctx context.Context, e *executor.Event) error { return f(ctx, e) } + +type noopHandler int + +func (_ noopHandler) HandleEvent(_ context.Context, _ *executor.Event) error { return nil } + +// NoopHandler is a Handler that does nothing and always returns nil +const NoopHandler = noopHandler(0) + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *executor.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *executor.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) diff --git a/api/v1/lib/executor/events/gen.go b/api/v1/lib/executor/events/gen.go new file mode 100644 index 00000000..673ecd79 --- /dev/null +++ b/api/v1/lib/executor/events/gen.go @@ -0,0 +1,3 @@ +package events + +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type diff --git a/api/v1/lib/extras/executor/callrules/callers_generated.go b/api/v1/lib/extras/executor/callrules/callers_generated.go new file mode 100644 index 00000000..dd03eb0f --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callers_generated.go @@ -0,0 +1,58 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/calls -type E:*executor.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/executor" + "github.com/mesos/mesos-go/api/v1/lib/executor/calls" +) + +// Call returns a Rule that invokes the given Caller +func Call(caller calls.Caller) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c *executor.Call, _ mesos.Response, _ error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf calls.CallerFunc) Rule { + return Call(calls.Caller(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller calls.Caller) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf calls.CallerFunc) Rule { + return r.Caller(calls.Caller(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c *executor.Call) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = calls.Caller(Rule(nil)) + _ = calls.Caller(Rules(nil)) +) diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated.go b/api/v1/lib/extras/executor/callrules/callrules_generated.go new file mode 100644 index 00000000..aa4b0ed3 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callrules_generated.go @@ -0,0 +1,400 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *executor.Call, mesos.Response, error, Chain) (context.Context, *executor.Call, mesos.Response, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *executor.Call, mesos.Response, error, Chain) (context.Context, *executor.Call, mesos.Response, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *executor.Call, mesos.Response, error) (context.Context, *executor.Call, mesos.Response, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) +) + +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + return ctx, e, z, err +} + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if r != nil { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(rs.Chain()(ctx, e, z, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *executor.Call, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return ChainIdentity + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +const ( + MsgNoErrors = "no errors" +) + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return MsgNoErrors + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + switch len(es) { + case 0: + return nil + case 1: + return es[0] + default: + return es + } +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, z, err = r(ctx, e, z, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, z, err = ch(ctx, e, z, err) + } + return ctx, e, z, err + } +} + +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } +} + +type Overflow int + +const ( + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise +) + +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + return limit(r, acquireChan(p), over, otherwise) +} + +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { + if tokenCh == nil { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } + } + return false + } + } + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-tokenCh: + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } + case <-ctx.Done(): + } + return false + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { + if r == nil { + return nil + } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } + blocking := false + switch over { + case OverflowOtherwise: + case OverflowWait: + blocking = true + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, z, err, ch) + } + return r(ctx, e, z, err, ch) + } +} + +/* TODO(jdef) not sure that this is very useful, leaving out for now... + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { + if nthTime < 2 || r == nil { + return r + } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { + var ( + i = 1 // begin with the first event seen + m sync.Mutex + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: + m.Lock() + i-- + if i <= 0 { + i = nthTime + result = true + } + m.Unlock() + } + return + } +} + +*/ + +// Drop aborts the Chain and returns the (context.Context, *executor.Call, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Call, mesos.Response, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, _ Chain) (context.Context, *executor.Call, mesos.Response, error) { + return r.Eval(ctx, e, z, err, ChainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if err != nil { + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/executor/callrules/callrules_generated_test.go b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go new file mode 100644 index 00000000..39ef4e4b --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/callrules_generated_test.go @@ -0,0 +1,719 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func prototype() *executor.Call { return &executor.Call{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + t.Log("executing", name) + return r(ctx, e, z, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error) (context.Context, *executor.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *executor.Call, _ mesos.Response, _ error) (context.Context, *executor.Call, mesos.Response, error) { + panic(x) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + var z0 mesos.Response + + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, ChainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + var z0 mesos.Response + var zp = &mesos.ResponseWrapper{} + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *executor.Call + z mesos.Response + err error + }{ + {nil, z0, nil}, + {nil, z0, a}, + {p, z0, nil}, + {p, z0, a}, + + {nil, zp, nil}, + {nil, zp, a}, + {p, zp, nil}, + {p, zp, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, ChainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, z, err + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, ChainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + var zp = &mesos.ResponseWrapper{} + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } + var ( + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() + ch4 = make(chan struct{}) // non-nil, closed + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + close(ch4) + for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 + ctx context.Context + ch <-chan struct{} + over Overflow + otherwise Rule + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + } { + var ( + i, j int + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain + ) + for k, r := range []Rule{r1, r2} { + // execute each rule twice + for x := 0; x < 2; x++ { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) + } + } + } + } + // test blocking capability via rateLimit + blocked := false + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, zp, nil, ChainIdentity) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } +} diff --git a/api/v1/lib/extras/executor/callrules/gen.go b/api/v1/lib/extras/executor/callrules/gen.go new file mode 100644 index 00000000..521598a8 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/gen.go @@ -0,0 +1,7 @@ +package callrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} + +//go:generate go run ../../gen/rule_callers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/calls -type E:*executor.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type ET:executor.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go diff --git a/api/v1/lib/extras/executor/callrules/metrics_generated.go b/api/v1/lib/extras/executor/callrules/metrics_generated.go new file mode 100644 index 00000000..7e716804 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/metrics_generated.go @@ -0,0 +1,49 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type ET:executor.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +// Labeler generates a set of strings that should be associated with metrics that are generated for the given event. +type Labeler func(ctx context.Context, e *executor.Call) []string + +var defaultLabels = func() map[executor.Call_Type][]string { + m := make(map[executor.Call_Type][]string) + for k, v := range executor.Call_Type_name { + m[executor.Call_Type(k)] = []string{strings.ToLower(v)} + } + return m +}() + +func defaultLabeler(ctx context.Context, e *executor.Call) []string { + return defaultLabels[e.GetType()] +} + +// Metrics generates a Rule that invokes the given harness for each event, using the labels generated by the Labeler. +// Panics if harness or labeler is nil. +func Metrics(harness metrics.Harness, labeler Labeler) Rule { + if harness == nil { + panic("harness is a required parameter") + } + if labeler == nil { + labeler = defaultLabeler + } + return func(ctx context.Context, e *executor.Call, z mesos.Response, err error, ch Chain) (context.Context, *executor.Call, mesos.Response, error) { + labels := labeler(ctx, e) + harness(func() error { + ctx, e, z, err = ch(ctx, e, z, err) + return err + }, labels...) + return ctx, e, z, err + } +} diff --git a/api/v1/lib/extras/executor/callrules/metrics_generated_test.go b/api/v1/lib/extras/executor/callrules/metrics_generated_test.go new file mode 100644 index 00000000..04d41bc2 --- /dev/null +++ b/api/v1/lib/extras/executor/callrules/metrics_generated_test.go @@ -0,0 +1,58 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Call:&executor.Call{} -type ET:executor.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func TestMetrics(t *testing.T) { + var ( + i int + ctx = context.Background() + p = &executor.Call{} + a = errors.New("a") + h = func(f func() error, _ ...string) error { + i++ + return f() + } + r = Metrics(h, func(_ context.Context, _ *executor.Call) []string { return nil }) + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + e *executor.Call + z mesos.Response + err error + }{ + {ctx, p, zp, a}, + {ctx, p, zp, nil}, + + {ctx, p, nil, a}, + {ctx, nil, zp, a}, + } { + c, e, z, err := r.Eval(tc.ctx, tc.e, tc.z, tc.err, ChainIdentity) + if !reflect.DeepEqual(c, tc.ctx) { + t.Errorf("test case %d: expected context %q instead of %q", ti, tc.ctx, c) + } + if !reflect.DeepEqual(e, tc.e) { + t.Errorf("test case %d: expected event %q instead of %q", ti, tc.e, e) + } + if !reflect.DeepEqual(z, tc.z) { + t.Errorf("expected return object %q instead of %q", z, tc.z) + } + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("test case %d: expected error %q instead of %q", ti, tc.err, err) + } + if y := ti + 1; y != i { + t.Errorf("test case %d: expected count %q instead of %q", ti, y, i) + } + } +} diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go new file mode 100644 index 00000000..327d85fe --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated.go @@ -0,0 +1,399 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *executor.Event, error, Chain) (context.Context, *executor.Event, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *executor.Event, error, Chain) (context.Context, *executor.Event, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *executor.Event, error) (context.Context, *executor.Event, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) +) + +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + return ctx, e, err +} + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if r != nil { + return r(ctx, e, err, ch) + } + return ch(ctx, e, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(rs.Chain()(ctx, e, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *executor.Event, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return ChainIdentity + } + return func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + return rs[0].Eval(ctx, e, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +const ( + MsgNoErrors = "no errors" +) + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return MsgNoErrors + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + switch len(es) { + case 0: + return nil + case 1: + return es[0] + default: + return es + } +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, err = r(ctx, e, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, err = ch(ctx, e, err) + } + return ctx, e, err + } +} + +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } +} + +type Overflow int + +const ( + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise +) + +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + return limit(r, acquireChan(p), over, otherwise) +} + +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { + if tokenCh == nil { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } + } + return false + } + } + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-tokenCh: + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } + case <-ctx.Done(): + } + return false + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { + if r == nil { + return nil + } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } + blocking := false + switch over { + case OverflowOtherwise: + case OverflowWait: + blocking = true + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, err, ch) + } + return r(ctx, e, err, ch) + } +} + +/* TODO(jdef) not sure that this is very useful, leaving out for now... + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { + if nthTime < 2 || r == nil { + return r + } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { + var ( + i = 1 // begin with the first event seen + m sync.Mutex + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: + m.Lock() + i-- + if i <= 0 { + i = nthTime + result = true + } + m.Unlock() + } + return + } +} + +*/ + +// Drop aborts the Chain and returns the (context.Context, *executor.Event, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *executor.Event, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *executor.Event, err error, _ Chain) (context.Context, *executor.Event, error) { + return r.Eval(ctx, e, err, ChainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(ctx, e, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if err != nil { + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go new file mode 100644 index 00000000..3d874d9b --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/eventrules_generated_test.go @@ -0,0 +1,646 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func prototype() *executor.Event { return &executor.Event{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + t.Log("executing", name) + return r(ctx, e, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + return ch(ctx, e, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *executor.Event, err error) (context.Context, *executor.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *executor.Event, _ error) (context.Context, *executor.Event, error) { + panic(x) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, ChainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *executor.Event + err error + }{ + {nil, nil}, + {nil, a}, + {p, nil}, + {p, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, err = rule(ctx, tc.e, tc.err, ChainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, err + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, ChainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, err := r2(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, err := r2(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, err = r3(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } + var ( + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() + ch4 = make(chan struct{}) // non-nil, closed + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() + ) + close(ch4) + for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 + ctx context.Context + ch <-chan struct{} + over Overflow + otherwise Rule + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + } { + var ( + i, j int + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain + ) + for k, r := range []Rule{r1, r2} { + // execute each rule twice + for x := 0; x < 2; x++ { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) + } + } + } + } + // test blocking capability via rateLimit + blocked := false + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, err := r(ctx, p, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, nil, ChainIdentity) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } +} diff --git a/api/v1/lib/extras/executor/eventrules/gen.go b/api/v1/lib/extras/executor/eventrules/gen.go new file mode 100644 index 00000000..d17240e1 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/gen.go @@ -0,0 +1,7 @@ +package eventrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} + +//go:generate go run ../../gen/rule_handlers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/events -type E:*executor.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type -output metrics_generated.go diff --git a/api/v1/lib/extras/executor/eventrules/handlers_generated.go b/api/v1/lib/extras/executor/eventrules/handlers_generated.go new file mode 100644 index 00000000..1cf7ee8c --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/handlers_generated.go @@ -0,0 +1,51 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -import github.com/mesos/mesos-go/api/v1/lib/executor/events -type E:*executor.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/executor" + "github.com/mesos/mesos-go/api/v1/lib/executor/events" +) + +// Handle generates a rule that executes the given events.Handler. +func Handle(h events.Handler) Rule { + if h == nil { + return nil + } + return func(ctx context.Context, e *executor.Event, err error, chain Chain) (context.Context, *executor.Event, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h events.HandlerFunc) Rule { + return Handle(events.Handler(h)) +} + +// Handle returns a Rule that invokes the receiver, then the given events.Handler +func (r Rule) Handle(h events.Handler) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h events.HandlerFunc) Rule { + return r.Handle(events.Handler(h)) +} + +// HandleEvent implements events.Handler for Rule +func (r Rule) HandleEvent(ctx context.Context, e *executor.Event) (err error) { + if r == nil { + return nil + } + _, _, err = r(ctx, e, nil, ChainIdentity) + return +} + +// HandleEvent implements events.Handler for Rules +func (rs Rules) HandleEvent(ctx context.Context, e *executor.Event) error { + return Rule(rs.Eval).HandleEvent(ctx, e) +} diff --git a/api/v1/lib/extras/executor/eventrules/metrics_generated.go b/api/v1/lib/extras/executor/eventrules/metrics_generated.go new file mode 100644 index 00000000..d85f4f57 --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/metrics_generated.go @@ -0,0 +1,48 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +// Labeler generates a set of strings that should be associated with metrics that are generated for the given event. +type Labeler func(ctx context.Context, e *executor.Event) []string + +var defaultLabels = func() map[executor.Event_Type][]string { + m := make(map[executor.Event_Type][]string) + for k, v := range executor.Event_Type_name { + m[executor.Event_Type(k)] = []string{strings.ToLower(v)} + } + return m +}() + +func defaultLabeler(ctx context.Context, e *executor.Event) []string { + return defaultLabels[e.GetType()] +} + +// Metrics generates a Rule that invokes the given harness for each event, using the labels generated by the Labeler. +// Panics if harness or labeler is nil. +func Metrics(harness metrics.Harness, labeler Labeler) Rule { + if harness == nil { + panic("harness is a required parameter") + } + if labeler == nil { + labeler = defaultLabeler + } + return func(ctx context.Context, e *executor.Event, err error, ch Chain) (context.Context, *executor.Event, error) { + labels := labeler(ctx, e) + harness(func() error { + ctx, e, err = ch(ctx, e, err) + return err + }, labels...) + return ctx, e, err + } +} diff --git a/api/v1/lib/extras/executor/eventrules/metrics_generated_test.go b/api/v1/lib/extras/executor/eventrules/metrics_generated_test.go new file mode 100644 index 00000000..b51fd90f --- /dev/null +++ b/api/v1/lib/extras/executor/eventrules/metrics_generated_test.go @@ -0,0 +1,50 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/executor -type E:*executor.Event:&executor.Event{} -type ET:executor.Event_Type -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/executor" +) + +func TestMetrics(t *testing.T) { + var ( + i int + ctx = context.Background() + p = &executor.Event{} + a = errors.New("a") + h = func(f func() error, _ ...string) error { + i++ + return f() + } + r = Metrics(h, func(_ context.Context, _ *executor.Event) []string { return nil }) + ) + for ti, tc := range []struct { + ctx context.Context + e *executor.Event + err error + }{ + {ctx, p, a}, + {ctx, p, nil}, + {ctx, nil, a}, + } { + c, e, err := r.Eval(tc.ctx, tc.e, tc.err, ChainIdentity) + if !reflect.DeepEqual(c, tc.ctx) { + t.Errorf("test case %d: expected context %q instead of %q", ti, tc.ctx, c) + } + if !reflect.DeepEqual(e, tc.e) { + t.Errorf("test case %d: expected event %q instead of %q", ti, tc.e, e) + } + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("test case %d: expected error %q instead of %q", ti, tc.err, err) + } + if y := ti + 1; y != i { + t.Errorf("test case %d: expected count %q instead of %q", ti, y, i) + } + } +} diff --git a/api/v1/lib/extras/gen/callers.go b/api/v1/lib/extras/gen/callers.go new file mode 100644 index 00000000..72b8fb4b --- /dev/null +++ b/api/v1/lib/extras/gen/callers.go @@ -0,0 +1,54 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "C" -}} +type ( + // Caller is the public interface this framework scheduler's should consume + Caller interface { + // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. + Call(context.Context, {{.Type "C"}}) (mesos.Response, error) + } + + // CallerFunc is the functional adaptation of the Caller interface + CallerFunc func(context.Context, {{.Type "C"}}) (mesos.Response, error) +) + +// Call implements the Caller interface for CallerFunc +func (f CallerFunc) Call(ctx context.Context, c {{.Type "C"}}) (mesos.Response, error) { + return f(ctx, c) +} + +// CallNoData is a convenience func that executes the given Call using the provided Caller +// and always drops the response data. +func CallNoData(ctx context.Context, caller Caller, call {{.Type "C"}}) error { + resp, err := caller.Call(ctx, call) + if resp != nil { + resp.Close() + } + return err +} +`)) diff --git a/api/v1/lib/extras/gen/gen.go b/api/v1/lib/extras/gen/gen.go new file mode 100644 index 00000000..fe88a517 --- /dev/null +++ b/api/v1/lib/extras/gen/gen.go @@ -0,0 +1,241 @@ +// +build ignore + +package main + +import ( + "errors" + "flag" + "fmt" + "log" + "os" + "strings" + "text/template" +) + +type ( + Type struct { + Notation string + Spec string + Prototype string + } + + TypeMap map[string]Type + + Config struct { + Package string + Imports []string + Args string // arguments that we were invoked with; TODO(jdef) rename this to Flags? + Types TypeMap + } +) + +func (c *Config) String() string { + if c == nil { + return "" + } + return fmt.Sprintf("%#v", ([]string)(c.Imports)) +} + +func (c *Config) Set(s string) error { + // TODO(jdef) this is gnarly, *Config shouldn't actually implement Set or String; define a new type for imports + c.Imports = append(c.Imports, s) + return nil +} + +func (tm *TypeMap) Set(s string) error { + tok := strings.SplitN(s, ":", 3) + + if len(tok) < 2 { + return errors.New("expected {notation}:{type-spec} syntax, instead of " + s) + } + + if *tm == nil { + *tm = make(TypeMap) + } + + t := (*tm)[tok[0]] + t.Notation, t.Spec = tok[0], tok[1] + + if t.Notation == "" { + return fmt.Errorf("type notation in %q may not be an empty string", s) + } + + if t.Spec == "" { + return fmt.Errorf("type specification in %q may not be an empty string", s) + } + + if len(tok) == 3 { + t.Prototype = tok[2] + + if t.Prototype == "" { + return fmt.Errorf("prototype specification in %q may not be an empty string", s) + } + } + + (*tm)[tok[0]] = t + return nil +} + +func (tm *TypeMap) String() string { + if tm == nil { + return "" + } + return fmt.Sprintf("%#v", *tm) +} + +func (c *Config) Var(notation string, names ...string) string { + t := c.Type(notation) + if t == "" || len(names) == 0 { + return "" + } + return "var " + strings.Join(names, ",") + " " + t +} + +func (c *Config) Arg(notation, name string) string { + t := c.Type(notation) + if t == "" { + return "" + } + if name == "" { + return t + } + if strings.HasSuffix(name, ",") { + return strings.TrimSpace(name[:len(name)-1]+" "+t) + ", " + } + return name + " " + t +} + +func (c *Config) Ref(notation, name string) (string, error) { + t := c.Type(notation) + if t == "" || name == "" { + return "", nil + } + if strings.HasSuffix(name, ",") { + if len(name) < 2 { + return "", errors.New("expected ref name before comma") + } + return name[:len(name)-1] + ", ", nil + } + return name, nil +} + +func (c *Config) RequireType(notation string) (string, error) { + _, ok := c.Types[notation] + if !ok { + return "", fmt.Errorf("type %q is required but not specified", notation) + } + return "", nil +} + +func (c *Config) RequirePrototype(notation string) (string, error) { + t, ok := c.Types[notation] + if !ok { + // needed for optional types: don't require the prototype if the optional type is not defined + return "", nil + } + if t.Prototype == "" { + return "", fmt.Errorf("prototype for type %q is required but not specified", notation) + } + return "", nil +} + +func (c *Config) Type(notation string) string { + t, ok := c.Types[notation] + if !ok { + return "" + } + return t.Spec +} + +func (c *Config) Prototype(notation string) string { + t, ok := c.Types[notation] + if !ok { + return "" + } + return t.Prototype +} + +func (c *Config) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&c.Package, "package", c.Package, "destination package") + fs.Var(c, "import", "packages to import") + fs.Var(&c.Types, "type", "auxilliary type mappings in {notation}:{type-spec}:{prototype-expr} format") +} + +func NewConfig() *Config { + var ( + c = Config{ + Package: os.Getenv("GOPACKAGE"), + } + ) + return &c +} + +func Run(src, test *template.Template, args ...string) { + if len(args) < 1 { + panic(errors.New("expected at least one arg")) + } + var ( + c = NewConfig() + defaultOutput = "foo.go" + output string + ) + if c.Package != "" { + defaultOutput = c.Package + "_generated.go" + } + + fs := flag.NewFlagSet(args[0], flag.PanicOnError) + fs.StringVar(&output, "output", output, "path of the to-be-generated file") + c.AddFlags(fs) + + if err := fs.Parse(args[1:]); err != nil { + if err == flag.ErrHelp { + fs.PrintDefaults() + } + panic(err) + } + + c.Args = strings.Join(args[1:], " ") + + if c.Package == "" { + c.Package = "foo" + } + + if output == "" { + output = defaultOutput + } + + genmap := make(map[string]*template.Template) + if src != nil { + genmap[output] = src + } + if test != nil { + testOutput := output + "_test" + if strings.HasSuffix(output, ".go") { + testOutput = output[:len(output)-3] + "_test.go" + } + genmap[testOutput] = test + } + if len(genmap) == 0 { + panic(errors.New("neither src or test templates were provided")) + } + + Generate(genmap, c, func(err error) { panic(err) }) +} + +func Generate(items map[string]*template.Template, data interface{}, eh func(error)) { + for filename, t := range items { + func() { + f, err := os.Create(filename) + if err != nil { + eh(err) + return + } + defer f.Close() + log.Println("generating file", filename) + err = t.Execute(f, data) + if err != nil { + eh(err) + } + }() + } +} diff --git a/api/v1/lib/extras/gen/handlers.go b/api/v1/lib/extras/gen/handlers.go new file mode 100644 index 00000000..0890b72a --- /dev/null +++ b/api/v1/lib/extras/gen/handlers.go @@ -0,0 +1,104 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequireType "ET" -}} +type ( + // Handler is invoked upon the occurrence of some scheduler event that is generated + // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) + Handler interface { + HandleEvent(context.Context, {{.Type "E"}}) error + } + + // HandlerFunc is a functional adaptation of the Handler interface + HandlerFunc func(context.Context, {{.Type "E"}}) error + + // Handlers executes an event Handler according to the event's type + Handlers map[{{.Type "ET"}}]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[{{.Type "ET"}}]HandlerFunc +) + +// HandleEvent implements Handler for HandlerFunc +func (f HandlerFunc) HandleEvent(ctx context.Context, e {{.Type "E"}}) error { return f(ctx, e) } + +type noopHandler int + +func (_ noopHandler) HandleEvent(_ context.Context, _ {{.Type "E"}}) error { return nil } + +// NoopHandler is a Handler that does nothing and always returns nil +const NoopHandler = noopHandler(0) + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e {{.Type "E"}}) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e {{.Type "E"}}) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) +`)) diff --git a/api/v1/lib/extras/gen/rule_callers.go b/api/v1/lib/extras/gen/rule_callers.go new file mode 100644 index 00000000..e637f864 --- /dev/null +++ b/api/v1/lib/extras/gen/rule_callers.go @@ -0,0 +1,75 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequireType "C" -}} +{{.RequireType "CF" -}} +// Call returns a Rule that invokes the given Caller +func Call(caller {{.Type "C"}}) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c {{.Type "E"}}, _ mesos.Response, _ error, ch Chain) (context.Context, {{.Type "E"}}, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf {{.Type "CF"}}) Rule { + return Call({{.Type "C"}}(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller {{.Type "C"}}) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf {{.Type "CF"}}) Rule { + return r.Caller({{.Type "C"}}(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c {{.Type "E"}}) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c {{.Type "E"}}) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = {{.Type "C"}}(Rule(nil)) + _ = {{.Type "C"}}(Rules(nil)) +) +`)) diff --git a/api/v1/lib/extras/gen/rule_handlers.go b/api/v1/lib/extras/gen/rule_handlers.go new file mode 100644 index 00000000..23592364 --- /dev/null +++ b/api/v1/lib/extras/gen/rule_handlers.go @@ -0,0 +1,68 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(handlersTemplate, nil, os.Args...) +} + +var handlersTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequireType "H" -}} +{{.RequireType "HF" -}} +// Handle generates a rule that executes the given {{.Type "H"}}. +func Handle(h {{.Type "H"}}) Rule { + if h == nil { + return nil + } + return func(ctx context.Context, e {{.Type "E"}}, err error, chain Chain) (context.Context, {{.Type "E"}}, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h {{.Type "HF"}}) Rule { + return Handle({{.Type "H"}}(h)) +} + +// Handle returns a Rule that invokes the receiver, then the given {{.Type "H"}} +func (r Rule) Handle(h {{.Type "H"}}) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h {{.Type "HF"}}) Rule { + return r.Handle({{.Type "H"}}(h)) +} + +// HandleEvent implements {{.Type "H"}} for Rule +func (r Rule) HandleEvent(ctx context.Context, e {{.Type "E"}}) (err error) { + if r == nil { + return nil + } + _, _, err = r(ctx, e, nil, ChainIdentity) + return +} + +// HandleEvent implements {{.Type "H"}} for Rules +func (rs Rules) HandleEvent(ctx context.Context, e {{.Type "E"}}) error { + return Rule(rs.Eval).HandleEvent(ctx, e) +} +`)) diff --git a/api/v1/lib/extras/gen/rule_metrics.go b/api/v1/lib/extras/gen/rule_metrics.go new file mode 100644 index 00000000..dafc66e1 --- /dev/null +++ b/api/v1/lib/extras/gen/rule_metrics.go @@ -0,0 +1,143 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(metricsTemplate, metricsTestTemplate, os.Args...) +} + +var metricsTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" +{{if .Type "ET"}} + "strings" +{{end}} + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} + +// Labeler generates a set of strings that should be associated with metrics that are generated for the given event. +type Labeler func(ctx context.Context, e {{.Type "E"}}) []string + +{{if .Type "ET" -}} +var defaultLabels = func() map[{{.Type "ET"}}][]string { + m := make(map[{{.Type "ET"}}][]string) + for k, v := range {{.Type "ET"}}_name { + m[{{.Type "ET"}}(k)] = []string{strings.ToLower(v)} + } + return m +}() + +func defaultLabeler(ctx context.Context, e {{.Type "E"}}) []string { + return defaultLabels[e.GetType()] +} + +{{end -}} +// Metrics generates a Rule that invokes the given harness for each event, using the labels generated by the Labeler. +// Panics if harness {{if .Type "ET"}}or labeler {{end}}is nil. +func Metrics(harness metrics.Harness, labeler Labeler) Rule { + if harness == nil { + panic("harness is a required parameter") + } + {{if .Type "ET" -}} + if labeler == nil { + labeler = defaultLabeler + } + {{else -}} + if labeler == nil { + panic("labeler is a required parameter") + } + {{end -}} + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + labels := labeler(ctx, e) + harness(func() error { + ctx, e, {{.Ref "Z" "z," -}} err = ch(ctx, e, {{.Ref "Z" "z," -}} err) + return err + }, labels...) + return ctx, e, {{.Ref "Z" "z," -}} err + } +} +`)) + +var metricsTestTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequirePrototype "E" -}} +{{.RequirePrototype "Z" -}} +func TestMetrics(t *testing.T) { + var ( + i int + ctx = context.Background() + p = {{.Prototype "E"}} + a = errors.New("a") + h = func(f func() error, _ ...string) error { + i++ + return f() + } + r = Metrics(h, func(_ context.Context, _ {{.Type "E"}}) []string { return nil }) + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for ti, tc := range []struct { + ctx context.Context + e {{.Type "E"}} + {{if .Type "Z"}} + {{- .Arg "Z" "z "}} + {{end -}} + err error + }{ + {ctx, p, {{.Ref "Z" "zp," -}} a}, + {ctx, p, {{.Ref "Z" "zp," -}} nil}, + {{if .Type "Z"}} + {ctx, p, {{.Ref "Z" "nil," -}} a}, + {{end -}} + {ctx, nil, {{.Ref "Z" "zp," -}} a}, + } { + c, e, {{.Ref "Z" "z," -}} err := r.Eval(tc.ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, ChainIdentity) + if !reflect.DeepEqual(c, tc.ctx) { + t.Errorf("test case %d: expected context %q instead of %q", ti, tc.ctx, c) + } + if !reflect.DeepEqual(e, tc.e) { + t.Errorf("test case %d: expected event %q instead of %q", ti, tc.e, e) + } + {{if .Type "Z" -}} + if !reflect.DeepEqual(z, tc.z) { + t.Errorf("expected return object %q instead of %q", z, tc.z) + } + {{end -}} + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("test case %d: expected error %q instead of %q", ti, tc.err, err) + } + if y := ti + 1; y != i { + t.Errorf("test case %d: expected count %q instead of %q", ti, y, i) + } + } +} +`)) diff --git a/api/v1/lib/extras/gen/rules.go b/api/v1/lib/extras/gen/rules.go new file mode 100644 index 00000000..942f62a8 --- /dev/null +++ b/api/v1/lib/extras/gen/rules.go @@ -0,0 +1,1199 @@ +// +build ignore + +package main + +import ( + "os" + "text/template" +) + +func main() { + Run(rulesTemplate, rulesTestTemplate, os.Args...) +} + +var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +{{.RequireType "E" -}} +{{.RequirePrototype "E" -}} +{{.RequirePrototype "Z" -}}{{/* Z is an optional type, but if it's specified then it needs a prototype */ -}} +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error, Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) +) + +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ctx, e, {{.Ref "Z" "z," -}} err +} + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if r != nil { + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + return ch(ctx, e, {{.Ref "Z" "z," -}} err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(rs.Chain()(ctx, e, {{.Ref "Z" "z," -}} err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, {{.Type "E"}}, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return ChainIdentity + } + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return rs[0].Eval(ctx, e, {{.Ref "Z" "z," -}} err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +const ( + MsgNoErrors = "no errors" +) + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return MsgNoErrors + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + switch len(es) { + case 0: + return nil + case 1: + return es[0] + default: + return es + } +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + ruleInvoked := false + once.Do(func() { + ctx, e, {{.Ref "Z" "z," -}} err = r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, {{.Ref "Z" "z," -}} err = ch(ctx, e, {{.Ref "Z" "z," -}} err) + } + return ctx, e, {{.Ref "Z" "z," -}} err + } +} + +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + select { + case <-ctx.Done(): + return ctx, e, {{.Ref "Z" "z," -}} Error2(err, ctx.Err()) + default: + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + } +} + +type Overflow int + +const ( + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise +) + +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + return limit(r, acquireChan(p), over, otherwise) +} + +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { + if tokenCh == nil { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } + } + return false + } + } + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-tokenCh: + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } + case <-ctx.Done(): + } + return false + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { + if r == nil { + return nil + } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } + blocking := false + switch over { + case OverflowOtherwise: + case OverflowWait: + blocking = true + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } +} + +/* TODO(jdef) not sure that this is very useful, leaving out for now... + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { + if nthTime < 2 || r == nil { + return r + } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { + var ( + i = 1 // begin with the first event seen + m sync.Mutex + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: + m.Lock() + i-- + if i <= 0 { + i = nthTime + result = true + } + m.Unlock() + } + return + } +} + +*/ + +// Drop aborts the Chain and returns the (context.Context, {{.Type "E"}}, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, _ Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ChainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if err != nil { + return ctx, e, {{.Ref "Z" "z," -}} err + } + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + if err == nil { + // bypass remainder of chain + return ctx, e, {{.Ref "Z" "z," -}} err + } + return r.Eval(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} +`)) + +var rulesTestTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate {{.Args}} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" +{{range .Imports}} + {{ printf "%q" . -}} +{{end}} +) + +func prototype() {{.Type "E"}} { return {{.Prototype "E"}} } + +func counter(i *int) Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + *i++ + return ch(ctx, e, {{.Ref "Z" "z," -}} err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + t.Log("executing", name) + return r(ctx, e, {{.Ref "Z" "z," -}} err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error, ch Chain) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + return ch(ctx, e, {{.Ref "Z" "z," -}} Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e {{.Type "E"}}, {{.Arg "Z" "z," -}} err error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + *i++ + return ch(ctx, e, {{.Ref "Z" "z," -}} err) + } +} + +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ {{.Type "E"}}, {{.Arg "Z" "_," -}} _ error) (context.Context, {{.Type "E"}}, {{.Arg "Z" "," -}} error) { + panic(x) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) +{{if .Type "Z"}} + {{.Var "Z" "z0"}} +{{end}} + _, e, {{.Ref "Z" "_," -}} err := Rules{counterRule}.Eval(context.Background(), nil, {{.Ref "Z" "z0," -}} nil, ChainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + {{if .Type "Z" -}} + {{.Var "Z" "z0"}} + var zp = {{.Prototype "Z"}} + {{end -}} + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e {{.Type "E"}} + {{if .Type "Z"}} + {{- .Arg "Z" "z "}} + {{end -}} + err error + }{ + {nil, {{.Ref "Z" "z0," -}} nil}, + {nil, {{.Ref "Z" "z0," -}} a}, + {p, {{.Ref "Z" "z0," -}} nil}, + {p, {{.Ref "Z" "z0," -}} a}, +{{if .Type "Z"}} + {nil, {{.Ref "Z" "zp," -}} nil}, + {nil, {{.Ref "Z" "zp," -}} a}, + {p, {{.Ref "Z" "zp," -}} nil}, + {p, {{.Ref "Z" "zp," -}} a}, +{{end}} } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, {{.Ref "Z" "zz," -}} err = rule(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, ChainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + {{if .Type "Z" -}} + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + {{end -}} + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, {{.Ref "Z" "z," -}} err + _, e, {{.Ref "Z" "zz," -}} err = Rules{}.Eval(ctx, tc.e, {{.Ref "Z" "tc.z," -}} tc.err, ChainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + {{if .Type "Z" -}} + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + {{end -}} + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + {{end -}} + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, {{.Ref "Z" "zz," -}} err := tc.r(ctx, p, {{.Ref "Z" "zp," -}} tc.initialError, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, {{.Ref "Z" "zz," -}} err := r2(ctx, p, {{.Ref "Z" "zp," -}} a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, {{.Ref "Z" "zz," -}} err = r3(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } + var ( + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() + ch4 = make(chan struct{}) // non-nil, closed + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() + ) + {{if .Type "Z" -}} + var zp = {{.Prototype "Z"}} + {{end -}} + close(ch4) + for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 + ctx context.Context + ch <-chan struct{} + over Overflow + otherwise Rule + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + } { + var ( + i, j int + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain + ) + for k, r := range []Rule{r1, r2} { + // execute each rule twice + for x := 0; x < 2; x++ { + _, e, {{.Ref "Z" "zz," -}} err := r(tc.ctx, p, {{.Ref "Z" "zp," -}} nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) + } + } + } + } + // test blocking capability via rateLimit + blocked := false + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, {{.Ref "Z" "zz," -}} err := r(ctx, p, {{.Ref "Z" "zp," -}} nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + {{if .Type "Z" -}} + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + {{end -}} + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, {{.Ref "Z" "zp," -}} nil, ChainIdentity) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } +} +`)) diff --git a/api/v1/lib/extras/latch/latch.go b/api/v1/lib/extras/latch/latch.go new file mode 100644 index 00000000..35f4d80b --- /dev/null +++ b/api/v1/lib/extras/latch/latch.go @@ -0,0 +1,48 @@ +package latch + +import "sync/atomic" + +// Interface funcs are safe to invoke concurrently. +type Interface interface { + // Done returns a chan that blocks until Close is called. It never returns data. + Done() <-chan struct{} + // Close closes the latch; all future calls to Closed return true. Safe to invoke multiple times. + Close() + // Closed returns false while the latch is "open" and true after it has been closed via Close. + Closed() bool +} + +type L struct { + value int32 + line chan struct{} +} + +// New returns a new "open" latch such that Closed returns false until Close is invoked. +func New() Interface { + return new(L).Reset() +} + +func (l *L) Done() <-chan struct{} { return l.line } + +// Close may panic for an uninitialized L +func (l *L) Close() { + if atomic.AddInt32(&l.value, 1) == 1 { + close(l.line) + } + <-l.line // concurrent calls to Close block until the latch is actually closed +} + +func (l *L) Closed() (result bool) { + select { + case <-l.line: + result = true + default: + } + return +} + +// Reset clears the state of the latch, not safe to execute concurrently with other L methods. +func (l *L) Reset() *L { + l.line, l.value = make(chan struct{}), 0 + return l +} diff --git a/api/v1/lib/extras/latch/latch_test.go b/api/v1/lib/extras/latch/latch_test.go new file mode 100644 index 00000000..caab35a5 --- /dev/null +++ b/api/v1/lib/extras/latch/latch_test.go @@ -0,0 +1,33 @@ +package latch_test + +import ( + "testing" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/latch" +) + +func TestInterface(t *testing.T) { + l := New() + if l == nil { + t.Fatalf("expected a valid latch, not nil") + } + if l.Closed() { + t.Fatalf("expected new latch to be non-closed") + } + select { + case <-l.Done(): + t.Fatalf("Done chan unexpectedly closed for a new latch") + default: + } + for i := 0; i < 2; i++ { + l.Close() // multiple calls to close should not panic + } + if !l.Closed() { + t.Fatalf("expected closed latch") + } + select { + case <-l.Done(): + default: + t.Fatalf("Done chan unexpectedly non-closed for a closed latch") + } +} diff --git a/api/v1/lib/extras/resources/aggregate.go b/api/v1/lib/extras/resources/aggregate.go new file mode 100644 index 00000000..ff52c0e0 --- /dev/null +++ b/api/v1/lib/extras/resources/aggregate.go @@ -0,0 +1,125 @@ +package resources + +import ( + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/resourcefilters" +) + +func SumScalars(rf resourcefilters.Filter, resources ...mesos.Resource) *mesos.Value_Scalar { + predicate := resourcefilters.Filters{rf, resourcefilters.Scalar} + var x *mesos.Value_Scalar + for i := range resources { + if !predicate.Accepts(&resources[i]) { + continue + } + x = x.Add(resources[i].GetScalar()) + } + return x +} + +func SumRanges(rf resourcefilters.Filter, resources ...mesos.Resource) *mesos.Value_Ranges { + predicate := resourcefilters.Filters{rf, resourcefilters.Range} + var x *mesos.Value_Ranges + for i := range resources { + if !predicate.Accepts(&resources[i]) { + continue + } + x = x.Add(resources[i].GetRanges()) + } + return x +} + +func SumSets(rf resourcefilters.Filter, resources ...mesos.Resource) *mesos.Value_Set { + predicate := resourcefilters.Filters{rf, resourcefilters.Set} + var x *mesos.Value_Set + for i := range resources { + if !predicate.Accepts(&resources[i]) { + continue + } + x = x.Add(resources[i].GetSet()) + } + return x +} + +func CPUs(resources ...mesos.Resource) (float64, bool) { + v := SumScalars(resourcefilters.Named(ResourceNameCPUs), resources...) + if v != nil { + return v.Value, true + } + return 0, false +} + +func GPUs(resources ...mesos.Resource) (float64, bool) { + v := SumScalars(resourcefilters.Named(ResourceNameGPUs), resources...) + if v != nil { + return v.Value, true + } + return 0, false +} + +func Memory(resources ...mesos.Resource) (uint64, bool) { + v := SumScalars(resourcefilters.Named(ResourceNameMem), resources...) + if v != nil { + return uint64(v.Value), true + } + return 0, false +} + +func Disk(resources ...mesos.Resource) (uint64, bool) { + v := SumScalars(resourcefilters.Named(ResourceNameDisk), resources...) + if v != nil { + return uint64(v.Value), true + } + return 0, false +} + +func Ports(resources ...mesos.Resource) (mesos.Ranges, bool) { + v := SumRanges(resourcefilters.Named(ResourceNamePorts), resources...) + if v != nil { + return mesos.Ranges(v.Range), true + } + return nil, false +} + +func Types(resources ...mesos.Resource) map[string]mesos.Value_Type { + m := map[string]mesos.Value_Type{} + for i := range resources { + m[resources[i].GetName()] = resources[i].GetType() + } + return m +} + +func Names(resources ...mesos.Resource) (names []string) { + m := map[string]struct{}{} + for i := range resources { + n := resources[i].GetName() + if _, ok := m[n]; !ok { + m[n] = struct{}{} + names = append(names, n) + } + } + return +} + +func SumAndCompare(expected mesos.Resources, resources ...mesos.Resource) bool { + // from: https://github.com/apache/mesos/blob/master/src/common/resources.cpp + // This is a sanity check to ensure the amount of each type of + // resource does not change. + // TODO(jieyu): Currently, we only check known resource types like + // cpus, mem, disk, ports, etc. We should generalize this. + var ( + c1, c2 = CPUs(expected...) + m1, m2 = Memory(expected...) + d1, d2 = Disk(expected...) + p1, p2 = Ports(expected...) + + c3, c4 = CPUs(resources...) + m3, m4 = Memory(resources...) + d3, d4 = Disk(resources...) + p3, p4 = Ports(resources...) + ) + return c1 == c3 && c2 == c4 && + m1 == m3 && m2 == m4 && + d1 == d3 && d2 == d4 && + p1.Equivalent(p3) && p2 == p4 +} diff --git a/api/v1/lib/extras/resources/aggregate_test.go b/api/v1/lib/extras/resources/aggregate_test.go new file mode 100644 index 00000000..432c7e76 --- /dev/null +++ b/api/v1/lib/extras/resources/aggregate_test.go @@ -0,0 +1,43 @@ +package resources_test + +import ( + "reflect" + "sort" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + rez "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + . "github.com/mesos/mesos-go/api/v1/lib/resourcetest" +) + +func TestResources_Types(t *testing.T) { + rs := Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role1")), + Resource(Name("cpus"), ValueScalar(4)), + Resource(Name("ports"), ValueRange(Span(1, 10)), Role("role1")), + Resource(Name("ports"), ValueRange(Span(11, 20))), + ) + types := rez.Types(rs...) + expected := map[string]mesos.Value_Type{ + "cpus": mesos.SCALAR, + "ports": mesos.RANGES, + } + if !reflect.DeepEqual(types, expected) { + t.Fatalf("expected %v instead of %v", expected, types) + } +} + +func TestResources_Names(t *testing.T) { + rs := Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role1")), + Resource(Name("cpus"), ValueScalar(4)), + Resource(Name("mem"), ValueScalar(10), Role("role1")), + Resource(Name("mem"), ValueScalar(10)), + ) + names := rez.Names(rs...) + sort.Strings(names) + expected := []string{"cpus", "mem"} + if !reflect.DeepEqual(names, expected) { + t.Fatalf("expected %v instead of %v", expected, names) + } +} diff --git a/api/v1/lib/extras/resources/builders.go b/api/v1/lib/extras/resources/builders.go new file mode 100644 index 00000000..f2d4b059 --- /dev/null +++ b/api/v1/lib/extras/resources/builders.go @@ -0,0 +1,80 @@ +package resources + +import ( + "github.com/mesos/mesos-go/api/v1/lib" +) + +type ( + // Builder simplifies construction of Resource objects + Builder struct{ mesos.Resource } + // RangeBuilder simplifies construction of Range objects + RangeBuilder struct{ mesos.Ranges } +) + +func NewCPUs(value float64) *Builder { + return Build().Name(ResourceNameCPUs).Scalar(value) +} + +func NewMemory(value float64) *Builder { + return Build().Name(ResourceNameMem).Scalar(value) +} + +func NewDisk(value float64) *Builder { + return Build().Name(ResourceNameDisk).Scalar(value) +} + +func NewGPUs(value uint) *Builder { + return Build().Name(ResourceNameGPUs).Scalar(float64(value)) +} + +func BuildRanges() *RangeBuilder { + return &RangeBuilder{Ranges: mesos.Ranges(nil)} +} + +// Span is a functional option for Ranges, defines the begin and end points of a +// continuous span within a range +func (rb *RangeBuilder) Span(bp, ep uint64) *RangeBuilder { + rb.Ranges = append(rb.Ranges, mesos.Value_Range{Begin: bp, End: ep}) + return rb +} + +func Build() *Builder { + return &Builder{} +} +func (rb *Builder) Name(name string) *Builder { + rb.Resource.Name = name + return rb +} +func (rb *Builder) Role(role string) *Builder { + rb.Resource.Role = &role + return rb +} +func (rb *Builder) Scalar(x float64) *Builder { + rb.Resource.Type = mesos.SCALAR.Enum() + rb.Resource.Scalar = &mesos.Value_Scalar{Value: x} + return rb +} +func (rb *Builder) Set(x ...string) *Builder { + rb.Resource.Type = mesos.SET.Enum() + rb.Resource.Set = &mesos.Value_Set{Item: x} + return rb +} +func (rb *Builder) Ranges(rs mesos.Ranges) *Builder { + rb.Resource.Type = mesos.RANGES.Enum() + rb.Resource.Ranges = rb.Resource.Ranges.Add(&mesos.Value_Ranges{Range: rs}) + return rb +} +func (rb *Builder) Disk(persistenceID, containerPath string) *Builder { + rb.Resource.Disk = &mesos.Resource_DiskInfo{} + if containerPath != "" { + rb.Resource.Disk.Volume = &mesos.Volume{ContainerPath: containerPath} + } + if persistenceID != "" { + rb.Resource.Disk.Persistence = &mesos.Resource_DiskInfo_Persistence{ID: persistenceID} + } + return rb +} +func (rb *Builder) Revocable() *Builder { + rb.Resource.Revocable = &mesos.Resource_RevocableInfo{} + return rb +} diff --git a/api/v1/lib/extras/resources/find.go b/api/v1/lib/extras/resources/find.go new file mode 100644 index 00000000..1f724699 --- /dev/null +++ b/api/v1/lib/extras/resources/find.go @@ -0,0 +1,53 @@ +package resources + +import ( + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/resourcefilters" +) + +func Find(wants mesos.Resources, from ...mesos.Resource) (total mesos.Resources) { + for i := range wants { + found := find(wants[i], from...) + + // each want *must* be found + if len(found) == 0 { + return nil + } + + total.Add(found...) + } + return total +} + +func find(want mesos.Resource, from ...mesos.Resource) mesos.Resources { + var ( + total = mesos.Resources(from).Clone() + remaining = mesos.Resources{want}.Flatten() + found mesos.Resources + predicates = resourcefilters.Filters{ + resourcefilters.ReservedByRole(want.GetRole()), + resourcefilters.Unreserved, + resourcefilters.Any, + } + ) + for _, predicate := range predicates { + filtered := resourcefilters.Select(predicate, total...) + for i := range filtered { + // need to flatten to ignore the roles in ContainsAll() + flattened := mesos.Resources{filtered[i]}.Flatten() + if flattened.ContainsAll(remaining) { + // want has been found, return the result + return found.Add(remaining.Flatten( + mesos.RoleName(filtered[i].GetRole()).Assign(), + filtered[i].Reservation.Assign())...) + } + if remaining.ContainsAll(flattened) { + found.Add1(filtered[i]) + total.Subtract1(filtered[i]) + remaining.Subtract(flattened...) + break + } + } + } + return nil +} diff --git a/api/v1/lib/extras/resources/find_test.go b/api/v1/lib/extras/resources/find_test.go new file mode 100644 index 00000000..ffaa8277 --- /dev/null +++ b/api/v1/lib/extras/resources/find_test.go @@ -0,0 +1,89 @@ +package resources_test + +import ( + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + rez "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + . "github.com/mesos/mesos-go/api/v1/lib/resourcetest" +) + +func TestResources_Find(t *testing.T) { + for i, tc := range []struct { + r1, targets, wants mesos.Resources + }{ + {nil, nil, nil}, + { + r1: Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role1")), + Resource(Name("mem"), ValueScalar(10), Role("role1")), + Resource(Name("cpus"), ValueScalar(4), Role("*")), + Resource(Name("mem"), ValueScalar(20), Role("*")), + ), + targets: Resources( + Resource(Name("cpus"), ValueScalar(3), Role("role1")), + Resource(Name("mem"), ValueScalar(15), Role("role1")), + ), + wants: Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role1")), + Resource(Name("mem"), ValueScalar(10), Role("role1")), + Resource(Name("cpus"), ValueScalar(1), Role("*")), + Resource(Name("mem"), ValueScalar(5), Role("*")), + ), + }, + { + r1: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("mem"), ValueScalar(5), Role("role1")), + Resource(Name("cpus"), ValueScalar(2), Role("role2")), + Resource(Name("mem"), ValueScalar(8), Role("role2")), + Resource(Name("cpus"), ValueScalar(1), Role("*")), + Resource(Name("mem"), ValueScalar(7), Role("*")), + ), + targets: Resources( + Resource(Name("cpus"), ValueScalar(3), Role("role1")), + Resource(Name("mem"), ValueScalar(15), Role("role1")), + ), + wants: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("mem"), ValueScalar(5), Role("role1")), + Resource(Name("cpus"), ValueScalar(1), Role("*")), + Resource(Name("mem"), ValueScalar(7), Role("*")), + Resource(Name("cpus"), ValueScalar(1), Role("role2")), + Resource(Name("mem"), ValueScalar(3), Role("role2")), + ), + }, + { + r1: Resources( + Resource(Name("cpus"), ValueScalar(5), Role("role1")), + Resource(Name("mem"), ValueScalar(5), Role("role1")), + Resource(Name("cpus"), ValueScalar(5), Role("*")), + Resource(Name("mem"), ValueScalar(5), Role("*")), + ), + targets: Resources( + Resource(Name("cpus"), ValueScalar(6)), + Resource(Name("mem"), ValueScalar(6)), + ), + wants: Resources( + Resource(Name("cpus"), ValueScalar(5), Role("*")), + Resource(Name("mem"), ValueScalar(5), Role("*")), + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("mem"), ValueScalar(1), Role("role1")), + ), + }, + { + r1: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("mem"), ValueScalar(1), Role("role1")), + ), + targets: Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role1")), + Resource(Name("mem"), ValueScalar(2), Role("role1")), + ), + wants: nil, + }, + } { + r := rez.Find(tc.targets, tc.r1...) + Expect(t, r.Equivalent(tc.wants), "test case %d failed: expected %+v instead of %+v", i, tc.wants, r) + } +} diff --git a/api/v1/lib/extras/resources/names.go b/api/v1/lib/extras/resources/names.go new file mode 100644 index 00000000..bf5dd9da --- /dev/null +++ b/api/v1/lib/extras/resources/names.go @@ -0,0 +1,9 @@ +package resources + +const ( + ResourceNameCPUs = "cpus" + ResourceNameDisk = "disk" + ResourceNameGPUs = "gpus" + ResourceNameMem = "mem" + ResourceNamePorts = "ports" +) diff --git a/api/v1/lib/extras/scheduler/callrules/callers_generated.go b/api/v1/lib/extras/scheduler/callrules/callers_generated.go new file mode 100644 index 00000000..f016e948 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callers_generated.go @@ -0,0 +1,58 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/calls -type E:*scheduler.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" +) + +// Call returns a Rule that invokes the given Caller +func Call(caller calls.Caller) Rule { + if caller == nil { + return nil + } + return func(ctx context.Context, c *scheduler.Call, _ mesos.Response, _ error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + resp, err := caller.Call(ctx, c) + return ch(ctx, c, resp, err) + } +} + +// CallF returns a Rule that invokes the given CallerFunc +func CallF(cf calls.CallerFunc) Rule { + return Call(calls.Caller(cf)) +} + +// Caller returns a Rule that invokes the receiver and then calls the given Caller +func (r Rule) Caller(caller calls.Caller) Rule { + return Rules{r, Call(caller)}.Eval +} + +// CallerF returns a Rule that invokes the receiver and then calls the given CallerFunc +func (r Rule) CallerF(cf calls.CallerFunc) Rule { + return r.Caller(calls.Caller(cf)) +} + +// Call implements the Caller interface for Rule +func (r Rule) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + if r == nil { + return nil, nil + } + _, _, resp, err := r(ctx, c, nil, nil, ChainIdentity) + return resp, err +} + +// Call implements the Caller interface for Rules +func (rs Rules) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + return Rule(rs.Eval).Call(ctx, c) +} + +var ( + _ = calls.Caller(Rule(nil)) + _ = calls.Caller(Rules(nil)) +) diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go new file mode 100644 index 00000000..b50d5070 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated.go @@ -0,0 +1,400 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *scheduler.Call, mesos.Response, error, Chain) (context.Context, *scheduler.Call, mesos.Response, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *scheduler.Call, mesos.Response, error, Chain) (context.Context, *scheduler.Call, mesos.Response, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *scheduler.Call, mesos.Response, error) (context.Context, *scheduler.Call, mesos.Response, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) +) + +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + return ctx, e, z, err +} + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if r != nil { + return r(ctx, e, z, err, ch) + } + return ch(ctx, e, z, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(rs.Chain()(ctx, e, z, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *scheduler.Call, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return ChainIdentity + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + return rs[0].Eval(ctx, e, z, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +const ( + MsgNoErrors = "no errors" +) + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return MsgNoErrors + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + switch len(es) { + case 0: + return nil + case 1: + return es[0] + default: + return es + } +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, z, err = r(ctx, e, z, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, z, err = ch(ctx, e, z, err) + } + return ctx, e, z, err + } +} + +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + select { + case <-ctx.Done(): + return ctx, e, z, Error2(err, ctx.Err()) + default: + return r(ctx, e, z, err, ch) + } + } +} + +type Overflow int + +const ( + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise +) + +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + return limit(r, acquireChan(p), over, otherwise) +} + +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { + if tokenCh == nil { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } + } + return false + } + } + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-tokenCh: + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } + case <-ctx.Done(): + } + return false + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { + if r == nil { + return nil + } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } + blocking := false + switch over { + case OverflowOtherwise: + case OverflowWait: + blocking = true + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, z, err, ch) + } + return r(ctx, e, z, err, ch) + } +} + +/* TODO(jdef) not sure that this is very useful, leaving out for now... + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { + if nthTime < 2 || r == nil { + return r + } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { + var ( + i = 1 // begin with the first event seen + m sync.Mutex + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: + m.Lock() + i-- + if i <= 0 { + i = nthTime + result = true + } + m.Unlock() + } + return + } +} + +*/ + +// Drop aborts the Chain and returns the (context.Context, *scheduler.Call, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Call, mesos.Response, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, _ Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return r.Eval(ctx, e, z, err, ChainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if err != nil { + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, z, err + } + return r.Eval(ctx, e, z, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go new file mode 100644 index 00000000..2bcf3450 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/callrules_generated_test.go @@ -0,0 +1,719 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func prototype() *scheduler.Call { return &scheduler.Call{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + t.Log("executing", name) + return r(ctx, e, z, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + return ch(ctx, e, z, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error) (context.Context, *scheduler.Call, mesos.Response, error) { + *i++ + return ch(ctx, e, z, err) + } +} + +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *scheduler.Call, _ mesos.Response, _ error) (context.Context, *scheduler.Call, mesos.Response, error) { + panic(x) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + var z0 mesos.Response + + _, e, _, err := Rules{counterRule}.Eval(context.Background(), nil, z0, nil, ChainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + var z0 mesos.Response + var zp = &mesos.ResponseWrapper{} + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *scheduler.Call + z mesos.Response + err error + }{ + {nil, z0, nil}, + {nil, z0, a}, + {p, z0, nil}, + {p, z0, a}, + + {nil, zp, nil}, + {nil, zp, a}, + {p, zp, nil}, + {p, zp, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, zz, err = rule(ctx, tc.e, tc.z, tc.err, ChainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, z, err + _, e, zz, err = Rules{}.Eval(ctx, tc.e, tc.z, tc.err, ChainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if zz != tc.z { + t.Errorf("expected return object %q instead of %q", tc.z, zz) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("test case %d failed: expected return object %q instead of %q", ti, zp, zz) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + var zp = &mesos.ResponseWrapper{} + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, zz, err := tc.r(ctx, p, zp, tc.initialError, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, zz, err := r2(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, zz, err := r2(ctx, p, zp, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, zz, err = r3(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + var zp = &mesos.ResponseWrapper{} + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + var zp = &mesos.ResponseWrapper{} + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, zz, err := r(ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } + var ( + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() + ch4 = make(chan struct{}) // non-nil, closed + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() + ) + var zp = &mesos.ResponseWrapper{} + close(ch4) + for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 + ctx context.Context + ch <-chan struct{} + over Overflow + otherwise Rule + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + } { + var ( + i, j int + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain + ) + for k, r := range []Rule{r1, r2} { + // execute each rule twice + for x := 0; x < 2; x++ { + _, e, zz, err := r(tc.ctx, p, zp, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) + } + } + } + } + // test blocking capability via rateLimit + blocked := false + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, zz, err := r(ctx, p, zp, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if zz != zp { + t.Errorf("expected return object %q instead of %q", zp, zz) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, zp, nil, ChainIdentity) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } +} diff --git a/api/v1/lib/extras/scheduler/callrules/gen.go b/api/v1/lib/extras/scheduler/callrules/gen.go new file mode 100644 index 00000000..da4fde28 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/gen.go @@ -0,0 +1,7 @@ +package callrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type Z:mesos.Response:&mesos.ResponseWrapper{} + +//go:generate go run ../../gen/rule_callers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/calls -type E:*scheduler.Call -type C:calls.Caller -type CF:calls.CallerFunc -output callers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type ET:scheduler.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go diff --git a/api/v1/lib/extras/scheduler/callrules/helpers.go b/api/v1/lib/extras/scheduler/callrules/helpers.go new file mode 100644 index 00000000..3826262e --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/helpers.go @@ -0,0 +1,25 @@ +package callrules + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// WithFrameworkID returns a Rule that injects a framework ID to outgoing calls, with the following exceptions: +// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) +// - calls are not modified when the detected framework ID is "" +func WithFrameworkID(frameworkID func() string) Rule { + return func(ctx context.Context, c *scheduler.Call, r mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + // never overwrite framework ID for subscribe calls; the scheduler must do that part + if c.GetType() != scheduler.Call_SUBSCRIBE { + if fid := frameworkID(); fid != "" { + c2 := *c + c2.FrameworkID = &mesos.FrameworkID{Value: fid} + c = &c2 + } + } + return ch(ctx, c, r, err) + } +} diff --git a/api/v1/lib/extras/scheduler/callrules/metrics_generated.go b/api/v1/lib/extras/scheduler/callrules/metrics_generated.go new file mode 100644 index 00000000..df31d228 --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/metrics_generated.go @@ -0,0 +1,49 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type ET:scheduler.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// Labeler generates a set of strings that should be associated with metrics that are generated for the given event. +type Labeler func(ctx context.Context, e *scheduler.Call) []string + +var defaultLabels = func() map[scheduler.Call_Type][]string { + m := make(map[scheduler.Call_Type][]string) + for k, v := range scheduler.Call_Type_name { + m[scheduler.Call_Type(k)] = []string{strings.ToLower(v)} + } + return m +}() + +func defaultLabeler(ctx context.Context, e *scheduler.Call) []string { + return defaultLabels[e.GetType()] +} + +// Metrics generates a Rule that invokes the given harness for each event, using the labels generated by the Labeler. +// Panics if harness or labeler is nil. +func Metrics(harness metrics.Harness, labeler Labeler) Rule { + if harness == nil { + panic("harness is a required parameter") + } + if labeler == nil { + labeler = defaultLabeler + } + return func(ctx context.Context, e *scheduler.Call, z mesos.Response, err error, ch Chain) (context.Context, *scheduler.Call, mesos.Response, error) { + labels := labeler(ctx, e) + harness(func() error { + ctx, e, z, err = ch(ctx, e, z, err) + return err + }, labels...) + return ctx, e, z, err + } +} diff --git a/api/v1/lib/extras/scheduler/callrules/metrics_generated_test.go b/api/v1/lib/extras/scheduler/callrules/metrics_generated_test.go new file mode 100644 index 00000000..13ec903d --- /dev/null +++ b/api/v1/lib/extras/scheduler/callrules/metrics_generated_test.go @@ -0,0 +1,58 @@ +package callrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Call:&scheduler.Call{} -type ET:scheduler.Call_Type -type Z:mesos.Response:&mesos.ResponseWrapper{} -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func TestMetrics(t *testing.T) { + var ( + i int + ctx = context.Background() + p = &scheduler.Call{} + a = errors.New("a") + h = func(f func() error, _ ...string) error { + i++ + return f() + } + r = Metrics(h, func(_ context.Context, _ *scheduler.Call) []string { return nil }) + ) + var zp = &mesos.ResponseWrapper{} + for ti, tc := range []struct { + ctx context.Context + e *scheduler.Call + z mesos.Response + err error + }{ + {ctx, p, zp, a}, + {ctx, p, zp, nil}, + + {ctx, p, nil, a}, + {ctx, nil, zp, a}, + } { + c, e, z, err := r.Eval(tc.ctx, tc.e, tc.z, tc.err, ChainIdentity) + if !reflect.DeepEqual(c, tc.ctx) { + t.Errorf("test case %d: expected context %q instead of %q", ti, tc.ctx, c) + } + if !reflect.DeepEqual(e, tc.e) { + t.Errorf("test case %d: expected event %q instead of %q", ti, tc.e, e) + } + if !reflect.DeepEqual(z, tc.z) { + t.Errorf("expected return object %q instead of %q", z, tc.z) + } + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("test case %d: expected error %q instead of %q", ti, tc.err, err) + } + if y := ti + 1; y != i { + t.Errorf("test case %d: expected count %q instead of %q", ti, y, i) + } + } +} diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index 540c1662..8edc409b 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -1,6 +1,8 @@ package controller import ( + "context" + "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/encoding" "github.com/mesos/mesos-go/api/v1/lib/scheduler" @@ -9,140 +11,142 @@ import ( ) type ( - Context interface { - // Done returns true when the controller should exit - Done() bool - - // FrameworkID returns the current Mesos-assigned framework ID. Frameworks are expected to - // track this ID (that comes from Mesos, in a SUBSCRIBED event). - FrameworkID() string + // Option modifies a Config, returns an Option that acts as an "undo" + Option func(*Config) Option - // Error is an error handler that is invoked at the end of every subscription cycle; the given - // error may be nil (if no errors occurred). - Error(error) + // Config is an opaque controller configuration. Properties are configured by applying Option funcs. + Config struct { + frameworkIDFunc func() string + handler events.Handler + registrationTokens <-chan struct{} + subscriptionTerminated func(error) } +) - ContextAdapter struct { - // FrameworkIDFunc is optional; nil tells the controller to always register as a new framework - // for each subscription attempt. - FrameworkIDFunc func() string - - // Done is optional; nil equates to a func that always returns false - DoneFunc func() bool - - // ErrorFunc is optional; if nil then errors are swallowed - ErrorFunc func(error) +// WithEventHandler sets the consumer of scheduler events. The controller's internal event processing +// loop is aborted if a Handler returns a non-nil error, after which the controller may attempt +// to re-register (subscribe) with Mesos. +func WithEventHandler(handler events.Handler) Option { + return func(c *Config) Option { + old := c.handler + c.handler = handler + return WithEventHandler(old) } +} - Config struct { - Context Context // Context is required - Framework *mesos.FrameworkInfo // FrameworkInfo is required - Caller calls.Caller // Caller is required - - // Handler (optional) processes scheduler events. The controller's internal event processing - // loop is aborted if a Handler returns a non-nil error, after which the controller may attempt - // to re-register (subscribe) with Mesos. - Handler events.Handler - - // RegistrationTokens (optional) limits the rate at which a framework (re)registers with Mesos. - // The returned chan should either be non-blocking (nil/closed), or should yield a struct{} in - // order to allow the framework registration process to continue. May be nil. - RegistrationTokens <-chan struct{} +// WithFrameworkID sets a fetcher for the current Mesos-assigned framework ID. Frameworks are expected to +// track this ID (that comes from Mesos, in a SUBSCRIBED event). +// frameworkIDFunc is optional; nil tells the controller to always register as a new framework +// for each subscription attempt. +func WithFrameworkID(frameworkIDFunc func() string) Option { + return func(c *Config) Option { + old := c.frameworkIDFunc + c.frameworkIDFunc = frameworkIDFunc + return WithFrameworkID(old) } +} - Controller interface { - // Run executes the controller using the given Config - Run(Config) error +// WithSubscriptionTerminated sets a handler that is invoked at the end of every subscription cycle; the +// given error may be nil if no error occurred. subscriptionTerminated is optional; if nil then errors are +// swallowed. +func WithSubscriptionTerminated(handler func(error)) Option { + return func(c *Config) Option { + old := c.subscriptionTerminated + c.subscriptionTerminated = handler + return WithSubscriptionTerminated(old) } +} - // ControllerFunc is a functional adaptation of a Controller - ControllerFunc func(Config) error - - controllerImpl int -) +// WithRegistrationTokens limits the rate at which a framework (re)registers with Mesos. +// A non-nil chan should yield a struct{} in order to allow the framework registration process to continue. +// When nil, there is no backoff delay between re-subscription attempts. +// A closed chan disables re-registration and terminates the Run control loop. +func WithRegistrationTokens(registrationTokens <-chan struct{}) Option { + return func(c *Config) Option { + old := c.registrationTokens + c.registrationTokens = registrationTokens + return WithRegistrationTokens(old) + } +} -// Run implements Controller for ControllerFunc -func (cf ControllerFunc) Run(config Config) error { return cf(config) } +func (c *Config) tryFrameworkID() (result string) { + if c.frameworkIDFunc != nil { + result = c.frameworkIDFunc() + } + return +} -func New() Controller { - return new(controllerImpl) +func isDone(ctx context.Context) (result bool) { + select { + case <-ctx.Done(): + return true + default: + return false + } } // Run executes a control loop that registers a framework with Mesos and processes the scheduler events -// that flow through the subscription. Upon disconnection, if the given Context reports !Done() then the -// controller will attempt to re-register the framework and continue processing events. -func (_ *controllerImpl) Run(config Config) (lastErr error) { - subscribe := calls.Subscribe(config.Framework) - for !config.Context.Done() { - frameworkID := config.Context.FrameworkID() - if config.Framework.GetFailoverTimeout() > 0 && frameworkID != "" { +// that flow through the subscription. Upon disconnection, if the current configuration reports "not done" +// then the controller will attempt to re-register the framework and continue processing events. +func Run(ctx context.Context, framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) (lastErr error) { + var config Config + for _, opt := range options { + if opt != nil { + opt(&config) + } + } + if config.handler == nil { + config.handler = DefaultHandler + } + subscribe := calls.Subscribe(framework) + for !isDone(ctx) { + frameworkID := config.tryFrameworkID() + if framework.GetFailoverTimeout() > 0 && frameworkID != "" { subscribe.With(calls.SubscribeTo(frameworkID)) } - <-config.RegistrationTokens - resp, err := config.Caller.Call(subscribe) - lastErr = processSubscription(config, resp, err) - config.Context.Error(lastErr) + if config.registrationTokens != nil { + select { + case _, ok := <-config.registrationTokens: + if !ok { + // re-registration canceled, exit Run loop + return + } + case <-ctx.Done(): + return ctx.Err() + } + } + resp, err := caller.Call(ctx, subscribe) + lastErr = processSubscription(ctx, config, resp, err) + if config.subscriptionTerminated != nil { + config.subscriptionTerminated(lastErr) + } } return } -func processSubscription(config Config, resp mesos.Response, err error) error { +func processSubscription(ctx context.Context, config Config, resp mesos.Response, err error) error { if resp != nil { defer resp.Close() } if err == nil { - err = eventLoop(config, resp) + err = eventLoop(ctx, config, resp) } return err } // eventLoop returns the framework ID received by mesos (if any); callers should check for a // framework ID regardless of whether error != nil. -func eventLoop(config Config, eventDecoder encoding.Decoder) (err error) { - h := config.Handler - if h == nil { - h = events.HandlerFunc(DefaultHandler) - } - for err == nil && !config.Context.Done() { +func eventLoop(ctx context.Context, config Config, eventDecoder encoding.Decoder) (err error) { + for err == nil && !isDone(ctx) { var e scheduler.Event if err = eventDecoder.Decode(&e); err == nil { - err = h.HandleEvent(&e) + err = config.handler.HandleEvent(ctx, &e) } } return err } -var _ = Context(&ContextAdapter{}) // ContextAdapter implements Context - -func (ca *ContextAdapter) Done() bool { - return ca.DoneFunc != nil && ca.DoneFunc() -} -func (ca *ContextAdapter) FrameworkID() (id string) { - if ca.FrameworkIDFunc != nil { - id = ca.FrameworkIDFunc() - } - return -} -func (ca *ContextAdapter) Error(err error) { - if ca.ErrorFunc != nil { - ca.ErrorFunc(err) - } -} - -// ErrEvent errors are generated by the DefaultHandler upon receiving an ERROR event from Mesos. -type ErrEvent string - -func (e ErrEvent) Error() string { - return string(e) -} - -// DefaultHandler provides the minimum implementation required for correct controller behavior. -func DefaultHandler(e *scheduler.Event) (err error) { - if e.GetType() == scheduler.Event_ERROR { - // it's recommended that we abort and re-try subscribing; returning an - // error here will cause the event loop to terminate and the connection - // will be reset. - err = ErrEvent(e.GetError().GetMessage()) - } - return -} +// DefaultHandler is invoked when no other handlers have been defined for the controller. +// The current implementation does nothing. +// TODO(jdef) a smarter default impl would decline all offers so as to avoid resource hoarding. +const DefaultHandler = events.NoopHandler diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go new file mode 100644 index 00000000..0232362e --- /dev/null +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -0,0 +1,136 @@ +package controller + +import ( + "context" + "fmt" + "log" + "time" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" +) + +// ErrEvent errors are generated by LiftErrors upon receiving an ERROR event from Mesos. +type ErrEvent string + +func (e ErrEvent) Error() string { + return string(e) +} + +// LiftErrors extract the error message from a scheduler error event and returns it as an ErrEvent +// so that downstream rules/handlers may continue processing. +func LiftErrors() Rule { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + if err != nil { + return chain(ctx, e, err) + } + if e.GetType() == scheduler.Event_ERROR { + // it's recommended that we abort and re-try subscribing; returning an + // error here will cause the event loop to terminate and the connection + // will be reset. + return chain(ctx, e, ErrEvent(e.GetError().GetMessage())) + } + return chain(ctx, e, nil) + } +} + +// StateError is returned when the system encounters an unresolvable state transition error and +// should likely exit. +type StateError string + +func (err StateError) Error() string { return string(err) } + +func TrackSubscription(frameworkIDStore store.Singleton, failoverTimeout time.Duration) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + if err != nil { + return chain(ctx, e, err) + } + if e.GetType() == scheduler.Event_SUBSCRIBED { + var ( + storedFrameworkID, err = frameworkIDStore.Get() + frameworkID = e.GetSubscribed().GetFrameworkID().GetValue() + ) + if err != nil && err != store.ErrNotFound { + return chain(ctx, e, err) + } + // order of `if` statements are important: tread carefully w/ respect to future changes + if frameworkID == "" { + // sanity check, should **never** happen + return chain(ctx, e, StateError("mesos sent an empty frameworkID?!")) + } + if storedFrameworkID != "" && storedFrameworkID != frameworkID && failoverTimeout > 0 { + return chain(ctx, e, StateError(fmt.Sprintf( + "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", failoverTimeout))) + } + if storedFrameworkID != frameworkID { + frameworkIDStore.Set(frameworkID) + } + } + return chain(ctx, e, nil) + } +} + +// AckStatusUpdates sends an acknowledgement of a task status update back to mesos and drops the event if +// sending the ack fails. If successful, the specified err param (if any) is forwarded. Acknowledgements +// are only attempted for task status updates tagged with a UUID. +func AckStatusUpdates(caller calls.Caller) Rule { + return AckStatusUpdatesF(func() calls.Caller { return caller }) +} + +// AckStatusUpdatesF is a functional adapter for AckStatusUpdates, useful for cases where the caller may +// change over time. An error that occurs while ack'ing the status update is returned as a calls.AckError. +func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + // aggressively attempt to ack updates: even if there's pre-existing error state attempt + // to acknowledge all status updates. + origErr := err + if e.GetType() == scheduler.Event_UPDATE { + var ( + s = e.GetUpdate().GetStatus() + uuid = s.GetUUID() + ) + // only ACK non-empty UUID's, as per mesos scheduler spec + if len(uuid) > 0 { + ack := calls.Acknowledge( + s.GetAgentID().GetValue(), + s.TaskID.Value, + uuid, + ) + err = calls.CallNoData(ctx, callerLookup(), ack) + if err != nil { + // TODO(jdef): not sure how important this is; if there's an error ack'ing + // because we beacame disconnected, then we'll just reconnect later and + // Mesos will ask us to ACK anyway -- why pay special attention to these + // call failures vs others? + err = &calls.AckError{Ack: ack, Cause: err} + return ctx, e, Error2(origErr, err) // drop (do not propagate to chain) + } + } + } + return chain(ctx, e, origErr) + } +} + +// DefaultEventLabel is, by default, logged as the first argument by DefaultEventLogger +const DefaultEventLabel = "event" + +// DefaultEventLogger logs the event via the `log` package. +func DefaultEventLogger(eventLabel string) func(*scheduler.Event) { + if eventLabel == "" { + return func(e *scheduler.Event) { log.Println(e) } + } + return func(e *scheduler.Event) { log.Println(eventLabel, e) } +} + +// LogEvents returns a rule that logs scheduler events to the EventLogger +func LogEvents(f func(*scheduler.Event)) Rule { + if f == nil { + f = DefaultEventLogger(DefaultEventLabel) + } + return Rule(func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + f(e) + return chain(ctx, e, err) + }) +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go new file mode 100644 index 00000000..08f8c794 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -0,0 +1,399 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + evaler interface { + // Eval executes a filter, rule, or decorator function; if the returned event is nil then + // no additional rule evaluation should be processed for the event. + // Eval implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + // Eval implementations SHOULD be safe to execute concurrently. + Eval(context.Context, *scheduler.Event, error, Chain) (context.Context, *scheduler.Event, error) + } + + // Rule is the functional adaptation of evaler. + // A nil Rule is valid: it is Eval'd as a noop. + Rule func(context.Context, *scheduler.Event, error, Chain) (context.Context, *scheduler.Event, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(context.Context, *scheduler.Event, error) (context.Context, *scheduler.Event, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +var ( + _ = evaler(Rule(nil)) + _ = evaler(Rules{}) +) + +// ChainIdentity is a Chain that returns the arguments as its results. +func ChainIdentity(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + return ctx, e, err +} + +// Eval is a convenience func that processes a nil Rule as a noop. +func (r Rule) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if r != nil { + return r(ctx, e, err, ch) + } + return ch(ctx, e, err) +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(rs.Chain()(ctx, e, err)) +} + +// Chain returns a Chain that evaluates the given Rules, in order, propagating the (context.Context, *scheduler.Event, error) +// from Rule to Rule. Chain is safe to invoke concurrently. +func (rs Rules) Chain() Chain { + if len(rs) == 0 { + return ChainIdentity + } + return func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + return rs[0].Eval(ctx, e, err, rs[1:].Chain()) + } +} + +// It is the semantic equivalent of Rules{r1, r2, ..., rn}.Rule() and exists purely for convenience. +func Concat(rs ...Rule) Rule { return Rules(rs).Eval } + +const ( + MsgNoErrors = "no errors" +) + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return MsgNoErrors + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + if list, ok := b.(ErrorList); ok { + return flatten(list).Err() + } + return b + } + if b == nil { + if list, ok := a.(ErrorList); ok { + return flatten(list).Err() + } + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + switch len(es) { + case 0: + return nil + case 1: + return es[0] + default: + return es + } +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + return flatten(es).Err() +} + +func flatten(errors []error) ErrorList { + if errors == nil || len(errors) == 0 { + return nil + } + result := make([]error, 0, len(errors)) + for _, err := range errors { + if err != nil { + if multi, ok := err.(ErrorList); ok { + result = append(result, flatten(multi)...) + } else { + result = append(result, err) + } + } + } + return ErrorList(result) +} + +// TODO(jdef): other ideas for Rule decorators: When(func() bool), WhenNot(func() bool) + +// If only executes the receiving rule if b is true; otherwise, the returned rule is a noop. +func (r Rule) If(b bool) Rule { + if b { + return r + } + return nil +} + +// Unless only executes the receiving rule if b is false; otherwise, the returned rule is a noop. +func (r Rule) Unless(b bool) Rule { + if !b { + return r + } + return nil +} + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + if r == nil { + return nil + } + var once sync.Once + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + ruleInvoked := false + once.Do(func() { + ctx, e, err = r(ctx, e, err, ch) + ruleInvoked = true + }) + if !ruleInvoked { + ctx, e, err = ch(ctx, e, err) + } + return ctx, e, err + } +} + +// UnlessDone returns a decorated rule that checks context.Done: if the context has been canceled then the rule chain +// is aborted and the context.Err is merged with the current error state. +// Returns nil (noop) if the receiving Rule is nil. +func (r Rule) UnlessDone() Rule { + if r == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + select { + case <-ctx.Done(): + return ctx, e, Error2(err, ctx.Err()) + default: + return r(ctx, e, err, ch) + } + } +} + +type Overflow int + +const ( + // OverflowWait waits until the rule may execute, or the context is canceled. + OverflowWait Overflow = iota + // OverflowOtherwise skips over the decorated rule and invoke an alternative instead. + OverflowOtherwise +) + +// RateLimit invokes the receiving Rule if a read of chan "p" succeeds (closed chan = no rate limit), otherwise proceeds +// according to the specified Overflow policy. May be useful, for example, when rate-limiting logged events. +// Returns nil (noop) if the receiver is nil, otherwise a nil chan will normally trigger an overflow. +// Panics when OverflowWait is specified with a nil chan, in order to prevent deadlock. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) RateLimit(p <-chan struct{}, over Overflow, otherwise Rule) Rule { + return limit(r, acquireChan(p), over, otherwise) +} + +// acquireChan wraps a signal chan with a func that can be used with rateLimit. +// should only be called by rate limiting funcs (that implement deadlock avoidance). +func acquireChan(tokenCh <-chan struct{}) func(context.Context, bool) bool { + if tokenCh == nil { + // always false: acquire never succeeds; panic if told to block (to avoid deadlock) + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-ctx.Done(): + default: + panic("deadlock detected: block should never be true when the token chan is nil") + } + } + return false + } + } + return func(ctx context.Context, block bool) bool { + if block { + select { + case <-tokenCh: + // tie breaker prefers Done + select { + case <-ctx.Done(): + default: + return true + } + case <-ctx.Done(): + } + return false + } + select { + case <-tokenCh: + return true + default: + return false + } + } +} + +// limit is a generic Rule decorator that limits invocations of said Rule. +// The "acquire" func SHOULD NOT block if the supplied Context is Done. +// MUST only invoke "acquire" once per event. +// TODO(jdef): leaving this as internal for now because the interface still feels too messy. +func limit(r Rule, acquire func(_ context.Context, block bool) bool, over Overflow, otherwise Rule) Rule { + if r == nil { + return nil + } + if acquire == nil { + panic("acquire func is not allowed to be nil") + } + blocking := false + switch over { + case OverflowOtherwise: + case OverflowWait: + blocking = true + default: + panic(fmt.Sprintf("unexpected Overflow type: %#v", over)) + } + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if !acquire(ctx, blocking) { + return otherwise.Eval(ctx, e, err, ch) + } + return r(ctx, e, err, ch) + } +} + +/* TODO(jdef) not sure that this is very useful, leaving out for now... + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then the receiver is returned, undecorated. +// The "otherwise" Rule (may be null) is invoked for every event in between the n'th invocations. +// A cancelled context will trigger the "otherwise" rule. +func (r Rule) EveryN(nthTime int, otherwise Rule) Rule { + if nthTime < 2 || r == nil { + return r + } + return limit(r, acquireEveryN(nthTime), OverflowOtherwise, otherwise) +} + +// acquireEveryN returns an "acquire" func (for use w/ rate-limiting) that returns true every N'th invocation. +// the returned func MUST NOT be used with a potentially blocking Overflow policy (or else it panics). +// nthTime SHOULD be greater than math.MinInt32, values less than 2 probably don't make sense in practice. +func acquireEveryN(nthTime int) func(context.Context, bool) bool { + var ( + i = 1 // begin with the first event seen + m sync.Mutex + ) + return func(ctx context.Context, block bool) (result bool) { + if block { + panic("acquireEveryN should never be asked to block") + } + select { + case <-ctx.Done(): + default: + m.Lock() + i-- + if i <= 0 { + i = nthTime + result = true + } + m.Unlock() + } + return + } +} + +*/ + +// Drop aborts the Chain and returns the (context.Context, *scheduler.Event, error) tuple as-is. +func Drop() Rule { + return Rule(nil).ThenDrop() +} + +// ThenDrop executes the receiving rule, but aborts the Chain, and returns the (context.Context, *scheduler.Event, error) tuple as-is. +func (r Rule) ThenDrop() Rule { + return func(ctx context.Context, e *scheduler.Event, err error, _ Chain) (context.Context, *scheduler.Event, error) { + return r.Eval(ctx, e, err, ChainIdentity) + } +} + +// Fail returns a Rule that injects the given error. +func Fail(injected error) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(ctx, e, Error2(err, injected)) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (e, err) is returned; otherwise control passes to the receiving rule. +func (r Rule) DropOnError() Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if err != nil { + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + if err == nil { + // bypass remainder of chain + return ctx, e, err + } + return r.Eval(ctx, e, err, ch) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go new file mode 100644 index 00000000..8b17f3ff --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -0,0 +1,646 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func prototype() *scheduler.Event { return &scheduler.Event{} } + +func counter(i *int) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func tracer(r Rule, name string, t *testing.T) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + t.Log("executing", name) + return r(ctx, e, err, ch) + } +} + +func returnError(re error) Rule { + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + return ch(ctx, e, Error2(err, re)) + } +} + +func chainCounter(i *int, ch Chain) Chain { + return func(ctx context.Context, e *scheduler.Event, err error) (context.Context, *scheduler.Event, error) { + *i++ + return ch(ctx, e, err) + } +} + +func chainPanic(x interface{}) Chain { + return func(_ context.Context, _ *scheduler.Event, _ error) (context.Context, *scheduler.Event, error) { + panic(x) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + _, e, err := Rules{counterRule}.Eval(context.Background(), nil, nil, ChainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestRules(t *testing.T) { + var ( + p = prototype() + a = errors.New("a") + ctx = context.Background() + ) + + // multiple rules in Rules should execute, dropping nil rules along the way + for _, tc := range []struct { + e *scheduler.Event + err error + }{ + {nil, nil}, + {nil, a}, + {p, nil}, + {p, a}, + } { + var ( + i int + rule = Concat( + nil, + tracer(counter(&i), "counter1", t), + nil, + tracer(counter(&i), "counter2", t), + nil, + ) + _, e, err = rule(ctx, tc.e, tc.err, ChainIdentity) + ) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + if i != 2 { + t.Error("expected 2 rule executions instead of", i) + } + + // empty Rules should not change event, err + _, e, err = Rules{}.Eval(ctx, tc.e, tc.err, ChainIdentity) + if e != tc.e { + t.Errorf("expected prototype event %q instead of %q", tc.e, e) + } + if err != tc.err { + t.Errorf("expected %q error instead of %q", tc.err, err) + } + } +} + +func TestError(t *testing.T) { + a := errors.New("a") + list := ErrorList{a} + + msg := list.Error() + if msg != a.Error() { + t.Errorf("expected %q instead of %q", a.Error(), msg) + } + + msg = ErrorList{}.Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } + + msg = ErrorList(nil).Error() + if msg != MsgNoErrors { + t.Errorf("expected %q instead of %q", MsgNoErrors, msg) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + wantsMessage string + }{ + {nil, nil, nil, ""}, + {nil, ErrorList{nil}, nil, ""}, + {ErrorList{nil}, ErrorList{nil}, nil, ""}, + {ErrorList{ErrorList{nil}}, ErrorList{nil}, nil, ""}, + {a, nil, a, "a"}, + {ErrorList{a}, nil, a, "a"}, + {ErrorList{nil, a, ErrorList{}}, nil, a, "a"}, + {nil, b, b, "b"}, + {nil, ErrorList{b}, b, "b"}, + {a, b, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{b}, ErrorList{a, b}, "a (and 1 more errors)"}, + {a, ErrorList{nil, ErrorList{b, ErrorList{}, nil}}, ErrorList{a, b}, "a (and 1 more errors)"}, + } { + var ( + sameError bool + result = Error2(tc.a, tc.b) + ) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + sameError = (!IsErrorList(result) && result == tc.wants) || + (IsErrorList(result) && reflect.DeepEqual(result, tc.wants)) + } + if !sameError { + t.Fatalf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } + if result != nil && tc.wantsMessage != result.Error() { + t.Fatalf("test case %d failed, expected message %q instead of %q", + i, tc.wantsMessage, result.Error()) + } + } +} + +func TestUnlessDone(t *testing.T) { + var ( + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + ) + for ti, tc := range []struct { + ctx context.Context + wantsError []error + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, []error{nil, nil}, []int{1, 2}, []int{1, 2}}, + {fin, []error{nil, context.Canceled}, []int{1, 1}, []int{1, 1}}, + } { + var ( + i, j int + r1 = counter(&i) + r2 = r1.UnlessDone() + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if err != tc.wantsError[k] { + t.Errorf("test case %d failed: unexpected error %v", ti, err) + } + if i != tc.wantsRuleCount[k] { + t.Errorf("test case %d failed: expected count of %d instead of %d", ti, tc.wantsRuleCount[k], i) + } + if j != tc.wantsChainCount[k] { + t.Errorf("test case %d failed: expected chain count of %d instead of %d", ti, tc.wantsRuleCount[k], j) + } + } + } + r := Rule(nil).UnlessDone() + if r != nil { + t.Error("expected nil result from UnlessDone") + } +} + +func TestAndThen(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rule(nil).AndThen(counter(&i)) + a = errors.New("a") + ) + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnFailure(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + a = errors.New("a") + r1 = counter(&i) + r2 = Fail(a).OnFailure(counter(&i)) + ) + for k, tc := range []struct { + r Rule + initialError error + }{ + {r1, a}, + {r2, nil}, + } { + _, e, err := tc.r(ctx, p, tc.initialError, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestDropOnError(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnError() + a = errors.New("a") + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + _, e, err := r2(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } +} + +func TestDropOnSuccess(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).DropOnSuccess() + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for _, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + a := errors.New("a") + _, e, err := r2(ctx, p, a, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != a { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 2 { + t.Errorf("expected chain count of 2 instead of %d", j) + } + + r3 := Rules{DropOnSuccess(), r1}.Eval + _, e, err = r3(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 2 { + t.Errorf("expected count of 2 instead of %d", i) + } + if j != 3 { + t.Errorf("expected chain count of 3 instead of %d", j) + } +} + +func TestThenDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = counter(&i).ThenDrop() + ) + // r1 and r2 should execute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != (k + 1) { + t.Errorf("expected count of %d instead of %d", (k + 1), i) + } + if j != 1 { + t.Errorf("expected chain count of 1 instead of %d", j) + } + } + } +} + +func TestDrop(t *testing.T) { + for _, anErr := range []error{nil, errors.New("a")} { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i) + r2 = Rules{Drop(), counter(&i)}.Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, anErr, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != anErr { + t.Errorf("expected %v instead of error %v", anErr, err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d with error %v", (k + 1), j, anErr) + } + } + } +} + +func TestIf(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).If(true).Eval + r2 = counter(&i).If(false).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestUnless(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Unless(false).Eval + r2 = counter(&i).Unless(true).Eval + ) + // r1 should execute the counter rule + // r2 should NOT exexute the counter rule + for k, r := range []Rule{r1, r2} { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if j != (k + 1) { + t.Errorf("expected chain count of %d instead of %d", (k + 1), j) + } + } +} + +func TestOnce(t *testing.T) { + var ( + i, j int + p = prototype() + ctx = context.Background() + r1 = counter(&i).Once().Eval + r2 = Rule(nil).Once().Eval + ) + for k, r := range []Rule{r1, r2} { + for x := 0; x < 5; x++ { + _, e, err := r(ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Error("unexpected error", err) + } + if i != 1 { + t.Errorf("expected count of 1 instead of %d", i) + } + if y := (k * 5) + x + 1; j != y { + t.Errorf("expected chain count of %d instead of %d", y, j) + } + } + } +} + +func TestRateLimit(t *testing.T) { + // non-blocking, then blocking + o := func() <-chan struct{} { + x := make(chan struct{}, 1) + x <- struct{}{} + return x + } + var ( + ch1 <-chan struct{} // always nil, blocking + ch2 = make(chan struct{}) // non-nil, blocking + // ch3 is o() + ch4 = make(chan struct{}) // non-nil, closed + p = prototype() + ctx = context.Background() + fin = func() context.Context { + c, cancel := context.WithCancel(context.Background()) + cancel() + return c + }() + + errOverflow = errors.New("overflow") + otherwiseSkip = Rule(nil) + otherwiseSkipWithError = Fail(errOverflow) + otherwiseDiscard = Drop() + otherwiseDiscardWithError = Fail(errOverflow).ThenDrop() + ) + close(ch4) + for ti, tc := range []struct { + // each set of inputs is executed 4 times: twice for r1, twice for r2 + ctx context.Context + ch <-chan struct{} + over Overflow + otherwise Rule + wantsError int // bitmask: lower 4 bits, one for each case; first case = highest bit + wantsRuleCount []int + wantsChainCount []int + }{ + {ctx, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkip, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkip, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowOtherwise, otherwiseSkipWithError, 0xC, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowOtherwise, otherwiseSkipWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowOtherwise, otherwiseSkipWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscard, 0x0, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscard, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {ctx, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {ctx, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {ctx, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, ch2, OverflowOtherwise, otherwiseDiscardWithError, 0xC, []int{0, 0, 0, 0}, []int{0, 0, 1, 2}}, + {fin, o(), OverflowOtherwise, otherwiseDiscardWithError, 0x4, []int{1, 1, 1, 1}, []int{1, 1, 2, 3}}, + {fin, ch4, OverflowOtherwise, otherwiseDiscardWithError, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + + {fin, ch1, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch2, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, o(), OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {fin, ch4, OverflowWait, nil, 0x0, []int{0, 0, 0, 0}, []int{1, 2, 3, 4}}, + {ctx, ch4, OverflowWait, nil, 0x0, []int{1, 2, 2, 2}, []int{1, 2, 3, 4}}, + } { + var ( + i, j int + r1 = counter(&i).RateLimit(tc.ch, tc.over, tc.otherwise).Eval + r2 = Rule(nil).RateLimit(tc.ch, tc.over, tc.otherwise).Eval // a nil rule still invokes the chain + ) + for k, r := range []Rule{r1, r2} { + // execute each rule twice + for x := 0; x < 2; x++ { + _, e, err := r(tc.ctx, p, nil, chainCounter(&j, ChainIdentity)) + if e != p { + t.Errorf("test case %d failed: expected event %q instead of %q", ti, p, e) + } + if b := 8 >> uint(k*2+x); ((b & tc.wantsError) != 0) != (err != nil) { + t.Errorf("test case (%d,%d,%d) failed: unexpected error %v", ti, k, x, err) + } + if y := tc.wantsRuleCount[k*2+x]; i != y { + t.Errorf("test case (%d,%d,%d) failed: expected count of %d instead of %d", + ti, k, x, y, i) + } + if y := tc.wantsChainCount[k*2+x]; j != y { + t.Errorf("test case (%d,%d,%d) failed: expected chain count of %d instead of %d", + ti, k, x, y, j) + } + } + } + } + // test blocking capability via rateLimit + blocked := false + r := limit(Rule(nil).Eval, func(_ context.Context, b bool) bool { blocked = b; return false }, OverflowWait, nil) + _, e, err := r(ctx, p, nil, ChainIdentity) + if e != p { + t.Errorf("expected event %q instead of %q", p, e) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } + if !blocked { + t.Error("expected OverflowWait to block rule execution") + } + // test RateLimit deadlock detector + didPanic := false + func() { + defer func() { didPanic = recover() != nil }() + Rule(Rule(nil).Eval).RateLimit(nil, OverflowWait, nil).Eval(ctx, p, nil, ChainIdentity) + }() + if !didPanic { + t.Error("expected panic because we configured a rule to deadlock") + } +} diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go new file mode 100644 index 00000000..8b668346 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -0,0 +1,7 @@ +package eventrules + +//go:generate go run ../../gen/rules.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} + +//go:generate go run ../../gen/rule_handlers.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/events -type E:*scheduler.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go + +//go:generate go run ../../gen/rule_metrics.go ../../gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type -output metrics_generated.go diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go new file mode 100644 index 00000000..3b682cbf --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/handlers_generated.go @@ -0,0 +1,51 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -import github.com/mesos/mesos-go/api/v1/lib/scheduler/events -type E:*scheduler.Event -type H:events.Handler -type HF:events.HandlerFunc -output handlers_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +// Handle generates a rule that executes the given events.Handler. +func Handle(h events.Handler) Rule { + if h == nil { + return nil + } + return func(ctx context.Context, e *scheduler.Event, err error, chain Chain) (context.Context, *scheduler.Event, error) { + newErr := h.HandleEvent(ctx, e) + return chain(ctx, e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h events.HandlerFunc) Rule { + return Handle(events.Handler(h)) +} + +// Handle returns a Rule that invokes the receiver, then the given events.Handler +func (r Rule) Handle(h events.Handler) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h events.HandlerFunc) Rule { + return r.Handle(events.Handler(h)) +} + +// HandleEvent implements events.Handler for Rule +func (r Rule) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { + if r == nil { + return nil + } + _, _, err = r(ctx, e, nil, ChainIdentity) + return +} + +// HandleEvent implements events.Handler for Rules +func (rs Rules) HandleEvent(ctx context.Context, e *scheduler.Event) error { + return Rule(rs.Eval).HandleEvent(ctx, e) +} diff --git a/api/v1/lib/extras/scheduler/eventrules/metrics_generated.go b/api/v1/lib/extras/scheduler/eventrules/metrics_generated.go new file mode 100644 index 00000000..dfe1bdb8 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/metrics_generated.go @@ -0,0 +1,48 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "strings" + + "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// Labeler generates a set of strings that should be associated with metrics that are generated for the given event. +type Labeler func(ctx context.Context, e *scheduler.Event) []string + +var defaultLabels = func() map[scheduler.Event_Type][]string { + m := make(map[scheduler.Event_Type][]string) + for k, v := range scheduler.Event_Type_name { + m[scheduler.Event_Type(k)] = []string{strings.ToLower(v)} + } + return m +}() + +func defaultLabeler(ctx context.Context, e *scheduler.Event) []string { + return defaultLabels[e.GetType()] +} + +// Metrics generates a Rule that invokes the given harness for each event, using the labels generated by the Labeler. +// Panics if harness or labeler is nil. +func Metrics(harness metrics.Harness, labeler Labeler) Rule { + if harness == nil { + panic("harness is a required parameter") + } + if labeler == nil { + labeler = defaultLabeler + } + return func(ctx context.Context, e *scheduler.Event, err error, ch Chain) (context.Context, *scheduler.Event, error) { + labels := labeler(ctx, e) + harness(func() error { + ctx, e, err = ch(ctx, e, err) + return err + }, labels...) + return ctx, e, err + } +} diff --git a/api/v1/lib/extras/scheduler/eventrules/metrics_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/metrics_generated_test.go new file mode 100644 index 00000000..b4c87b59 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/metrics_generated_test.go @@ -0,0 +1,50 @@ +package eventrules + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type -output metrics_generated.go +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func TestMetrics(t *testing.T) { + var ( + i int + ctx = context.Background() + p = &scheduler.Event{} + a = errors.New("a") + h = func(f func() error, _ ...string) error { + i++ + return f() + } + r = Metrics(h, func(_ context.Context, _ *scheduler.Event) []string { return nil }) + ) + for ti, tc := range []struct { + ctx context.Context + e *scheduler.Event + err error + }{ + {ctx, p, a}, + {ctx, p, nil}, + {ctx, nil, a}, + } { + c, e, err := r.Eval(tc.ctx, tc.e, tc.err, ChainIdentity) + if !reflect.DeepEqual(c, tc.ctx) { + t.Errorf("test case %d: expected context %q instead of %q", ti, tc.ctx, c) + } + if !reflect.DeepEqual(e, tc.e) { + t.Errorf("test case %d: expected event %q instead of %q", ti, tc.e, e) + } + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("test case %d: expected error %q instead of %q", ti, tc.err, err) + } + if y := ti + 1; y != i { + t.Errorf("test case %d: expected count %q instead of %q", ti, y, i) + } + } +} diff --git a/api/v1/lib/extras/scheduler/offers/filters.go b/api/v1/lib/extras/scheduler/offers/filters.go new file mode 100644 index 00000000..9a872665 --- /dev/null +++ b/api/v1/lib/extras/scheduler/offers/filters.go @@ -0,0 +1,69 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Filter accepts or rejects a mesos Offer + Filter interface { + Accept(*Offer) bool + } + + // FilterFunc returns true if the given Offer passes the filter + FilterFunc func(*Offer) bool +) + +// Accept implements Filter for FilterFunc +func (f FilterFunc) Accept(o *Offer) bool { return f(o) } + +func nilFilter(_ *Offer) bool { return true } + +func not(f Filter) Filter { + return FilterFunc(func(offer *Offer) bool { return !f.Accept(offer) }) +} + +// ByHostname returns a Filter that accepts offers with a matching Hostname +func ByHostname(hostname string) Filter { + if hostname == "" { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return o.Hostname == hostname + }) +} + +// ByAttributes returns a Filter that accepts offers with an attribute set accepted by +// the provided Attribute filter func. +func ByAttributes(f func(attr []Attribute) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Attributes) + }) +} + +func ByExecutors(f func(exec []ExecutorID) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.ExecutorIDs) + }) +} + +func ByUnavailability(f func(u *Unavailability) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Unavailability) + }) +} + +// ContainsResources returns a filter function that returns true if the Resources of an Offer +// contain the wanted Resources. +func ContainsResources(wanted Resources) Filter { + return FilterFunc(func(o *Offer) bool { + return Resources(o.Resources).Flatten().ContainsAll(wanted) + }) +} diff --git a/api/v1/lib/extras/scheduler/offers/offers.go b/api/v1/lib/extras/scheduler/offers/offers.go new file mode 100644 index 00000000..34c275e5 --- /dev/null +++ b/api/v1/lib/extras/scheduler/offers/offers.go @@ -0,0 +1,247 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Slice is a convenience type wrapper for a slice of mesos Offer messages + Slice []Offer + + // Index is a convenience type wrapper for a dictionary of Offer messages + Index map[interface{}]*Offer + + // KeyFunc generates a key used for indexing offers + KeyFunc func(*Offer) interface{} +) + +// IDs extracts the ID field from a Slice of offers +func (offers Slice) IDs() []OfferID { + ids := make([]OfferID, len(offers)) + for i := range offers { + ids[i] = offers[i].ID + } + return ids +} + +// IDs extracts the ID field from a Index of offers +func (offers Index) IDs() []OfferID { + ids := make([]OfferID, 0, len(offers)) + for _, offer := range offers { + ids = append(ids, offer.GetID()) + } + return ids +} + +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. +func (offers Slice) Find(filter Filter) *Offer { + for i := range offers { + offer := &offers[i] + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. +func (offers Index) Find(filter Filter) *Offer { + for _, offer := range offers { + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Filter returns the subset of the Slice that matches the given filter. +func (offers Slice) Filter(filter Filter) (result Slice) { + if sz := len(result); sz > 0 { + result = make(Slice, 0, sz) + for i := range offers { + if filter.Accept(&offers[i]) { + result = append(result, offers[i]) + } + } + } + return +} + +// Filter returns the subset of the Index that matches the given filter. +func (offers Index) Filter(filter Filter) (result Index) { + if sz := len(result); sz > 0 { + result = make(Index, sz) + for id, offer := range offers { + if filter.Accept(offer) { + result[id] = offer + } + } + } + return +} + +// FilterNot returns the subset of the Slice that does not match the given filter. +func (offers Slice) FilterNot(filter Filter) Slice { return offers.Filter(not(filter)) } + +// FilterNot returns the subset of the Index that does not match the given filter. +func (offers Index) FilterNot(filter Filter) Index { return offers.Filter(not(filter)) } + +// DefaultKeyFunc indexes offers by their OfferID. +var DefaultKeyFunc = KeyFunc(KeyFuncByOfferID) + +func KeyFuncByOfferID(o *Offer) interface{} { return o.GetID() } + +// NewIndex returns a new Index constructed from the list of mesos offers. +// If the KeyFunc is nil then offers are indexed by DefaultKeyFunc. +// The values of the returned Index are pointers to (not copies of) the offers of the slice receiver. +func NewIndex(slice []Offer, kf KeyFunc) Index { + if slice == nil { + return nil + } + if kf == nil { + kf = DefaultKeyFunc + } + index := make(Index, len(slice)) + for i := range slice { + offer := &slice[i] + index[kf(offer)] = offer + } + return index +} + +// ToSlice returns a Slice from the offers in the Index. +// The returned slice will contain shallow copies of the offers from the Index. +func (s Index) ToSlice() (slice Slice) { + if sz := len(s); sz > 0 { + slice = make(Slice, 0, sz) + for _, offer := range s { + slice = append(slice, *offer) + } + } + return +} + +type ( + Reducer interface { + Reduce(_, _ *Offer) *Offer + } + + ReduceFunc func(_, _ *Offer) *Offer +) + +func (f ReduceFunc) Reduce(a, b *Offer) *Offer { return f(a, b) } + +var _ = Reducer(ReduceFunc(func(_, _ *Offer) *Offer { return nil })) // sanity check + +func (slice Slice) Reduce(def Offer, r Reducer) (result Offer) { + result = def + if r != nil { + acc := &result + for i := range slice { + acc = r.Reduce(&result, &slice[i]) + } + if acc == nil { + result = Offer{} + } else { + result = *acc + } + } + return +} + +func (index Index) Reduce(def *Offer, r Reducer) (result *Offer) { + result = def + if r != nil { + for i := range index { + result = r.Reduce(result, index[i]) + } + } + return +} + +func (slice Slice) GroupBy(kf KeyFunc) map[interface{}]Slice { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(slice) == 0 { + return nil + } + result := make(map[interface{}]Slice) + for i := range slice { + groupKey := kf(&slice[i]) + result[groupKey] = append(result[groupKey], slice[i]) + } + return result +} + +func (index Index) GroupBy(kf KeyFunc) map[interface{}]Index { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(index) == 0 { + return nil + } + result := make(map[interface{}]Index) + for i, offer := range index { + groupKey := kf(offer) + group, ok := result[groupKey] + if !ok { + group = make(Index) + result[groupKey] = group + } + group[i] = offer + } + return result +} + +func (index Index) Partition(f Filter) (accepted, rejected Index) { + if f == nil { + return index, nil + } + if len(index) > 0 { + accepted, rejected = make(Index), make(Index) + for id, offer := range index { + if f.Accept(offer) { + accepted[id] = offer + } else { + rejected[id] = offer + } + } + } + return +} + +func (s Slice) Partition(f Filter) (accepted, rejected []int) { + if f == nil { + accepted = make([]int, len(s)) + for i := range s { + accepted[i] = i + } + return + } + if sz := len(s); sz > 0 { + accepted, rejected = make([]int, 0, sz/2), make([]int, 0, sz/2) + for i := range s { + offer := &s[i] + if f.Accept(offer) { + accepted = append(accepted, i) + } else { + rejected = append(rejected, i) + } + } + } + return +} + +func (index Index) Reindex(kf KeyFunc) Index { + sz := len(index) + if kf == nil || sz == 0 { + return index + } + result := make(Index, sz) + for _, offer := range index { + key := kf(offer) + result[key] = offer + } + return result +} diff --git a/api/v1/lib/operations.go b/api/v1/lib/extras/scheduler/operations/operations.go similarity index 51% rename from api/v1/lib/operations.go rename to api/v1/lib/extras/scheduler/operations/operations.go index 562a8270..a944a692 100644 --- a/api/v1/lib/operations.go +++ b/api/v1/lib/extras/scheduler/operations/operations.go @@ -1,65 +1,67 @@ -package mesos +package operations import ( "errors" "fmt" "github.com/gogo/protobuf/proto" + "github.com/mesos/mesos-go/api/v1/lib" + rez "github.com/mesos/mesos-go/api/v1/lib/extras/resources" ) type ( - OperationErrorType int + operationErrorType int - OperationError struct { - errorType OperationErrorType - opType Offer_Operation_Type + operationError struct { + errorType operationErrorType + opType mesos.Offer_Operation_Type cause error } - offerResourceOp func(Offer_Operation, Resources) (Resources, error) + offerResourceOp func(*mesos.Offer_Operation, mesos.Resources) (mesos.Resources, error) opErrorHandler func(cause error) error ) const ( - OperationErrorTypeInvalid OperationErrorType = iota - OperationErrorTypeUnknown + operationErrorTypeInvalid operationErrorType = iota + operationErrorTypeUnknown ) var ( - opNoop = func(_ Offer_Operation, _ Resources) (_ Resources, _ error) { return } - offerResourceOpMap = map[Offer_Operation_Type]offerResourceOp{ - Offer_Operation_LAUNCH: opNoop, - Offer_Operation_LAUNCH_GROUP: opNoop, - Offer_Operation_RESERVE: handleOpErrors(invalidOp(Offer_Operation_RESERVE), validateOpTotals(opReserve)), - Offer_Operation_UNRESERVE: handleOpErrors(invalidOp(Offer_Operation_UNRESERVE), validateOpTotals(opUnreserve)), - Offer_Operation_CREATE: handleOpErrors(invalidOp(Offer_Operation_CREATE), validateOpTotals(opCreate)), - Offer_Operation_DESTROY: handleOpErrors(invalidOp(Offer_Operation_DESTROY), validateOpTotals(opDestroy)), + opNoop = func(_ *mesos.Offer_Operation, _ mesos.Resources) (_ mesos.Resources, _ error) { return } + offerResourceOpMap = map[mesos.Offer_Operation_Type]offerResourceOp{ + mesos.Offer_Operation_LAUNCH: opNoop, + mesos.Offer_Operation_LAUNCH_GROUP: opNoop, + mesos.Offer_Operation_RESERVE: handleOpErrors(invalidOp(mesos.Offer_Operation_RESERVE), validateOpTotals(opReserve)), + mesos.Offer_Operation_UNRESERVE: handleOpErrors(invalidOp(mesos.Offer_Operation_UNRESERVE), validateOpTotals(opUnreserve)), + mesos.Offer_Operation_CREATE: handleOpErrors(invalidOp(mesos.Offer_Operation_CREATE), validateOpTotals(opCreate)), + mesos.Offer_Operation_DESTROY: handleOpErrors(invalidOp(mesos.Offer_Operation_DESTROY), validateOpTotals(opDestroy)), } ) -func (err *OperationError) Cause() error { return err.cause } -func (err *OperationError) Type() OperationErrorType { return err.errorType } -func (err *OperationError) Operation() Offer_Operation_Type { return err.opType } +func (err *operationError) Cause() error { return err.cause } +func (err *operationError) Type() operationErrorType { return err.errorType } +func (err *operationError) Operation() mesos.Offer_Operation_Type { return err.opType } -func (err *OperationError) Error() string { +func (err *operationError) Error() string { switch err.errorType { - case OperationErrorTypeInvalid: + case operationErrorTypeInvalid: return fmt.Sprintf("invalid "+err.opType.String()+" operation: %+v", err.cause) - case OperationErrorTypeUnknown: + case operationErrorTypeUnknown: return err.cause.Error() default: return fmt.Sprintf("operation error: %+v", err.cause) } } -func invalidOp(t Offer_Operation_Type) opErrorHandler { +func invalidOp(t mesos.Offer_Operation_Type) opErrorHandler { return func(cause error) error { - return &OperationError{errorType: OperationErrorTypeInvalid, opType: t, cause: cause} + return &operationError{errorType: operationErrorTypeInvalid, opType: t, cause: cause} } } func handleOpErrors(f opErrorHandler, op offerResourceOp) offerResourceOp { - return func(operation Offer_Operation, resources Resources) (Resources, error) { + return func(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result, err := op(operation, resources) if err != nil { err = f(err) @@ -69,11 +71,11 @@ func handleOpErrors(f opErrorHandler, op offerResourceOp) offerResourceOp { } func validateOpTotals(f offerResourceOp) offerResourceOp { - return func(operation Offer_Operation, resources Resources) (Resources, error) { + return func(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result, err := f(operation, resources) if err == nil { // sanity CHECK, same as apache/mesos does - if !resources.sameTotals(result) { + if !rez.SumAndCompare(resources, result...) { panic(fmt.Sprintf("result %+v != resources %+v", result, resources)) } } @@ -81,9 +83,9 @@ func validateOpTotals(f offerResourceOp) offerResourceOp { } } -func opReserve(operation Offer_Operation, resources Resources) (Resources, error) { +func opReserve(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result := resources.Clone() - opRes := Resources(operation.GetReserve().GetResources()) + opRes := mesos.Resources(operation.GetReserve().GetResources()) err := opRes.Validate() if err != nil { return nil, err @@ -95,19 +97,19 @@ func opReserve(operation Offer_Operation, resources Resources) (Resources, error if opRes[i].GetReservation() == nil { return nil, errors.New("missing 'reservation'") } - unreserved := Resources{opRes[i]}.Flatten() + unreserved := mesos.Resources{opRes[i]}.Flatten() if !result.ContainsAll(unreserved) { return nil, fmt.Errorf("%+v does not contain %+v", result, unreserved) } result.Subtract(unreserved...) - result.add(opRes[i]) + result.Add1(opRes[i]) } return result, nil } -func opUnreserve(operation Offer_Operation, resources Resources) (Resources, error) { +func opUnreserve(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result := resources.Clone() - opRes := Resources(operation.GetUnreserve().GetResources()) + opRes := mesos.Resources(operation.GetUnreserve().GetResources()) err := opRes.Validate() if err != nil { return nil, err @@ -122,16 +124,16 @@ func opUnreserve(operation Offer_Operation, resources Resources) (Resources, err if !result.Contains(opRes[i]) { return nil, errors.New("resources do not contain unreserve amount") //TODO(jdef) should output nicely formatted resource quantities here } - unreserved := Resources{opRes[i]}.Flatten() - result.subtract(opRes[i]) + unreserved := mesos.Resources{opRes[i]}.Flatten() + result.Subtract1(opRes[i]) result.Add(unreserved...) } return result, nil } -func opCreate(operation Offer_Operation, resources Resources) (Resources, error) { +func opCreate(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result := resources.Clone() - volumes := Resources(operation.GetCreate().GetVolumes()) + volumes := mesos.Resources(operation.GetCreate().GetVolumes()) err := volumes.Validate() if err != nil { return nil, err @@ -150,20 +152,20 @@ func opCreate(operation Offer_Operation, resources Resources) (Resources, error) // now. Persistent volumes can only be be created from regular // disk resources. Revisit this once we start to support // non-persistent volumes. - stripped := proto.Clone(&volumes[i]).(*Resource) + stripped := proto.Clone(&volumes[i]).(*mesos.Resource) stripped.Disk = nil if !result.Contains(*stripped) { return nil, errors.New("invalid CREATE operation: insufficient disk resources") } - result.subtract(*stripped) - result.add(volumes[i]) + result.Subtract1(*stripped) + result.Add1(volumes[i]) } return result, nil } -func opDestroy(operation Offer_Operation, resources Resources) (Resources, error) { +func opDestroy(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { result := resources.Clone() - volumes := Resources(operation.GetDestroy().GetVolumes()) + volumes := mesos.Resources(operation.GetDestroy().GetVolumes()) err := volumes.Validate() if err != nil { return nil, err @@ -178,19 +180,19 @@ func opDestroy(operation Offer_Operation, resources Resources) (Resources, error if !result.Contains(volumes[i]) { return nil, errors.New("persistent volume does not exist") } - stripped := proto.Clone(&volumes[i]).(*Resource) + stripped := proto.Clone(&volumes[i]).(*mesos.Resource) stripped.Disk = nil - result.subtract(volumes[i]) - result.add(*stripped) + result.Subtract1(volumes[i]) + result.Add1(*stripped) } return result, nil } -func (operation Offer_Operation) Apply(resources Resources) (Resources, error) { +func Apply(operation *mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) { f, ok := offerResourceOpMap[operation.GetType()] if !ok { - return nil, &OperationError{ - errorType: OperationErrorTypeUnknown, + return nil, &operationError{ + errorType: operationErrorTypeUnknown, cause: errors.New("unknown offer operation: " + operation.GetType().String()), } } diff --git a/api/v1/lib/extras/scheduler/operations/operations_test.go b/api/v1/lib/extras/scheduler/operations/operations_test.go new file mode 100644 index 00000000..39910a12 --- /dev/null +++ b/api/v1/lib/extras/scheduler/operations/operations_test.go @@ -0,0 +1,113 @@ +package operations_test + +import ( + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/operations" + . "github.com/mesos/mesos-go/api/v1/lib/resourcetest" +) + +func TestOpCreate(t *testing.T) { + var ( + total = Resources( + Resource(Name("cpus"), ValueScalar(1)), + Resource(Name("mem"), ValueScalar(512)), + Resource(Name("disk"), ValueScalar(1000), Role("role")), + ) + volume1 = Resource(Name("disk"), ValueScalar(200), Role("role"), Disk("1", "path")) + volume2 = Resource(Name("disk"), ValueScalar(2000), Role("role"), Disk("1", "path")) + ) + op := Create(Resources(volume1)) + rs, err := operations.Apply(op, total) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + expected := Resources( + Resource(Name("cpus"), ValueScalar(1)), + Resource(Name("mem"), ValueScalar(512)), + Resource(Name("disk"), ValueScalar(800), Role("role")), + volume1, + ) + if !expected.Equivalent(rs) { + t.Fatalf("expected %v instead of %v", expected, rs) + } + + // check the case of insufficient disk resources + op = Create(Resources(volume2)) + _, err = operations.Apply(op, total) + if err == nil { + t.Fatalf("expected an error due to insufficient disk resources") + } +} + +func TestOpUnreserve(t *testing.T) { + var ( + reservedCPU = Resources( + Resource(Name("cpus"), + ValueScalar(1), + Role("role"), + Reservation(ReservedBy("principal")))) + reservedMem = Resources( + Resource(Name("mem"), + ValueScalar(512), + Role("role"), + Reservation(ReservedBy("principal")))) + reserved = reservedCPU.Plus(reservedMem...) + ) + + // test case 1: unreserve some amount of CPU that's already been reserved + unreservedCPU := reservedCPU.Flatten() + t.Log("unreservedCPU=" + unreservedCPU.String()) + + wantsUnreserved := reservedMem.Plus(unreservedCPU...) + actualUnreserved, err := operations.Apply(Unreserve(reservedCPU), reserved) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !wantsUnreserved.Equivalent(actualUnreserved) { + t.Errorf("expected resources %+v instead of %+v", wantsUnreserved, actualUnreserved) + } + + // test case 2: unreserve some amount of CPU greater than that which already been reserved + reservedCPU2 := Resources( + Resource(Name("cpus"), + ValueScalar(2), + Role("role"), + Reservation(ReservedBy("principal")))) + _, err = operations.Apply(Unreserve(reservedCPU2), reserved) + if err == nil { + t.Fatalf("expected reservation error") + } +} + +func TestOpReserve(t *testing.T) { + // func opReserve(operation mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) + var ( + unreservedCPU = Resources(Resource(Name("cpus"), ValueScalar(1))) + unreservedMem = Resources(Resource(Name("mem"), ValueScalar(512))) + unreserved = unreservedCPU.Plus(unreservedMem...) + reservedCPU1 = unreservedCPU.Flatten(mesos.RoleName("role").Assign(), ReservedBy("principal").Assign()) + ) + + // test case 1: reserve an amount of CPU that's available + wantsReserved := unreservedMem.Plus(reservedCPU1...) + actualReserved, err := operations.Apply(Reserve(reservedCPU1), unreserved) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !wantsReserved.Equivalent(actualReserved) { + t.Errorf("expected resources %+v instead of %+v", wantsReserved, actualReserved) + } + + // test case 2: reserve an amount of CPU that's NOT available + reservedCPU2 := Resources( + Resource(Name("cpus"), + ValueScalar(2), + Role("role"), + Reservation(ReservedBy("principal")))) + _, err = operations.Apply(Reserve(reservedCPU2), unreserved) + if err == nil { + t.Fatalf("expected reservation error") + } +} diff --git a/api/v1/lib/extras/store/singleton.go b/api/v1/lib/extras/store/singleton.go new file mode 100644 index 00000000..fab5bc3b --- /dev/null +++ b/api/v1/lib/extras/store/singleton.go @@ -0,0 +1,157 @@ +package store + +import ( + "errors" + "sync/atomic" +) + +type ( + Getter interface { + Get() (string, error) + } + + GetFunc func() (string, error) + + Setter interface { + Set(string) error + } + + SetFunc func(string) error + + // Singleton is a thread-safe abstraction to load and store a string + Singleton interface { + Getter + Setter + } + + SingletonAdapter struct { + GetFunc + SetFunc + } + + SingletonDecorator interface { + Decorate(Singleton) Singleton + } +) + +func (f GetFunc) Get() (string, error) { return f() } +func (f SetFunc) Set(s string) error { return f(s) } + +var ErrNotFound = errors.New("value not found in store") + +func NewInMemorySingleton() Singleton { + var value atomic.Value + return &SingletonAdapter{ + func() (string, error) { + x := value.Load() + if x == nil { + return "", ErrNotFound + } + return x.(string), nil + }, + func(s string) error { + value.Store(s) + return nil + }, + } +} + +type ( + GetFuncDecorator func(Getter, string, error) (string, error) + SetFuncDecorator func(Setter, string, error) error +) + +func DoSet() SetFuncDecorator { + return func(s Setter, v string, _ error) error { + return s.Set(v) + } +} + +func (f SetFuncDecorator) AndThen(f2 SetFuncDecorator) SetFuncDecorator { + return func(s Setter, v string, err error) error { + err = f(s, v, err) + if err != nil { + return err + } + return f2(s, v, nil) + } +} + +func (f SetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + s.Get, + SetFunc(func(v string) error { + return f(s, v, nil) + }), + } +} + +func DoGet() GetFuncDecorator { + return func(s Getter, _ string, _ error) (string, error) { + return s.Get() + } +} + +func (f GetFuncDecorator) AndThen(f2 GetFuncDecorator) GetFuncDecorator { + return func(s Getter, v string, err error) (string, error) { + v, err = f(s, v, err) + if err != nil { + return v, err + } + return f2(s, v, nil) + } +} + +func (f GetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + GetFunc(func() (string, error) { + return f(s, "", nil) + }), + s.Set, + } +} + +func DecorateSingleton(s Singleton, ds ...SingletonDecorator) Singleton { + for _, d := range ds { + if d != nil { + s = d.Decorate(s) + } + } + return s +} + +// GetOrPanic curries the result of a Getter invocation: the returned func only ever returns the string component when +// the error component of the underlying Get() call is nil. If Get() generates an error then the curried func panics. +func GetOrPanic(g Getter) func() string { + return func() string { + v, err := g.Get() + if err != nil { + panic(err) + } + return v + } +} + +func GetIgnoreErrors(g Getter) func() string { + return func() string { + v, _ := g.Get() + return v + } +} + +// SetOrPanic curries the result of a Setter invocation: the returned func only ever returns normally when the error +// component of the underlying Set() call is nil. If Set() generates an error then the curried func panics. +func SetOrPanic(s Setter) func(v string) { + return func(v string) { + err := s.Set(v) + if err != nil { + panic(err) + } + } +} diff --git a/api/v1/lib/httpcli/apierrors/apierrors.go b/api/v1/lib/httpcli/apierrors/apierrors.go index 5351b7b6..d38d0259 100644 --- a/api/v1/lib/httpcli/apierrors/apierrors.go +++ b/api/v1/lib/httpcli/apierrors/apierrors.go @@ -1,6 +1,7 @@ package apierrors import ( + "io" "io/ioutil" "net/http" ) @@ -8,7 +9,7 @@ import ( // Code is a Mesos HTTP v1 API response status code type Code int -var ( +const ( // MsgNotLeader is returned by Do calls that are sent to a non leading Mesos master. MsgNotLeader = "call sent to a non-leading master" // MsgAuth is returned by Do calls that are not successfully authenticated. @@ -40,6 +41,10 @@ var ( CodeMesosUnavailable = Code(http.StatusServiceUnavailable) CodeNotFound = Code(http.StatusNotFound) + MaxSizeDetails = 4 * 1024 // MaxSizeDetails limits the length of the details message read from a response body +) + +var ( // ErrorTable maps HTTP response codes to their respective Mesos v1 API error messages. ErrorTable = map[Code]string{ CodeNotLeader: MsgNotLeader, @@ -56,9 +61,8 @@ var ( // Error captures HTTP v1 API error codes and messages generated by Mesos. type Error struct { - Code Code // Code is the HTTP response status code generated by Mesos - Message string // Message briefly summarizes the nature of the error - Details string // Details captures the HTTP response entity, if any, supplied by Mesos + code Code // code is the HTTP response status code generated by Mesos + message string // message briefly summarizes the nature of the error, possibly includes details from Mesos } // IsErrorCode returns true for all HTTP status codes that are not considered informational or successful. @@ -81,22 +85,42 @@ func FromResponse(res *http.Response) error { return nil } - err := &Error{Code: code} + var details string if res.Body != nil { defer res.Body.Close() - buf, _ := ioutil.ReadAll(res.Body) - err.Details = string(buf) + buf, _ := ioutil.ReadAll(io.LimitReader(res.Body, MaxSizeDetails)) + details = string(buf) } - err.Message = ErrorTable[code] + return code.Error(details) +} + +// Error generates an error from the given status code and detail string. +func (code Code) Error(details string) error { + if !code.IsError() { + return nil + } + err := &Error{ + code: code, + message: ErrorTable[code], + } + if details != "" { + err.message = err.message + ": " + details + } return err } // Error implements error interface -func (err *Error) Error() string { - if err.Details != "" { - return err.Message + ": " + err.Details +func (err *Error) Error() string { return err.message } + +func (err *Error) Code() Code { return err.code } + +// IsNotLeader returns true if the given error is an apierror for CodeNotLeader +func IsNotLeader(err error) bool { + if err == nil { + return false } - return err.Message + apiErr, ok := err.(*Error) + return ok && apiErr.code == CodeNotLeader } diff --git a/api/v1/lib/httpcli/apierrors/apierrors_test.go b/api/v1/lib/httpcli/apierrors/apierrors_test.go new file mode 100644 index 00000000..6c026429 --- /dev/null +++ b/api/v1/lib/httpcli/apierrors/apierrors_test.go @@ -0,0 +1,52 @@ +package apierrors + +import ( + "bytes" + "io/ioutil" + "net/http" + "reflect" + "testing" +) + +func TestFromResponse(t *testing.T) { + for _, tt := range []struct { + r *http.Response + e error + }{ + {nil, nil}, + { + &http.Response{StatusCode: 200}, + nil, + }, + { + &http.Response{StatusCode: 400, Body: ioutil.NopCloser(bytes.NewBufferString("missing framework id"))}, + &Error{400, ErrorTable[CodeMalformedRequest] + ": missing framework id"}, + }, + } { + rr := FromResponse(tt.r) + if !reflect.DeepEqual(tt.e, rr) { + t.Errorf("Expected: %v, got: %v", tt.e, rr) + } + } +} + +func TestError(t *testing.T) { + for _, tt := range []struct { + code Code + isErr bool + details string + wantsMessage string + }{ + {200, false, "", ""}, + {400, true, "", "malformed request"}, + {400, true, "foo", "malformed request: foo"}, + } { + err := tt.code.Error(tt.details) + if tt.isErr != (err != nil) { + t.Errorf("expected isErr %v but error was %q", tt.isErr, err) + } + if err != nil && err.Error() != tt.wantsMessage { + t.Errorf("Expected: %s, got: %s", tt.wantsMessage, err.Error()) + } + } +} diff --git a/api/v1/lib/httpcli/apierrors/apierrroes_test.go b/api/v1/lib/httpcli/apierrors/apierrroes_test.go deleted file mode 100644 index 8c30557d..00000000 --- a/api/v1/lib/httpcli/apierrors/apierrroes_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package apierrors - -import ( - "testing" - "net/http" - "io/ioutil" - "bytes" - "reflect" -) - -func TestFromResponse(t *testing.T) { - for _, tt := range([]struct { - r *http.Response - e error - } { - { nil, nil }, - { - &http.Response{ StatusCode: 200 }, - nil, - }, - { - &http.Response{ StatusCode: 400, Body: ioutil.NopCloser(bytes.NewBufferString("missing framework id")) }, - &Error{ 400, ErrorTable[CodeMalformedRequest], "missing framework id"}, - }, - }) { - rr := FromResponse(tt.r) - if !reflect.DeepEqual(tt.e, rr) { - t.Errorf("Expected: %v, got: %v", tt.e, rr) - } - } -} - -func TestErrorMessage(t *testing.T) { - for _, tt := range([]struct { - e Error - m string - } { - { Error{400, "malformed request", "missing framework id"}, "malformed request: missing framework id" }, - { Error{400, "malformed request", ""}, "malformed request" }, - }) { - if tt.e.Error() != tt.m { - t.Errorf("Expected: %s, got: %s", tt.m, tt.e.Error()) - } - } -} \ No newline at end of file diff --git a/api/v1/lib/httpcli/http.go b/api/v1/lib/httpcli/http.go index ecc8bc17..44fb3626 100644 --- a/api/v1/lib/httpcli/http.go +++ b/api/v1/lib/httpcli/http.go @@ -2,6 +2,7 @@ package httpcli import ( "bytes" + "context" "crypto/tls" "fmt" "io" @@ -25,13 +26,8 @@ type ProtocolError string // Error implements error interface func (pe ProtocolError) Error() string { return string(pe) } -var defaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) - const ( debug = false // TODO(jdef) kill me at some point - - indexRequestContentType = 0 // index into Client.codec.MediaTypes for request content type - indexResponseContentType = 1 // index into Client.codec.MediaTypes for expected response content type ) // DoFunc sends an HTTP request and returns an HTTP response. @@ -77,15 +73,31 @@ type Client struct { handleResponse ResponseHandler } +var ( + DefaultCodec = &encoding.ProtobufCodec + DefaultHeaders = http.Header{} + + // DefaultConfigOpt represents the default client config options. + DefaultConfigOpt = []ConfigOpt{ + Transport(func(t *http.Transport) { + // all calls should be ack'd by the server within this interval. + t.ResponseHeaderTimeout = 15 * time.Second + t.MaxIdleConnsPerHost = 2 // don't depend on go's default + }), + } + + DefaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) +) + // New returns a new Client with the given Opts applied. // Callers are expected to configure the URL, Do, and Codec options prior to // invoking Do. func New(opts ...Opt) *Client { c := &Client{ - codec: &encoding.ProtobufCodec, + codec: DefaultCodec, do: With(DefaultConfigOpt...), - header: http.Header{}, - errorMapper: defaultErrorMapper, + header: DefaultHeaders, + errorMapper: DefaultErrorMapper, } c.buildRequest = c.BuildRequest c.handleResponse = c.HandleResponse @@ -154,8 +166,8 @@ func (c *Client) BuildRequest(m encoding.Marshaler, opt ...RequestOpt) (*http.Re return helper. withOptions(c.requestOpts, opt). withHeaders(c.header). - withHeader("Content-Type", c.codec.MediaTypes[indexRequestContentType]). - withHeader("Accept", c.codec.MediaTypes[indexResponseContentType]). + withHeader("Content-Type", c.codec.RequestContentType()). + withHeader("Accept", c.codec.ResponseContentType()). Request, nil } @@ -183,11 +195,11 @@ func (c *Client) HandleResponse(res *http.Response, err error) (mesos.Response, log.Println("request OK, decoding response") } ct := res.Header.Get("Content-Type") - if ct != c.codec.MediaTypes[indexResponseContentType] { + if ct != c.codec.ResponseContentType() { res.Body.Close() return nil, ProtocolError(fmt.Sprintf("unexpected content type: %q", ct)) } - result.Decoder = c.codec.NewDecoder(recordio.NewFrameReader(res.Body)) + result.Decoder = c.codec.NewDecoder(recordio.NewReader(res.Body)) case http.StatusAccepted: if debug { @@ -305,6 +317,14 @@ func Header(k, v string) RequestOpt { return func(r *http.Request) { r.Header.Ad // Close returns a RequestOpt that determines whether to close the underlying connection after sending the request. func Close(b bool) RequestOpt { return func(r *http.Request) { r.Close = b } } +// Context returns a RequestOpt that sets the request's Context (ctx must be non-nil) +func Context(ctx context.Context) RequestOpt { + return func(r *http.Request) { + r2 := r.WithContext(ctx) + *r = *r2 + } +} + type Config struct { client *http.Client dialer *net.Dialer @@ -313,9 +333,6 @@ type Config struct { type ConfigOpt func(*Config) -// DefaultConfigOpt represents the default client config options. -var DefaultConfigOpt []ConfigOpt - // With returns a DoFunc that executes HTTP round-trips. // The default implementation provides reasonable defaults for timeouts: // keep-alive, connection, request/response read/write, and TLS handshake. diff --git a/api/v1/lib/httpcli/httpsched/httpsched.go b/api/v1/lib/httpcli/httpsched/httpsched.go index 0f53d905..77d08a34 100644 --- a/api/v1/lib/httpcli/httpsched/httpsched.go +++ b/api/v1/lib/httpcli/httpsched/httpsched.go @@ -1,6 +1,7 @@ package httpsched import ( + "context" "log" "net/http" "net/url" @@ -43,7 +44,7 @@ type ( calls.Caller // httpDo is intentionally package-private; clients of this package may extend a Caller // generated by this package by overriding the Call func but may not customize httpDo. - httpDo(encoding.Marshaler, ...httpcli.RequestOpt) (mesos.Response, error) + httpDo(context.Context, encoding.Marshaler, ...httpcli.RequestOpt) (mesos.Response, error) } callerInternal interface { @@ -64,22 +65,22 @@ type ( } ) -func (ct *callerTemporary) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { +func (ct *callerTemporary) httpDo(ctx context.Context, m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { ct.callerInternal.WithTemporary(ct.opt, func() error { if len(opt) == 0 { opt = ct.requestOpts } else if len(ct.requestOpts) > 0 { opt = append(opt[:], ct.requestOpts...) } - resp, err = ct.callerInternal.httpDo(m, opt...) + resp, err = ct.callerInternal.httpDo(ctx, m, opt...) return nil }) return } -func (ct *callerTemporary) Call(call *scheduler.Call) (resp mesos.Response, err error) { +func (ct *callerTemporary) Call(ctx context.Context, call *scheduler.Call) (resp mesos.Response, err error) { ct.callerInternal.WithTemporary(ct.opt, func() error { - resp, err = ct.callerInternal.Call(call) + resp, err = ct.callerInternal.Call(ctx, call) return nil }) return @@ -113,7 +114,7 @@ func NewCaller(cl *httpcli.Client, opts ...Option) calls.Caller { // httpDo decorates the inherited behavior w/ support for HTTP redirection to follow Mesos leadership changes. // NOTE: this implementation will change the state of the client upon Mesos leadership changes. -func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { +func (cli *client) httpDo(ctx context.Context, m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp mesos.Response, err error) { var ( done chan struct{} // avoid allocating these chans unless we actually need to redirect redirectBackoff <-chan struct{} @@ -130,6 +131,7 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp close(done) } }() + opt = append(opt, httpcli.Context(ctx)) for attempt := 0; ; attempt++ { resp, err = cli.Client.Do(m, opt...) redirectErr, ok := err.(*mesosRedirectionError) @@ -139,7 +141,11 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp if attempt < cli.redirect.MaxAttempts { log.Println("redirecting to " + redirectErr.newURL) cli.With(httpcli.Endpoint(redirectErr.newURL)) - <-getBackoff() + select { + case <-getBackoff(): + case <-ctx.Done(): + return nil, ctx.Err() + } continue } return @@ -147,8 +153,8 @@ func (cli *client) httpDo(m encoding.Marshaler, opt ...httpcli.RequestOpt) (resp } // Call implements Client -func (cli *client) Call(call *scheduler.Call) (mesos.Response, error) { - return cli.httpDo(call) +func (cli *client) Call(ctx context.Context, call *scheduler.Call) (mesos.Response, error) { + return cli.httpDo(ctx, call) } type mesosRedirectionError struct{ newURL string } @@ -157,14 +163,6 @@ func (mre *mesosRedirectionError) Error() string { return "mesos server sent redirect to: " + mre.newURL } -func isErrNotLeader(err error) bool { - if err == nil { - return false - } - apiErr, ok := err.(*apierrors.Error) - return ok && apiErr.Code == apierrors.CodeNotLeader -} - // redirectHandler returns a config options that decorates the default response handling routine; // it transforms normal Mesos redirect "errors" into mesosRedirectionErrors by parsing the Location // header and computing the address of the next endpoint that should be used to replay the failed @@ -172,7 +170,7 @@ func isErrNotLeader(err error) bool { func (cli *client) redirectHandler() httpcli.Opt { return httpcli.HandleResponse(func(hres *http.Response, err error) (mesos.Response, error) { resp, err := cli.HandleResponse(hres, err) // default response handler - if err == nil || !isErrNotLeader(err) { + if err == nil || !apierrors.IsNotLeader(err) { return resp, err } // TODO(jdef) for now, we're tightly coupled to the httpcli package's Response type diff --git a/api/v1/lib/httpcli/httpsched/state.go b/api/v1/lib/httpcli/httpsched/state.go index a273b243..b40daf87 100644 --- a/api/v1/lib/httpcli/httpsched/state.go +++ b/api/v1/lib/httpcli/httpsched/state.go @@ -1,6 +1,7 @@ package httpsched import ( + "context" "fmt" "log" "net/http" @@ -42,7 +43,7 @@ type ( err error // err is the error from the most recently executed call } - stateFn func(*state) stateFn + stateFn func(context.Context, *state) stateFn ) func maybeLogged(f httpcli.DoFunc) httpcli.DoFunc { @@ -108,11 +109,11 @@ func disconnectionDecoder(decoder encoding.Decoder, disconnect func()) encoding. }) } -func disconnectedFn(state *state) stateFn { +func disconnectedFn(ctx context.Context, state *state) stateFn { // (a) validate call = SUBSCRIBE if state.call.GetType() != scheduler.Call_SUBSCRIBE { state.resp = nil - state.err = &apierrors.Error{Code: apierrors.CodeUnsubscribed} + state.err = apierrors.CodeUnsubscribed.Error("") return disconnectedFn } @@ -144,7 +145,7 @@ func disconnectedFn(state *state) stateFn { ) // (c) execute the call, save the result in resp, err - stateResp, stateErr := subscribeCaller.Call(state.call) + stateResp, stateErr := subscribeCaller.Call(ctx, state.call) state.err = stateErr // (d) if err != nil return disconnectedFn since we're unsubscribed @@ -192,13 +193,13 @@ var CodesIndicatingSubscriptionLoss = func(codes ...apierrors.Code) map[apierror func errorIndicatesSubscriptionLoss(err error) (result bool) { if apiError, ok := err.(*apierrors.Error); ok { - _, result = CodesIndicatingSubscriptionLoss[apiError.Code] + _, result = CodesIndicatingSubscriptionLoss[apiError.Code()] } // TODO(jdef) should other error types be considered here as well? return } -func connectedFn(state *state) stateFn { +func connectedFn(ctx context.Context, state *state) stateFn { // (a) validate call != SUBSCRIBE if state.call.GetType() == scheduler.Call_SUBSCRIBE { state.resp = nil @@ -216,7 +217,7 @@ func connectedFn(state *state) stateFn { } // (b) execute call, save the result in resp, err - state.resp, state.err = state.caller.Call(state.call) + state.resp, state.err = state.caller.Call(ctx, state.call) if errorIndicatesSubscriptionLoss(state.err) { // properly transition back to a disconnected state if mesos thinks that we're unsubscribed @@ -227,11 +228,11 @@ func connectedFn(state *state) stateFn { return connectedFn } -func (state *state) Call(call *scheduler.Call) (resp mesos.Response, err error) { +func (state *state) Call(ctx context.Context, call *scheduler.Call) (resp mesos.Response, err error) { state.m.Lock() defer state.m.Unlock() state.call = call - state.fn = state.fn(state) + state.fn = state.fn(ctx, state) if debug && state.err != nil { log.Print(*call, state.err) diff --git a/api/v1/lib/httpcli/httpsched/state_test.go b/api/v1/lib/httpcli/httpsched/state_test.go index 9210fe35..92a3e1fe 100644 --- a/api/v1/lib/httpcli/httpsched/state_test.go +++ b/api/v1/lib/httpcli/httpsched/state_test.go @@ -5,29 +5,16 @@ import ( "testing" "github.com/mesos/mesos-go/api/v1/lib/encoding" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) -type latch struct{ line chan struct{} } - -func newLatch() *latch { return &latch{make(chan struct{})} } -func (l *latch) Reset() { l.line = make(chan struct{}) } -func (l *latch) Close() { close(l.line) } -func (l *latch) Closed() (result bool) { - select { - case <-l.line: - result = true - default: - } - return -} - func TestDisconnectionDecoder(t *testing.T) { // invoke disconnect upon decoder errors expected := errors.New("unmarshaler error") decoder := encoding.DecoderFunc(func(_ encoding.Unmarshaler) error { return expected }) - latch := newLatch() + latch := new(latch.L).Reset() d := disconnectionDecoder(decoder, latch.Close) err := d.Decode(nil) diff --git a/api/v1/lib/operations_test.go b/api/v1/lib/operations_test.go deleted file mode 100644 index b95b5aee..00000000 --- a/api/v1/lib/operations_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package mesos_test - -import ( - "testing" - - "github.com/mesos/mesos-go/api/v1/lib" -) - -func TestOpCreate(t *testing.T) { - var ( - total = resources( - resource(name("cpus"), valueScalar(1)), - resource(name("mem"), valueScalar(512)), - resource(name("disk"), valueScalar(1000), role("role")), - ) - volume1 = resource(name("disk"), valueScalar(200), role("role"), disk("1", "path")) - volume2 = resource(name("disk"), valueScalar(2000), role("role"), disk("1", "path")) - ) - op := create(resources(volume1)) - rs, err := op.Apply(total) - if err != nil { - t.Fatalf("unexpected error: %+v", err) - } - expected := resources( - resource(name("cpus"), valueScalar(1)), - resource(name("mem"), valueScalar(512)), - resource(name("disk"), valueScalar(800), role("role")), - volume1, - ) - if !expected.Equivalent(rs) { - t.Fatalf("expected %v instead of %v", expected, rs) - } - - // check the case of insufficient disk resources - op = create(resources(volume2)) - _, err = op.Apply(total) - if err == nil { - t.Fatalf("expected an error due to insufficient disk resources") - } -} - -func TestOpUnreserve(t *testing.T) { - var ( - reservedCPU = resources( - resource(name("cpus"), - valueScalar(1), - role("role"), - reservation(reservedBy("principal")))) - reservedMem = resources( - resource(name("mem"), - valueScalar(512), - role("role"), - reservation(reservedBy("principal")))) - reserved = reservedCPU.Plus(reservedMem...) - ) - - // test case 1: unreserve some amount of CPU that's already been reserved - unreservedCPU := reservedCPU.Flatten() - t.Log("unreservedCPU=" + unreservedCPU.String()) - - wantsUnreserved := reservedMem.Plus(unreservedCPU...) - actualUnreserved, err := unreserve(reservedCPU).Apply(reserved) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !wantsUnreserved.Equivalent(actualUnreserved) { - t.Errorf("expected resources %+v instead of %+v", wantsUnreserved, actualUnreserved) - } - - // test case 2: unreserve some amount of CPU greater than that which already been reserved - reservedCPU2 := resources( - resource(name("cpus"), - valueScalar(2), - role("role"), - reservation(reservedBy("principal")))) - _, err = unreserve(reservedCPU2).Apply(reserved) - if err == nil { - t.Fatalf("expected reservation error") - } -} - -func TestOpReserve(t *testing.T) { - // func opReserve(operation mesos.Offer_Operation, resources mesos.Resources) (mesos.Resources, error) - var ( - unreservedCPU = resources(resource(name("cpus"), valueScalar(1))) - unreservedMem = resources(resource(name("mem"), valueScalar(512))) - unreserved = unreservedCPU.Plus(unreservedMem...) - reservedCPU1 = unreservedCPU.Flatten(mesos.RoleName("role").Assign(), reservedBy("principal").Assign()) - ) - - // test case 1: reserve an amount of CPU that's available - wantsReserved := unreservedMem.Plus(reservedCPU1...) - actualReserved, err := reserve(reservedCPU1).Apply(unreserved) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !wantsReserved.Equivalent(actualReserved) { - t.Errorf("expected resources %+v instead of %+v", wantsReserved, actualReserved) - } - - // test case 2: reserve an amount of CPU that's NOT available - reservedCPU2 := resources( - resource(name("cpus"), - valueScalar(2), - role("role"), - reservation(reservedBy("principal")))) - _, err = reserve(reservedCPU2).Apply(unreserved) - if err == nil { - t.Fatalf("expected reservation error") - } -} - -func reservedBy(principal string) *mesos.Resource_ReservationInfo { - result := &mesos.Resource_ReservationInfo{} - if principal != "" { - result.Principal = &principal - } - return result -} - -func reserve(r mesos.Resources) *mesos.Offer_Operation { - return &mesos.Offer_Operation{ - Type: mesos.Offer_Operation_RESERVE.Enum(), - Reserve: &mesos.Offer_Operation_Reserve{ - Resources: r, - }, - } -} - -func unreserve(r mesos.Resources) *mesos.Offer_Operation { - return &mesos.Offer_Operation{ - Type: mesos.Offer_Operation_UNRESERVE.Enum(), - Unreserve: &mesos.Offer_Operation_Unreserve{ - Resources: r, - }, - } -} - -func create(r mesos.Resources) *mesos.Offer_Operation { - return &mesos.Offer_Operation{ - Type: mesos.Offer_Operation_CREATE.Enum(), - Create: &mesos.Offer_Operation_Create{ - Volumes: r, - }, - } -} - -func reservation(ri *mesos.Resource_ReservationInfo) resourceOpt { - return func(r *mesos.Resource) { - r.Reservation = ri - } -} - -func disk(persistenceID, containerPath string) resourceOpt { - return func(r *mesos.Resource) { - r.Disk = &mesos.Resource_DiskInfo{} - if containerPath != "" { - r.Disk.Volume = &mesos.Volume{ContainerPath: containerPath} - } - if persistenceID != "" { - r.Disk.Persistence = &mesos.Resource_DiskInfo_Persistence{ID: persistenceID} - } - } -} diff --git a/api/v1/lib/recordio/reader.go b/api/v1/lib/recordio/reader.go index 709499e3..b1f3eae3 100644 --- a/api/v1/lib/recordio/reader.go +++ b/api/v1/lib/recordio/reader.go @@ -4,94 +4,168 @@ import ( "bufio" "bytes" "io" - "strconv" + "io/ioutil" + "log" "github.com/mesos/mesos-go/api/v1/lib/encoding/framing" ) -// NewReader returns an io.Reader that unpacks the data read from r out of -// RecordIO framing before returning it. -func NewReader(r io.Reader) io.Reader { - br, ok := r.(*bufio.Reader) - if !ok { - br = bufio.NewReader(r) +type Debugger bool + +const Debug = Debugger(false) + +func (d Debugger) Log(v ...interface{}) { + if d { + log.Print(v...) } - return &reader{r: br} } -func NewFrameReader(r io.Reader) framing.Reader { - br, ok := r.(*bufio.Reader) - if !ok { - br = bufio.NewReader(r) +func (d Debugger) Logf(s string, v ...interface{}) { + if d { + log.Printf(s, v...) } - return &reader{r: br} } -type reader struct { - r *bufio.Reader - pending uint64 -} +type ( + Reader interface { + framing.Reader + io.Closer + } -func (rr *reader) ReadFrame(p []byte) (endOfFrame bool, n int, err error) { - for err == nil && len(p) > 0 && !endOfFrame { - if rr.pending == 0 { - if n > 0 { - endOfFrame = true - // We've read enough. Don't potentially block reading the next header. + Opt func(*reader) - // Only send back 1 frame at a time; note if pending==0 here then we basically - // skip over reporting an empty frame because the next time ReadFrame() is invoked - // we'll have no idea if we previously read pending==0 here, or if we're new. - break - } - rr.pending, err = rr.size() - continue + reader struct { + io.Closer + *bufio.Scanner + pend int + splitf func(data []byte, atEOF bool) (int, []byte, error) + maxf int // max frame size + } +) + +// NewReader returns a reader that parses frames from a recordio stream. +func NewReader(read io.Reader, opt ...Opt) Reader { + Debug.Log("new frame reader") + rc, ok := read.(io.ReadCloser) + if !ok { + rc = ioutil.NopCloser(read) + } + r := &reader{Scanner: bufio.NewScanner(rc)} + r.Split(func(data []byte, atEOF bool) (int, []byte, error) { + // Scanner panics if we invoke Split after scanning has started, + // use this proxy func as a work-around. + return r.splitf(data, atEOF) + }) + buf := make([]byte, 16*1024) + r.Buffer(buf, 1<<22) // 1<<22 == max protobuf size + r.splitf = r.splitSize + r.Closer = rc + // apply options + for _, f := range opt { + if f != nil { + f(r) } - read, hi := 0, min(rr.pending, uint64(len(p))) - read, err = rr.r.Read(p[:hi]) - n += read - p = p[read:] - rr.pending -= uint64(read) } - return + return r } -func (rr *reader) Read(p []byte) (n int, err error) { - for err == nil && len(p) > 0 { - if rr.pending == 0 { - if n > 0 && !rr.more() { - // We've read enough. Don't potentially block reading the next header. - break - } - rr.pending, err = rr.size() - continue - } - read, hi := 0, min(rr.pending, uint64(len(p))) - read, err = rr.r.Read(p[:hi]) - n += read - p = p[read:] - rr.pending -= uint64(read) +// MaxMessageSize returns a functional option that configures the internal Scanner's buffer and max token (message) +// length, in bytes. +func MaxMessageSize(max int) Opt { + return func(r *reader) { + buf := make([]byte, max>>1) + r.Buffer(buf, max) + r.maxf = max } - return n, err } -func (rr *reader) more() bool { - peek, err := rr.r.Peek(rr.r.Buffered()) - return err != nil && bytes.IndexByte(peek, '\n') >= 0 +func (r *reader) splitSize(data []byte, atEOF bool) (int, []byte, error) { + const maxTokenLength = 20 // textual length of largest uint64 number + if atEOF { + x := len(data) + switch { + case x == 0: + Debug.Log("EOF and empty frame, returning io.EOF") + return 0, nil, io.EOF + case x < 2: // min frame size + Debug.Log("remaining data less than min total frame length") + return 0, nil, framing.ErrorUnderrun + } + // otherwise, we may have a valid frame... + } + Debug.Log("len(data)=", len(data)) + adv := 0 + for { + i := 0 + for ; i < maxTokenLength && i < len(data) && data[i] != '\n'; i++ { + } + Debug.Log("i=", i) + if i == len(data) { + Debug.Log("need more input") + return 0, nil, nil // need more input + } + if i == maxTokenLength && data[i] != '\n' { + Debug.Log("frame size: max token length exceeded") + return 0, nil, framing.ErrorBadSize + } + n, err := ParseUintBytes(bytes.TrimSpace(data[:i]), 10, 64) + if err != nil { + Debug.Log("failed to parse frame size field:", err) + return 0, nil, framing.ErrorBadSize + } + if r.maxf != 0 && int(n) > r.maxf { + Debug.Log("frame size max length exceeded:", n) + return 0, nil, framing.ErrorOversizedFrame + } + if n == 0 { + // special case... don't invoke splitData, just parse the next size header + adv += i + 1 + data = data[i+1:] + continue + } + r.pend = int(n) + r.splitf = r.splitFrame + Debug.Logf("split next frame: %d, %d", n, adv+i+1) + return adv + i + 1, data[:0], nil // returning a nil token screws up the Scanner, so return empty + } } -func (rr *reader) size() (uint64, error) { - header, err := rr.r.ReadSlice('\n') - if err != nil { - return 0, err +func (r *reader) splitFrame(data []byte, atEOF bool) (advance int, token []byte, err error) { + x := len(data) + Debug.Log("splitFrame:x=", x, ",eof=", atEOF) + if atEOF { + if x < r.pend { + return 0, nil, framing.ErrorUnderrun + } + } + if r.pend == 0 { + panic("asked to read frame data, but no data left in frame") + } + if x < int(r.pend) { + // need more data + return 0, nil, nil } - // NOTE(tsenart): https://github.com/golang/go/issues/2632 - return strconv.ParseUint(string(bytes.TrimSpace(header)), 10, 64) + r.splitf = r.splitSize + adv := int(r.pend) + r.pend = 0 + return adv, data[:adv], nil } -func min(a, b uint64) uint64 { - if a < b { - return a +// ReadFrame implements framing.Reader +func (r *reader) ReadFrame() (tok []byte, err error) { + for r.Scan() { + b := r.Bytes() + if len(b) == 0 { + continue + } + tok = b + Debug.Log("len(tok)", len(tok)) + break } - return b + // either scan failed, or it succeeded and we have a token... + err = r.Err() + if err == nil && len(tok) == 0 { + err = io.EOF + } + return } diff --git a/api/v1/lib/recordio/reader_test.go b/api/v1/lib/recordio/reader_test.go index c5bcc424..09a9519b 100644 --- a/api/v1/lib/recordio/reader_test.go +++ b/api/v1/lib/recordio/reader_test.go @@ -1,61 +1,87 @@ -package recordio +package recordio_test import ( "bytes" "fmt" "io" - "io/ioutil" "math/rand" "reflect" "strconv" "strings" "testing" + + "github.com/mesos/mesos-go/api/v1/lib/encoding/framing" + "github.com/mesos/mesos-go/api/v1/lib/recordio" ) func Example() { - r := NewReader(strings.NewReader("6\nhello 0\n6\nworld!")) - records, err := ioutil.ReadAll(r) - fmt.Println(string(records), err) + var ( + r = recordio.NewReader(strings.NewReader("6\nhello 0\n6\nworld!")) + lines []string + ) + for { + fr, err := r.ReadFrame() + if err == io.EOF { + break + } + if err != nil { + panic(err) + } + lines = append(lines, string(fr)) + } + fmt.Println(lines) // Output: - // hello world! + // [hello world!] } -func TestReader(t *testing.T) { - for i, tt := range []struct { +func TestReadFrame(t *testing.T) { + list := func(v ...string) []string { return v } + for ti, tc := range []struct { in string - out []byte - fwd, n int + frames []string err error }{ - {"1\na0\n1\nb", []byte("a"), 0, 1, nil}, - {"1\na0\n1\nb", []byte("b"), 1, 1, nil}, - {"1\na", []byte{}, 0, 0, nil}, - {"2\nab", []byte("a"), 0, 1, nil}, - {"2\nab", []byte("ab"), 0, 2, nil}, - {"2\nab", []byte("b"), 1, 1, nil}, - {"2\nab", []byte{'b', 0}, 1, 1, nil}, - {"2\nab", []byte{0}, 2, 0, io.EOF}, - { // size = (2 << 63) + 1 - "18446744073709551616\n", []byte{0}, 0, 0, &strconv.NumError{ - Func: "ParseUint", - Num: "18446744073709551616", - Err: strconv.ErrRange, - }, - }, + {"", nil, nil}, + {"a", nil, framing.ErrorUnderrun}, + {"aaaaaaaaaaaaaaaaaaaaa", nil, framing.ErrorBadSize}, // 21 digits is too large for frame size + {"111111111111111111111", nil, framing.ErrorBadSize}, + {"a\n", nil, framing.ErrorBadSize}, + {"0\n", nil, nil}, + {"00000000000000000000\n", nil, nil}, + {"000000000000000000000\n", nil, framing.ErrorBadSize}, + {"0\n0\n0\n", nil, nil}, + {"1\n", nil, framing.ErrorUnderrun}, + {"1\na", list("a"), nil}, + {"2\na", nil, framing.ErrorUnderrun}, + {"1\na1\nb1\nc", list("a", "b", "c"), nil}, + {"5\nabcde", list("abcde"), nil}, + {"5\nabcde3\nfgh", list("abcde", "fgh"), nil}, + {"5\nabcde5\nfgh", list("abcde"), framing.ErrorUnderrun}, + {"23\n", nil, framing.ErrorOversizedFrame}, // 23 exceeds max of 22 } { - type expect struct { - p []byte - n int - err error + var ( + r = recordio.NewReader(strings.NewReader(tc.in), recordio.MaxMessageSize(22)) + frames []string + lastErr error + ) + for lastErr == nil { + fr, err := r.ReadFrame() + if err == nil || err == io.EOF { + if fr != nil { + println("read frame " + string(fr)) + frames = append(frames, string(fr)) + } + } + lastErr = err + } + if tc.err == nil && lastErr != io.EOF { + t.Fatalf("test case %d failed: unexpected error %q", ti, lastErr) } - r := NewReader(strings.NewReader(tt.in)) - if n, err := r.Read(make([]byte, tt.fwd)); err != nil || n != tt.fwd { - t.Fatalf("test #%d: failed to read forward %d bytes: %v", i, n, err) + if tc.err != nil && lastErr != tc.err { + t.Fatalf("test case %d failed: expected error %q instead of error %q", ti, tc.err, lastErr) } - want := expect{[]byte(tt.out), tt.n, tt.err} - got := expect{p: make([]byte, len(tt.out))} - if got.n, got.err = r.Read(got.p); !reflect.DeepEqual(got, want) { - t.Errorf("test #%d: got: %+v, want: %+v", i, got, want) + if !reflect.DeepEqual(tc.frames, frames) { + t.Fatalf("test case %d failed: expected frames %#v instead of frames %#v", ti, tc.frames, frames) } } } @@ -64,18 +90,17 @@ func BenchmarkReader(b *testing.B) { var buf bytes.Buffer genRecords(b, &buf) - r := NewReader(&buf) - p := make([]byte, 256) + r := recordio.NewReader(&buf) b.StopTimer() b.ResetTimer() b.StartTimer() for i := 0; i < b.N; i++ { - if n, err := r.Read(p); err != nil && err != io.EOF { + if tok, err := r.ReadFrame(); err != nil && err != io.EOF { b.Fatal(err) } else { - b.SetBytes(int64(n)) + b.SetBytes(int64(len(tok))) } } } diff --git a/api/v1/lib/recordio/strconv.go b/api/v1/lib/recordio/strconv.go new file mode 100644 index 00000000..6e2b2681 --- /dev/null +++ b/api/v1/lib/recordio/strconv.go @@ -0,0 +1,117 @@ +/* +Copyright 2013 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recordio + +import ( + "errors" + "strconv" +) + +// ParseUintBytes is like strconv.ParseUint, but using a []byte. +func ParseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) { + var cutoff, maxVal uint64 + + if bitSize == 0 { + bitSize = int(strconv.IntSize) + } + + s0 := s + switch { + case len(s) < 1: + err = strconv.ErrSyntax + goto Error + + case 2 <= base && base <= 36: + // valid base; nothing to do + + case base == 0: + // Look for octal, hex prefix. + switch { + case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): + base = 16 + s = s[2:] + if len(s) < 1 { + err = strconv.ErrSyntax + goto Error + } + case s[0] == '0': + base = 8 + default: + base = 10 + } + + default: + err = errors.New("invalid base " + strconv.Itoa(base)) + goto Error + } + + n = 0 + cutoff = cutoff64(base) + maxVal = 1<= base { + n = 0 + err = strconv.ErrSyntax + goto Error + } + + if n >= cutoff { + // n*base overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n *= uint64(base) + + n1 := n + uint64(v) + if n1 < n || n1 > maxVal { + // n+v overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n = n1 + } + + return n, nil + +Error: + return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err} +} + +// Return the first number n such that n*base >= 1<<64. +func cutoff64(base int) uint64 { + if base < 2 { + return 0 + } + return (1<<64-1)/uint64(base) + 1 +} diff --git a/api/v1/lib/resourcefilters/resources.go b/api/v1/lib/resourcefilters/resources.go new file mode 100644 index 00000000..7d74c341 --- /dev/null +++ b/api/v1/lib/resourcefilters/resources.go @@ -0,0 +1,130 @@ +package resourcefilters + +import ( + "github.com/mesos/mesos-go/api/v1/lib" +) + +type ( + Interface interface { + Accepts(*mesos.Resource) bool + } + Filter func(*mesos.Resource) bool + Filters []Interface +) + +var _ = Interface(Filter(nil)) + +func (f Filter) Accepts(r *mesos.Resource) bool { + if f != nil { + return f(r) + } + return true +} + +type any int + +const Any = any(0) + +func (_ any) Accepts(r *mesos.Resource) bool { + return r != nil && !r.IsEmpty() +} + +var _ = Interface(Any) + +type unreserved int + +const Unreserved = unreserved(0) + +func (_ unreserved) Accepts(r *mesos.Resource) bool { + return r.IsUnreserved() +} + +var _ = Interface(Unreserved) + +type persistentVolumes int + +func (_ persistentVolumes) Accepts(r *mesos.Resource) bool { + return r.IsPersistentVolume() +} + +const PersistentVolumes = persistentVolumes(0) + +var _ = Interface(PersistentVolumes) + +type revocable int + +func (_ revocable) Accepts(r *mesos.Resource) bool { + return r.IsRevocable() +} + +const Revocable = revocable(0) + +var _ = Interface(Revocable) + +type scalar int + +func (_ scalar) Accepts(r *mesos.Resource) bool { + return r.GetType() == mesos.SCALAR +} + +const Scalar = scalar(0) + +var _ = Interface(Scalar) + +type rrange int + +func (_ rrange) Accepts(r *mesos.Resource) bool { + return r.GetType() == mesos.RANGES +} + +const Range = rrange(0) + +var _ = Interface(Range) + +type set int + +func (_ set) Accepts(r *mesos.Resource) bool { + return r.GetType() == mesos.SET +} + +const Set = set(0) + +var _ = Interface(Set) + +func (rf Filter) Or(f Filter) Filter { + return Filter(func(r *mesos.Resource) bool { + return rf(r) || f(r) + }) +} + +func Select(rf Interface, resources ...mesos.Resource) (result mesos.Resources) { + for i := range resources { + if rf.Accepts(&resources[i]) { + result.Add1(resources[i]) + } + } + return +} + +func (rf Filters) Accepts(r *mesos.Resource) bool { + for _, f := range rf { + if !f.Accepts(r) { + return false + } + } + return true +} + +var _ = Interface(Filters(nil)) + +func ReservedByRole(role string) Filter { + return Filter(func(r *mesos.Resource) bool { + return r.IsReserved(role) + }) +} + +func Named(name string) Filter { + return Filter(func(r *mesos.Resource) bool { + return r.GetName() == name + }) +} diff --git a/api/v1/lib/resources.go b/api/v1/lib/resources.go index 14d0be78..974e9686 100644 --- a/api/v1/lib/resources.go +++ b/api/v1/lib/resources.go @@ -11,12 +11,10 @@ import ( type ( RoleName string Resources []Resource - ResourceFilter func(*Resource) bool - ResourceFilters []ResourceFilter - ResourceErrorType int + resourceErrorType int - ResourceError struct { - errorType ResourceErrorType + resourceError struct { + errorType resourceErrorType reason string spec Resource } @@ -33,54 +31,32 @@ type ( const ( RoleDefault = RoleName("*") - ResourceErrorTypeIllegalName ResourceErrorType = iota - ResourceErrorTypeIllegalType - ResourceErrorTypeUnsupportedType - ResourceErrorTypeIllegalScalar - ResourceErrorTypeIllegalRanges - ResourceErrorTypeIllegalSet - ResourceErrorTypeIllegalDisk - ResourceErrorTypeIllegalReservation + resourceErrorTypeIllegalName resourceErrorType = iota + resourceErrorTypeIllegalType + resourceErrorTypeUnsupportedType + resourceErrorTypeIllegalScalar + resourceErrorTypeIllegalRanges + resourceErrorTypeIllegalSet + resourceErrorTypeIllegalDisk + resourceErrorTypeIllegalReservation noReason = "" // make error generation code more readable ) var ( - AnyResources = ResourceFilter(func(r *Resource) bool { - return r != nil && !r.IsEmpty() - }) - UnreservedResources = ResourceFilter(func(r *Resource) bool { - return r.IsUnreserved() - }) - PersistentVolumes = ResourceFilter(func(r *Resource) bool { - return r.IsPersistentVolume() - }) - RevocableResources = ResourceFilter(func(r *Resource) bool { - return r.IsRevocable() - }) - ScalarResources = ResourceFilter(func(r *Resource) bool { - return r.GetType() == SCALAR - }) - RangeResources = ResourceFilter(func(r *Resource) bool { - return r.GetType() == RANGES - }) - SetResources = ResourceFilter(func(r *Resource) bool { - return r.GetType() == SET - }) - - resourceErrorMessages = map[ResourceErrorType]string{ - ResourceErrorTypeIllegalName: "missing or illegal resource name", - ResourceErrorTypeIllegalType: "missing or illegal resource type", - ResourceErrorTypeUnsupportedType: "unsupported resource type", - ResourceErrorTypeIllegalScalar: "illegal scalar resource", - ResourceErrorTypeIllegalRanges: "illegal ranges resource", - ResourceErrorTypeIllegalSet: "illegal set resource", - ResourceErrorTypeIllegalDisk: "illegal disk resource", - ResourceErrorTypeIllegalReservation: "illegal resource reservation", + resourceErrorMessages = map[resourceErrorType]string{ + resourceErrorTypeIllegalName: "missing or illegal resource name", + resourceErrorTypeIllegalType: "missing or illegal resource type", + resourceErrorTypeUnsupportedType: "unsupported resource type", + resourceErrorTypeIllegalScalar: "illegal scalar resource", + resourceErrorTypeIllegalRanges: "illegal ranges resource", + resourceErrorTypeIllegalSet: "illegal set resource", + resourceErrorTypeIllegalDisk: "illegal disk resource", + resourceErrorTypeIllegalReservation: "illegal resource reservation", } ) -func (t ResourceErrorType) Generate(reason string) error { +func (t resourceErrorType) Generate(reason string) error { msg := resourceErrorMessages[t] if reason != noReason { if msg != "" { @@ -89,20 +65,26 @@ func (t ResourceErrorType) Generate(reason string) error { msg = reason } } - return &ResourceError{errorType: t, reason: msg} + return &resourceError{errorType: t, reason: msg} } -func (err *ResourceError) Type() ResourceErrorType { return err.errorType } -func (err *ResourceError) Reason() string { return err.reason } -func (err *ResourceError) Resource() Resource { return err.spec } +func (err *resourceError) Type() resourceErrorType { return err.errorType } +func (err *resourceError) Reason() string { return err.reason } +func (err *resourceError) Resource() Resource { return err.spec } -func (err *ResourceError) Error() string { +func (err *resourceError) Error() string { + // TODO(jdef) include additional context here? (type, resource) if err.reason != "" { return "resource error: " + err.reason } return "resource error" } +func IsResourceError(err error) (ok bool) { + _, ok = err.(*resourceError) + return +} + func (r RoleName) IsDefault() bool { return r == RoleDefault } @@ -118,206 +100,6 @@ func (r RoleName) Proto() *string { return &s } -func (rf ResourceFilter) Or(f ResourceFilter) ResourceFilter { - return ResourceFilter(func(r *Resource) bool { - return rf(r) || f(r) - }) -} - -func (rf ResourceFilter) And(f ResourceFilter) ResourceFilter { - return ResourceFilters{rf, f}.Predicate() -} - -func (rf ResourceFilter) Select(resources Resources) (result Resources) { - for i := range resources { - if rf(&resources[i]) { - result.add(resources[i]) - } - } - return -} - -func (rf ResourceFilters) Predicate() ResourceFilter { - return ResourceFilter(func(r *Resource) bool { - for _, f := range rf { - if !f(r) { - return false - } - } - return true - }) -} - -func ReservedResources(role string) ResourceFilter { - return ResourceFilter(func(r *Resource) bool { - return r.IsReserved(role) - }) -} - -func NamedResources(name string) ResourceFilter { - return ResourceFilter(func(r *Resource) bool { - return r.GetName() == name - }) -} - -func (resources Resources) CPUs() (float64, bool) { - v := resources.SumScalars(NamedResources("cpus")) - if v != nil { - return v.Value, true - } - return 0, false -} - -func (resources Resources) Memory() (uint64, bool) { - v := resources.SumScalars(NamedResources("mem")) - if v != nil { - return uint64(v.Value), true - } - return 0, false -} - -func (resources Resources) Disk() (uint64, bool) { - v := resources.SumScalars(NamedResources("disk")) - if v != nil { - return uint64(v.Value), true - } - return 0, false -} - -func (resources Resources) Ports() (Ranges, bool) { - v := resources.SumRanges(NamedResources("ports")) - if v != nil { - return Ranges(v.Range), true - } - return nil, false -} - -func (resources Resources) SumScalars(rf ResourceFilter) *Value_Scalar { - predicate := ResourceFilters{rf, ScalarResources}.Predicate() - var x *Value_Scalar - for i := range resources { - if !predicate(&resources[i]) { - continue - } - x = x.Add(resources[i].GetScalar()) - } - return x -} - -func (resources Resources) SumRanges(rf ResourceFilter) *Value_Ranges { - predicate := ResourceFilters{rf, RangeResources}.Predicate() - var x *Value_Ranges - for i := range resources { - if !predicate(&resources[i]) { - continue - } - x = x.Add(resources[i].GetRanges()) - } - return x -} - -func (resources Resources) SumSets(rf ResourceFilter) *Value_Set { - predicate := ResourceFilters{rf, SetResources}.Predicate() - var x *Value_Set - for i := range resources { - if !predicate(&resources[i]) { - continue - } - x = x.Add(resources[i].GetSet()) - } - return x -} - -func (resources Resources) Types() map[string]Value_Type { - m := map[string]Value_Type{} - for i := range resources { - m[resources[i].GetName()] = resources[i].GetType() - } - return m -} - -func (resources Resources) Names() (names []string) { - m := map[string]struct{}{} - for i := range resources { - n := resources[i].GetName() - if _, ok := m[n]; !ok { - m[n] = struct{}{} - names = append(names, n) - } - } - return -} - -func (resources Resources) sameTotals(result Resources) bool { - // from: https://github.com/apache/mesos/blob/master/src/common/resources.cpp - // This is a sanity check to ensure the amount of each type of - // resource does not change. - // TODO(jieyu): Currently, we only check known resource types like - // cpus, mem, disk, ports, etc. We should generalize this. - var ( - c1, c2 = result.CPUs() - m1, m2 = result.Memory() - d1, d2 = result.Disk() - p1, p2 = result.Ports() - - c3, c4 = resources.CPUs() - m3, m4 = resources.Memory() - d3, d4 = resources.Disk() - p3, p4 = resources.Ports() - ) - return c1 == c3 && c2 == c4 && - m1 == m3 && m2 == m4 && - d1 == d3 && d2 == d4 && - p1.Equivalent(p3) && p2 == p4 -} - -func (resources Resources) Find(targets Resources) (total Resources) { - for i := range targets { - found := resources.find(targets[i]) - - // each target *must* be found - if len(found) == 0 { - return nil - } - - total.Add(found...) - } - return total -} - -func (resources Resources) find(target Resource) Resources { - var ( - total = resources.Clone() - remaining = Resources{target}.Flatten() - found Resources - predicates = ResourceFilters{ - ReservedResources(target.GetRole()), - UnreservedResources, - AnyResources, - } - ) - for _, predicate := range predicates { - filtered := predicate.Select(total) - for i := range filtered { - // need to flatten to ignore the roles in ContainsAll() - flattened := Resources{filtered[i]}.Flatten() - if flattened.ContainsAll(remaining) { - // target has been found, return the result - return found.Add(remaining.Flatten( - RoleName(filtered[i].GetRole()).Assign(), - filtered[i].Reservation.Assign())...) - } - if remaining.ContainsAll(flattened) { - found.add(filtered[i]) - total.subtract(filtered[i]) - remaining.Subtract(flattened...) - break - } - } - } - return nil -} - func (ri *Resource_ReservationInfo) Assign() FlattenOpt { return func(fc *FlattenConfig) { fc.Reservation = ri @@ -336,7 +118,7 @@ func (resources Resources) Flatten(opts ...FlattenOpt) (flattened Resources) { for _, r := range resources { r.Role = &fc.Role r.Reservation = fc.Reservation - flattened.add(r) + flattened.Add1(r) } return } @@ -345,8 +127,8 @@ func (resources Resources) Validate() error { for i := range resources { err := resources[i].Validate() if err != nil { - // augment ResourceError's with the resource that failed to validate - if resourceError, ok := err.(*ResourceError); ok { + // augment resourceError's with the resource that failed to validate + if resourceError, ok := err.(*resourceError); ok { r := proto.Clone(&resources[i]).(*Resource) resourceError.spec = *r } @@ -398,7 +180,7 @@ func (resources Resources) ContainsAll(that Resources) bool { if !remaining.contains(that[i]) { return false } - remaining.subtract(that[i]) + remaining.Subtract1(that[i]) } return true } @@ -420,7 +202,7 @@ func (resources *Resources) Subtract(that ...Resource) (rs Resources) { that = x for i := range that { - resources.subtract(that[i]) + resources.Subtract1(that[i]) } } rs = *resources @@ -450,9 +232,9 @@ func (resources *Resources) Add(that ...Resource) (rs Resources) { return } -// add adds `that` to the receiving `resources` and returns the result (the modified +// Add1 adds `that` to the receiving `resources` and returns the result (the modified // `resources` receiver). -func (resources *Resources) add(that Resource) (rs Resources) { +func (resources *Resources) Add1(that Resource) (rs Resources) { if resources != nil { rs = *resources } @@ -483,12 +265,12 @@ func (resources Resources) _add(that Resource) Resources { // the receiving `resources` or `that`. func (resources *Resources) minus(that Resource) Resources { x := resources.Clone() - return x.subtract(that) + return x.Subtract1(that) } -// subtract subtracts `that` from the receiving `resources` and returns the result (the modified +// Subtract1 subtracts `that` from the receiving `resources` and returns the result (the modified // `resources` receiver). -func (resources *Resources) subtract(that Resource) Resources { +func (resources *Resources) Subtract1(that Resource) Resources { if resources == nil { return nil } @@ -588,60 +370,60 @@ func (resources Resources) String() string { func (left *Resource) Validate() error { if left.GetName() == "" { - return ResourceErrorTypeIllegalName.Generate(noReason) + return resourceErrorTypeIllegalName.Generate(noReason) } if _, ok := Value_Type_name[int32(left.GetType())]; !ok { - return ResourceErrorTypeIllegalType.Generate(noReason) + return resourceErrorTypeIllegalType.Generate(noReason) } switch left.GetType() { case SCALAR: if s := left.GetScalar(); s == nil || left.GetRanges() != nil || left.GetSet() != nil { - return ResourceErrorTypeIllegalScalar.Generate(noReason) + return resourceErrorTypeIllegalScalar.Generate(noReason) } else if s.GetValue() < 0 { - return ResourceErrorTypeIllegalScalar.Generate("value < 0") + return resourceErrorTypeIllegalScalar.Generate("value < 0") } case RANGES: if r := left.GetRanges(); left.GetScalar() != nil || r == nil || left.GetSet() != nil { - return ResourceErrorTypeIllegalRanges.Generate(noReason) + return resourceErrorTypeIllegalRanges.Generate(noReason) } else { for i, rr := range r.GetRange() { // ensure that ranges are not inverted if rr.Begin > rr.End { - return ResourceErrorTypeIllegalRanges.Generate("begin > end") + return resourceErrorTypeIllegalRanges.Generate("begin > end") } // ensure that ranges don't overlap (but not necessarily squashed) for j := i + 1; j < len(r.GetRange()); j++ { r2 := r.GetRange()[j] if rr.Begin <= r2.Begin && r2.Begin <= rr.End { - return ResourceErrorTypeIllegalRanges.Generate("overlapping ranges") + return resourceErrorTypeIllegalRanges.Generate("overlapping ranges") } } } } case SET: if s := left.GetSet(); left.GetScalar() != nil || left.GetRanges() != nil || s == nil { - return ResourceErrorTypeIllegalSet.Generate(noReason) + return resourceErrorTypeIllegalSet.Generate(noReason) } else { unique := make(map[string]struct{}, len(s.GetItem())) for _, x := range s.GetItem() { if _, found := unique[x]; found { - return ResourceErrorTypeIllegalSet.Generate("duplicated elements") + return resourceErrorTypeIllegalSet.Generate("duplicated elements") } unique[x] = struct{}{} } } default: - return ResourceErrorTypeUnsupportedType.Generate(noReason) + return resourceErrorTypeUnsupportedType.Generate(noReason) } // check for disk resource if left.GetDisk() != nil && left.GetName() != "disk" { - return ResourceErrorTypeIllegalDisk.Generate("DiskInfo should not be set for \"" + left.GetName() + "\" resource") + return resourceErrorTypeIllegalDisk.Generate("DiskInfo should not be set for \"" + left.GetName() + "\" resource") } // check for invalid state of (role,reservation) pair if left.GetRole() == string(RoleDefault) && left.GetReservation() != nil { - return ResourceErrorTypeIllegalReservation.Generate("default role cannot be dynamically assigned") + return resourceErrorTypeIllegalReservation.Generate("default role cannot be dynamically assigned") } return nil diff --git a/api/v1/lib/resources_bench_test.go b/api/v1/lib/resources_bench_test.go index 1fe84cc8..9bc74916 100644 --- a/api/v1/lib/resources_bench_test.go +++ b/api/v1/lib/resources_bench_test.go @@ -2,11 +2,13 @@ package mesos_test import ( "testing" + + . "github.com/mesos/mesos-go/api/v1/lib/resourcetest" ) func BenchmarkPrecisionScalarMath(b *testing.B) { var ( - start = resources(resource(name("cpus"), valueScalar(1.001))) + start = Resources(Resource(Name("cpus"), ValueScalar(1.001))) current = start.Clone() ) for i := 0; i < b.N; i++ { diff --git a/api/v1/lib/resources_test.go b/api/v1/lib/resources_test.go index d3b1c837..5e90ac37 100644 --- a/api/v1/lib/resources_test.go +++ b/api/v1/lib/resources_test.go @@ -1,22 +1,23 @@ package mesos_test import ( - "reflect" - "sort" "testing" "github.com/mesos/mesos-go/api/v1/lib" + rez "github.com/mesos/mesos-go/api/v1/lib/extras/resources" + "github.com/mesos/mesos-go/api/v1/lib/resourcefilters" + . "github.com/mesos/mesos-go/api/v1/lib/resourcetest" ) func TestResources_PrecisionRounding(t *testing.T) { var ( - cpu = resources(resource(name("cpus"), valueScalar(1.5015))) + cpu = Resources(Resource(Name("cpus"), ValueScalar(1.5015))) r1 = cpu.Plus(cpu...).Plus(cpu...).Minus(cpu...).Minus(cpu...) ) if !cpu.Equivalent(r1) { t.Fatalf("expected %v instead of %v", cpu, r1) } - actual, ok := r1.CPUs() + actual, ok := rez.CPUs(r1...) if !(ok && actual == 1.502) { t.Fatalf("expected 1.502 cpus instead of %v", actual) } @@ -24,13 +25,13 @@ func TestResources_PrecisionRounding(t *testing.T) { func TestResources_PrecisionLost(t *testing.T) { var ( - cpu = resources(resource(name("cpus"), valueScalar(1.5011))) + cpu = Resources(Resource(Name("cpus"), ValueScalar(1.5011))) r1 = cpu.Plus(cpu...).Plus(cpu...).Minus(cpu...).Minus(cpu...) ) if !cpu.Equivalent(r1) { t.Fatalf("expected %v instead of %v", cpu, r1) } - actual, ok := r1.CPUs() + actual, ok := rez.CPUs(r1...) if !(ok && actual == 1.501) { t.Fatalf("expected 1.501 cpus instead of %v", actual) } @@ -38,7 +39,7 @@ func TestResources_PrecisionLost(t *testing.T) { func TestResources_PrecisionManyConsecutiveOps(t *testing.T) { var ( - start = resources(resource(name("cpus"), valueScalar(1.001))) + start = Resources(Resource(Name("cpus"), ValueScalar(1.001))) increment = start.Clone() current = start.Clone() ) @@ -55,13 +56,13 @@ func TestResources_PrecisionManyConsecutiveOps(t *testing.T) { func TestResources_PrecisionManyOps(t *testing.T) { var ( - start = resources(resource(name("cpus"), valueScalar(1.001))) + start = Resources(Resource(Name("cpus"), ValueScalar(1.001))) current = start.Clone() next mesos.Resources ) for i := 0; i < 2500; i++ { next = current.Plus(current...).Plus(current...).Minus(current...).Minus(current...) - actual, ok := next.CPUs() + actual, ok := rez.CPUs(next...) if !(ok && actual == 1.001) { t.Fatalf("expected 1.001 cpus instead of %v", next) } @@ -76,10 +77,10 @@ func TestResources_PrecisionManyOps(t *testing.T) { func TestResources_PrecisionSimple(t *testing.T) { var ( - cpu = resources(resource(name("cpus"), valueScalar(1.001))) - zero = mesos.Resources{resource(name("cpus"), valueScalar(0))} // don't validate + cpu = Resources(Resource(Name("cpus"), ValueScalar(1.001))) + zero = mesos.Resources{Resource(Name("cpus"), ValueScalar(0))} // don't validate ) - actual, ok := cpu.CPUs() + actual, ok := rez.CPUs(cpu...) if !(ok && actual == 1.001) { t.Errorf("expected 1.001 instead of %f", actual) } @@ -91,51 +92,19 @@ func TestResources_PrecisionSimple(t *testing.T) { } } -func TestResources_Types(t *testing.T) { - rs := resources( - resource(name("cpus"), valueScalar(2), role("role1")), - resource(name("cpus"), valueScalar(4)), - resource(name("ports"), valueRange(span(1, 10)), role("role1")), - resource(name("ports"), valueRange(span(11, 20))), - ) - types := rs.Types() - expected := map[string]mesos.Value_Type{ - "cpus": mesos.SCALAR, - "ports": mesos.RANGES, - } - if !reflect.DeepEqual(types, expected) { - t.Fatalf("expected %v instead of %v", expected, types) - } -} - -func TestResources_Names(t *testing.T) { - rs := resources( - resource(name("cpus"), valueScalar(2), role("role1")), - resource(name("cpus"), valueScalar(4)), - resource(name("mem"), valueScalar(10), role("role1")), - resource(name("mem"), valueScalar(10)), - ) - names := rs.Names() - sort.Strings(names) - expected := []string{"cpus", "mem"} - if !reflect.DeepEqual(names, expected) { - t.Fatalf("expected %v instead of %v", expected, names) - } -} - func TestResource_RevocableResources(t *testing.T) { rs := mesos.Resources{ - resource(name("cpus"), valueScalar(1), role("*"), revocable()), - resource(name("cpus"), valueScalar(1), role("*")), + Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable()), + Resource(Name("cpus"), ValueScalar(1), Role("*")), } for i, tc := range []struct { r1, wants mesos.Resources }{ - {resources(rs[0]), resources(rs[0])}, - {resources(rs[1]), resources()}, - {resources(rs[0], rs[1]), resources(rs[0])}, + {Resources(rs[0]), Resources(rs[0])}, + {Resources(rs[1]), Resources()}, + {Resources(rs[0], rs[1]), Resources(rs[0])}, } { - x := mesos.RevocableResources.Select(tc.r1) + x := resourcefilters.Select(resourcefilters.Revocable, tc.r1...) if !tc.wants.Equivalent(x) { t.Errorf("test case %d failed: expected %v instead of %v", i, tc.wants, x) } @@ -144,39 +113,39 @@ func TestResource_RevocableResources(t *testing.T) { func TestResources_PersistentVolumes(t *testing.T) { var ( - rs = resources( - resource(name("cpus"), valueScalar(1)), - resource(name("mem"), valueScalar(512)), - resource(name("disk"), valueScalar(1000)), + rs = Resources( + Resource(Name("cpus"), ValueScalar(1)), + Resource(Name("mem"), ValueScalar(512)), + Resource(Name("disk"), ValueScalar(1000)), ) disk = mesos.Resources{ - resource(name("disk"), valueScalar(10), role("role1"), disk("1", "path")), - resource(name("disk"), valueScalar(20), role("role2"), disk("", "")), + Resource(Name("disk"), ValueScalar(10), Role("role1"), Disk("1", "path")), + Resource(Name("disk"), ValueScalar(20), Role("role2"), Disk("", "")), } ) rs.Add(disk...) - pv := mesos.PersistentVolumes.Select(rs) - if !resources(disk[0]).Equivalent(pv) { - t.Fatalf("expected %v instead of %v", resources(disk[0]), pv) + pv := resourcefilters.Select(resourcefilters.PersistentVolumes, rs...) + if !Resources(disk[0]).Equivalent(pv) { + t.Fatalf("expected %v instead of %v", Resources(disk[0]), pv) } } func TestResources_Validation(t *testing.T) { - // don't use resources(...) because that implicitly validates and skips invalid resources + // don't use Resources(...) because that implicitly validates and skips invalid resources rs := mesos.Resources{ - resource(name("cpus"), valueScalar(2), role("*"), disk("1", "path")), + Resource(Name("cpus"), ValueScalar(2), Role("*"), Disk("1", "path")), } err := rs.Validate() - if resourceErr, ok := err.(*mesos.ResourceError); !ok || resourceErr.Type() != mesos.ResourceErrorTypeIllegalDisk { + if !mesos.IsResourceError(err) { t.Fatalf("expected error because cpu resources can't contain disk info") } - err = mesos.Resources{resource(name("disk"), valueScalar(10), role("role"), disk("1", "path"))}.Validate() + err = mesos.Resources{Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("1", "path"))}.Validate() if err != nil { t.Fatalf("unexpected error: %+v", err) } - err = mesos.Resources{resource(name("disk"), valueScalar(10), role("role"), disk("", "path"))}.Validate() + err = mesos.Resources{Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "path"))}.Validate() if err != nil { t.Fatalf("unexpected error: %+v", err) } @@ -184,151 +153,71 @@ func TestResources_Validation(t *testing.T) { // reserved resources // unreserved: - err = mesos.Resources{resource(name("cpus"), valueScalar(8), role("*"))}.Validate() + err = mesos.Resources{Resource(Name("cpus"), ValueScalar(8), Role("*"))}.Validate() if err != nil { t.Fatalf("unexpected error validating unreserved resource: %+v", err) } // statically role reserved: - err = mesos.Resources{resource(name("cpus"), valueScalar(8), role("role"))}.Validate() + err = mesos.Resources{Resource(Name("cpus"), ValueScalar(8), Role("role"))}.Validate() if err != nil { t.Fatalf("unexpected error validating statically role reserved resource: %+v", err) } // dynamically role reserved: - err = mesos.Resources{resource(name("cpus"), valueScalar(8), role("role"), reservation(reservedBy("principal2")))}.Validate() + err = mesos.Resources{Resource(Name("cpus"), ValueScalar(8), Role("role"), Reservation(ReservedBy("principal2")))}.Validate() if err != nil { t.Fatalf("unexpected error validating dynamically role reserved resource: %+v", err) } // invalid - err = mesos.Resources{resource(name("cpus"), valueScalar(8), role("*"), reservation(reservedBy("principal1")))}.Validate() + err = mesos.Resources{Resource(Name("cpus"), ValueScalar(8), Role("*"), Reservation(ReservedBy("principal1")))}.Validate() if err == nil { t.Fatalf("expected error for invalid reserved resource") } } -func TestResources_Find(t *testing.T) { - for i, tc := range []struct { - r1, targets, wants mesos.Resources - }{ - {nil, nil, nil}, - { - r1: resources( - resource(name("cpus"), valueScalar(2), role("role1")), - resource(name("mem"), valueScalar(10), role("role1")), - resource(name("cpus"), valueScalar(4), role("*")), - resource(name("mem"), valueScalar(20), role("*")), - ), - targets: resources( - resource(name("cpus"), valueScalar(3), role("role1")), - resource(name("mem"), valueScalar(15), role("role1")), - ), - wants: resources( - resource(name("cpus"), valueScalar(2), role("role1")), - resource(name("mem"), valueScalar(10), role("role1")), - resource(name("cpus"), valueScalar(1), role("*")), - resource(name("mem"), valueScalar(5), role("*")), - ), - }, - { - r1: resources( - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("mem"), valueScalar(5), role("role1")), - resource(name("cpus"), valueScalar(2), role("role2")), - resource(name("mem"), valueScalar(8), role("role2")), - resource(name("cpus"), valueScalar(1), role("*")), - resource(name("mem"), valueScalar(7), role("*")), - ), - targets: resources( - resource(name("cpus"), valueScalar(3), role("role1")), - resource(name("mem"), valueScalar(15), role("role1")), - ), - wants: resources( - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("mem"), valueScalar(5), role("role1")), - resource(name("cpus"), valueScalar(1), role("*")), - resource(name("mem"), valueScalar(7), role("*")), - resource(name("cpus"), valueScalar(1), role("role2")), - resource(name("mem"), valueScalar(3), role("role2")), - ), - }, - { - r1: resources( - resource(name("cpus"), valueScalar(5), role("role1")), - resource(name("mem"), valueScalar(5), role("role1")), - resource(name("cpus"), valueScalar(5), role("*")), - resource(name("mem"), valueScalar(5), role("*")), - ), - targets: resources( - resource(name("cpus"), valueScalar(6)), - resource(name("mem"), valueScalar(6)), - ), - wants: resources( - resource(name("cpus"), valueScalar(5), role("*")), - resource(name("mem"), valueScalar(5), role("*")), - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("mem"), valueScalar(1), role("role1")), - ), - }, - { - r1: resources( - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("mem"), valueScalar(1), role("role1")), - ), - targets: resources( - resource(name("cpus"), valueScalar(2), role("role1")), - resource(name("mem"), valueScalar(2), role("role1")), - ), - wants: nil, - }, - } { - r := tc.r1.Find(tc.targets) - expect(t, r.Equivalent(tc.wants), "test case %d failed: expected %+v instead of %+v", i, tc.wants, r) - } -} - func TestResources_Flatten(t *testing.T) { for i, tc := range []struct { r1, wants mesos.Resources }{ {nil, nil}, { - r1: resources( - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("cpus"), valueScalar(2), role("role2")), - resource(name("mem"), valueScalar(5), role("role1")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("cpus"), ValueScalar(2), Role("role2")), + Resource(Name("mem"), ValueScalar(5), Role("role1")), ), - wants: resources( - resource(name("cpus"), valueScalar(3)), - resource(name("mem"), valueScalar(5)), + wants: Resources( + Resource(Name("cpus"), ValueScalar(3)), + Resource(Name("mem"), ValueScalar(5)), ), }, { - r1: resources( - resource(name("cpus"), valueScalar(3), role("role1")), - resource(name("mem"), valueScalar(15), role("role1")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(3), Role("role1")), + Resource(Name("mem"), ValueScalar(15), Role("role1")), ), - wants: resources( - resource(name("cpus"), valueScalar(3), role("*")), - resource(name("mem"), valueScalar(15), role("*")), + wants: Resources( + Resource(Name("cpus"), ValueScalar(3), Role("*")), + Resource(Name("mem"), ValueScalar(15), Role("*")), ), }, } { r := tc.r1.Flatten() - expect(t, r.Equivalent(tc.wants), "test case %d failed: expected %+v instead of %+v", i, tc.wants, r) + Expect(t, r.Equivalent(tc.wants), "test case %d failed: expected %+v instead of %+v", i, tc.wants, r) } } func TestResources_Equivalent(t *testing.T) { disks := mesos.Resources{ - resource(name("disk"), valueScalar(10), role("*"), disk("", "")), - resource(name("disk"), valueScalar(10), role("*"), disk("", "path1")), - resource(name("disk"), valueScalar(10), role("*"), disk("", "path2")), - resource(name("disk"), valueScalar(10), role("role"), disk("", "path2")), - resource(name("disk"), valueScalar(10), role("role"), disk("1", "path1")), - resource(name("disk"), valueScalar(10), role("role"), disk("1", "path2")), - resource(name("disk"), valueScalar(10), role("role"), disk("2", "path2")), + Resource(Name("disk"), ValueScalar(10), Role("*"), Disk("", "")), + Resource(Name("disk"), ValueScalar(10), Role("*"), Disk("", "path1")), + Resource(Name("disk"), ValueScalar(10), Role("*"), Disk("", "path2")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "path2")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("1", "path1")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("1", "path2")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("2", "path2")), } for i, tc := range []struct { r1, r2 mesos.Resources @@ -336,76 +225,76 @@ func TestResources_Equivalent(t *testing.T) { }{ {r1: nil, r2: nil, wants: true}, { // 1 - r1: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), wants: true, }, { // 2 - r1: resources( - resource(name("cpus"), valueScalar(50), role("role1")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("role1")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("role2")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("role2")), ), wants: false, }, { // 3 - r1: resources(resource(name("ports"), valueRange(span(20, 40)), role("*"))), - r2: resources(resource(name("ports"), valueRange(span(20, 30), span(31, 39), span(40, 40)), role("*"))), + r1: Resources(Resource(Name("ports"), ValueRange(Span(20, 40)), Role("*"))), + r2: Resources(Resource(Name("ports"), ValueRange(Span(20, 30), Span(31, 39), Span(40, 40)), Role("*"))), wants: true, }, { // 4 - r1: resources(resource(name("disks"), valueSet("sda1"), role("*"))), - r2: resources(resource(name("disks"), valueSet("sda1"), role("*"))), + r1: Resources(Resource(Name("disks"), ValueSet("sda1"), Role("*"))), + r2: Resources(Resource(Name("disks"), ValueSet("sda1"), Role("*"))), wants: true, }, { // 5 - r1: resources(resource(name("disks"), valueSet("sda1"), role("*"))), - r2: resources(resource(name("disks"), valueSet("sda2"), role("*"))), + r1: Resources(Resource(Name("disks"), ValueSet("sda1"), Role("*"))), + r2: Resources(Resource(Name("disks"), ValueSet("sda2"), Role("*"))), wants: false, }, - {resources(disks[0]), resources(disks[1]), true}, // 6 - {resources(disks[1]), resources(disks[2]), true}, // 7 - {resources(disks[4]), resources(disks[5]), true}, // 8 - {resources(disks[5]), resources(disks[6]), false}, // 9 - {resources(disks[3]), resources(disks[6]), false}, // 10 + {Resources(disks[0]), Resources(disks[1]), true}, // 6 + {Resources(disks[1]), Resources(disks[2]), true}, // 7 + {Resources(disks[4]), Resources(disks[5]), true}, // 8 + {Resources(disks[5]), Resources(disks[6]), false}, // 9 + {Resources(disks[3]), Resources(disks[6]), false}, // 10 { // 11 - r1: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - r2: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), + r1: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + r2: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), wants: true, }, { // 12 - r1: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - r2: resources(resource(name("cpus"), valueScalar(1), role("*"))), + r1: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + r2: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"))), wants: false, }, } { actual := tc.r1.Equivalent(tc.r2) - expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) + Expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) } possiblyReserved := mesos.Resources{ // unreserved - resource(name("cpus"), valueScalar(8), role("*")), + Resource(Name("cpus"), ValueScalar(8), Role("*")), // statically role reserved - resource(name("cpus"), valueScalar(8), role("role1")), - resource(name("cpus"), valueScalar(8), role("role2")), + Resource(Name("cpus"), ValueScalar(8), Role("role1")), + Resource(Name("cpus"), ValueScalar(8), Role("role2")), // dynamically role reserved: - resource(name("cpus"), valueScalar(8), role("role1"), reservation(reservedBy("principal1"))), - resource(name("cpus"), valueScalar(8), role("role2"), reservation(reservedBy("principal2"))), + Resource(Name("cpus"), ValueScalar(8), Role("role1"), Reservation(ReservedBy("principal1"))), + Resource(Name("cpus"), ValueScalar(8), Role("role2"), Reservation(ReservedBy("principal2"))), } for i := 0; i < len(possiblyReserved); i++ { for j := 0; j < len(possiblyReserved); j++ { if i == j { continue } - if resources(possiblyReserved[i]).Equivalent(resources(possiblyReserved[j])) { + if Resources(possiblyReserved[i]).Equivalent(Resources(possiblyReserved[j])) { t.Errorf("unexpected equivalence between %v and %v", possiblyReserved[i], possiblyReserved[j]) } } @@ -414,39 +303,39 @@ func TestResources_Equivalent(t *testing.T) { func TestResources_ContainsAll(t *testing.T) { var ( - ports1 = resources(resource(name("ports"), valueRange(span(2, 2), span(4, 5)), role("*"))) - ports2 = resources(resource(name("ports"), valueRange(span(1, 10)), role("*"))) - ports3 = resources(resource(name("ports"), valueRange(span(2, 3)), role("*"))) - ports4 = resources(resource(name("ports"), valueRange(span(1, 2), span(4, 6)), role("*"))) - ports5 = resources(resource(name("ports"), valueRange(span(1, 4), span(5, 5)), role("*"))) + ports1 = Resources(Resource(Name("ports"), ValueRange(Span(2, 2), Span(4, 5)), Role("*"))) + ports2 = Resources(Resource(Name("ports"), ValueRange(Span(1, 10)), Role("*"))) + ports3 = Resources(Resource(Name("ports"), ValueRange(Span(2, 3)), Role("*"))) + ports4 = Resources(Resource(Name("ports"), ValueRange(Span(1, 2), Span(4, 6)), Role("*"))) + ports5 = Resources(Resource(Name("ports"), ValueRange(Span(1, 4), Span(5, 5)), Role("*"))) - disks1 = resources(resource(name("disks"), valueSet("sda1", "sda2"), role("*"))) - disks2 = resources(resource(name("disks"), valueSet("sda1", "sda3", "sda4", "sda2"), role("*"))) + disks1 = Resources(Resource(Name("disks"), ValueSet("sda1", "sda2"), Role("*"))) + disks2 = Resources(Resource(Name("disks"), ValueSet("sda1", "sda3", "sda4", "sda2"), Role("*"))) disks = mesos.Resources{ - resource(name("disk"), valueScalar(10), role("role"), disk("1", "path")), - resource(name("disk"), valueScalar(10), role("role"), disk("2", "path")), - resource(name("disk"), valueScalar(20), role("role"), disk("1", "path")), - resource(name("disk"), valueScalar(20), role("role"), disk("", "path")), - resource(name("disk"), valueScalar(20), role("role"), disk("2", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("1", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("2", "path")), + Resource(Name("disk"), ValueScalar(20), Role("role"), Disk("1", "path")), + Resource(Name("disk"), ValueScalar(20), Role("role"), Disk("", "path")), + Resource(Name("disk"), ValueScalar(20), Role("role"), Disk("2", "path")), } - summedDisks = resources(disks[0]).Plus(disks[1]) - summedDisks2 = resources(disks[0]).Plus(disks[4]) + summedDisks = Resources(disks[0]).Plus(disks[1]) + summedDisks2 = Resources(disks[0]).Plus(disks[4]) revocables = mesos.Resources{ - resource(name("cpus"), valueScalar(1), role("*"), revocable()), - resource(name("cpus"), valueScalar(1), role("*")), - resource(name("cpus"), valueScalar(2), role("*")), - resource(name("cpus"), valueScalar(2), role("*"), revocable()), + Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable()), + Resource(Name("cpus"), ValueScalar(1), Role("*")), + Resource(Name("cpus"), ValueScalar(2), Role("*")), + Resource(Name("cpus"), ValueScalar(2), Role("*"), Revocable()), } - summedRevocables = resources(revocables[0]).Plus(revocables[1]) - summedRevocables2 = resources(revocables[0]).Plus(revocables[0]) + summedRevocables = Resources(revocables[0]).Plus(revocables[1]) + summedRevocables2 = Resources(revocables[0]).Plus(revocables[0]) possiblyReserved = mesos.Resources{ - resource(name("cpus"), valueScalar(8), role("role")), - resource(name("cpus"), valueScalar(12), role("role"), reservation(reservedBy("principal"))), + Resource(Name("cpus"), ValueScalar(8), Role("role")), + Resource(Name("cpus"), ValueScalar(12), Role("role"), Reservation(ReservedBy("principal"))), } - sumPossiblyReserved = resources(possiblyReserved...) + sumPossiblyReserved = Resources(possiblyReserved...) ) for i, tc := range []struct { r1, r2 mesos.Resources @@ -456,47 +345,47 @@ func TestResources_ContainsAll(t *testing.T) { {r1: nil, r2: nil, wants: true}, // test case 1 { - r1: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), wants: true, }, // test case 2 { - r1: resources( - resource(name("cpus"), valueScalar(50), role("role1")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("role1")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("role2")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("role2")), ), wants: false, }, // test case 3 { - r1: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(3072), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(3072), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), wants: false, }, // test case 4 { - r1: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(3072), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(3072), Role("*")), ), wants: true, }, @@ -524,27 +413,27 @@ func TestResources_ContainsAll(t *testing.T) { {disks1, disks2, false}, // test case 16 {disks2, disks1, true}, - {r1: summedDisks, r2: resources(disks[0]), wants: true}, - {r1: summedDisks, r2: resources(disks[1]), wants: true}, - {r1: summedDisks, r2: resources(disks[2]), wants: false}, - {r1: summedDisks, r2: resources(disks[3]), wants: false}, - {r1: resources(disks[0]), r2: summedDisks, wants: false}, - {r1: resources(disks[1]), r2: summedDisks, wants: false}, - {r1: summedDisks2, r2: resources(disks[0]), wants: true}, - {r1: summedDisks2, r2: resources(disks[4]), wants: true}, - {r1: summedRevocables, r2: resources(revocables[0]), wants: true}, - {r1: summedRevocables, r2: resources(revocables[1]), wants: true}, - {r1: summedRevocables, r2: resources(revocables[2]), wants: false}, - {r1: summedRevocables, r2: resources(revocables[3]), wants: false}, - {r1: resources(revocables[0]), r2: summedRevocables2, wants: false}, - {r1: summedRevocables2, r2: resources(revocables[0]), wants: true}, + {r1: summedDisks, r2: Resources(disks[0]), wants: true}, + {r1: summedDisks, r2: Resources(disks[1]), wants: true}, + {r1: summedDisks, r2: Resources(disks[2]), wants: false}, + {r1: summedDisks, r2: Resources(disks[3]), wants: false}, + {r1: Resources(disks[0]), r2: summedDisks, wants: false}, + {r1: Resources(disks[1]), r2: summedDisks, wants: false}, + {r1: summedDisks2, r2: Resources(disks[0]), wants: true}, + {r1: summedDisks2, r2: Resources(disks[4]), wants: true}, + {r1: summedRevocables, r2: Resources(revocables[0]), wants: true}, + {r1: summedRevocables, r2: Resources(revocables[1]), wants: true}, + {r1: summedRevocables, r2: Resources(revocables[2]), wants: false}, + {r1: summedRevocables, r2: Resources(revocables[3]), wants: false}, + {r1: Resources(revocables[0]), r2: summedRevocables2, wants: false}, + {r1: summedRevocables2, r2: Resources(revocables[0]), wants: true}, {r1: summedRevocables2, r2: summedRevocables2, wants: true}, - {r1: resources(possiblyReserved[0]), r2: sumPossiblyReserved, wants: false}, - {r1: resources(possiblyReserved[1]), r2: sumPossiblyReserved, wants: false}, + {r1: Resources(possiblyReserved[0]), r2: sumPossiblyReserved, wants: false}, + {r1: Resources(possiblyReserved[1]), r2: sumPossiblyReserved, wants: false}, {r1: sumPossiblyReserved, r2: sumPossiblyReserved, wants: true}, } { actual := tc.r1.ContainsAll(tc.r2) - expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) + Expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) } } @@ -553,27 +442,27 @@ func TestResource_IsEmpty(t *testing.T) { r mesos.Resource wants bool }{ - {resource(), true}, - {resource(valueScalar(0)), true}, - {resource(valueSet()), true}, - {resource(valueSet([]string{}...)), true}, - {resource(valueSet()), true}, - {resource(valueSet("")), false}, - {resource(valueRange()), true}, - {resource(valueRange(span(0, 0))), false}, + {Resource(), true}, + {Resource(ValueScalar(0)), true}, + {Resource(ValueSet()), true}, + {Resource(ValueSet([]string{}...)), true}, + {Resource(ValueSet()), true}, + {Resource(ValueSet("")), false}, + {Resource(ValueRange()), true}, + {Resource(ValueRange(Span(0, 0))), false}, } { actual := tc.r.IsEmpty() - expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) + Expect(t, tc.wants == actual, "test case %d failed: wants (%v) != actual (%v)", i, tc.wants, actual) } } func TestResources_Minus(t *testing.T) { disks := mesos.Resources{ - resource(name("disk"), valueScalar(10), role("role"), disk("", "path")), - resource(name("disk"), valueScalar(10), role("role"), disk("", "")), - resource(name("disk"), valueScalar(10), role("role"), disk("1", "path")), - resource(name("disk"), valueScalar(10), role("role"), disk("2", "path")), - resource(name("disk"), valueScalar(10), role("role"), disk("2", "path2")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("1", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("2", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("2", "path2")), } for i, tc := range []struct { r1, r2 mesos.Resources @@ -582,150 +471,150 @@ func TestResources_Minus(t *testing.T) { wantsMemory uint64 }{ {r1: nil, r2: nil, wants: nil}, - {r1: resources(), r2: resources(), wants: resources()}, + {r1: Resources(), r2: Resources(), wants: Resources()}, // simple scalars, same roles for everything { - r1: resources( - resource(name("cpus"), valueScalar(50), role("*")), - resource(name("mem"), valueScalar(4096), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(50), Role("*")), + Resource(Name("mem"), ValueScalar(4096), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(0.5), role("*")), - resource(name("mem"), valueScalar(1024), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(0.5), Role("*")), + Resource(Name("mem"), ValueScalar(1024), Role("*")), ), - wants: resources( - resource(name("cpus"), valueScalar(49.5), role("*")), - resource(name("mem"), valueScalar(3072), role("*")), + wants: Resources( + Resource(Name("cpus"), ValueScalar(49.5), Role("*")), + Resource(Name("mem"), ValueScalar(3072), Role("*")), ), wantsCPU: 49.5, wantsMemory: 3072, }, // multi-role, scalar subtraction { - r1: resources( - resource(name("cpus"), valueScalar(5), role("role1")), - resource(name("cpus"), valueScalar(3), role("role2")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(5), Role("role1")), + Resource(Name("cpus"), ValueScalar(3), Role("role2")), ), - r2: resources( - resource(name("cpus"), valueScalar(1), role("role1")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), ), - wants: resources( - resource(name("cpus"), valueScalar(4), role("role1")), - resource(name("cpus"), valueScalar(3), role("role2")), + wants: Resources( + Resource(Name("cpus"), ValueScalar(4), Role("role1")), + Resource(Name("cpus"), ValueScalar(3), Role("role2")), ), wantsCPU: 7, }, // simple ranges, same roles, lower-edge overlap { - r1: resources( - resource(name("ports"), valueRange(span(20000, 40000)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(20000, 40000)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(10000, 20000), span(30000, 50000)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(10000, 20000), Span(30000, 50000)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(20001, 29999)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(20001, 29999)), Role("*")), ), }, // simple ranges, same roles, single port/lower-edge { - r1: resources( - resource(name("ports"), valueRange(span(50000, 60000)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 60000)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(50000, 50000)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 50000)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(50001, 60000)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(50001, 60000)), Role("*")), ), }, // simple ranges, same roles, multi port/lower-edge { - r1: resources( - resource(name("ports"), valueRange(span(50000, 60000)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 60000)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(50000, 50001)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 50001)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(50002, 60000)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(50002, 60000)), Role("*")), ), }, // simple ranges, same roles, identical overlap { - r1: resources( - resource(name("ports"), valueRange(span(50000, 60000)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 60000)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(50000, 60000)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(50000, 60000)), Role("*")), ), - wants: resources(), + wants: Resources(), }, // multiple ranges, same roles, swiss cheese { - r1: resources( - resource(name("ports"), valueRange(span(1, 10), span(20, 30), span(40, 50)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(1, 10), Span(20, 30), Span(40, 50)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(2, 9), span(15, 45), span(48, 50)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(2, 9), Span(15, 45), Span(48, 50)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(1, 1), span(10, 10), span(46, 47)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(1, 1), Span(10, 10), Span(46, 47)), Role("*")), ), }, // multiple ranges, same roles, no overlap { - r1: resources( - resource(name("ports"), valueRange(span(1, 10)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(1, 10)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(11, 20)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(11, 20)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(1, 10)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(1, 10)), Role("*")), ), }, // simple set, same roles { - r1: resources( - resource(name("disks"), valueSet("sda1", "sda2", "sda3", "sda4"), role("*")), + r1: Resources( + Resource(Name("disks"), ValueSet("sda1", "sda2", "sda3", "sda4"), Role("*")), ), - r2: resources( - resource(name("disks"), valueSet("sda2", "sda3", "sda4"), role("*")), + r2: Resources( + Resource(Name("disks"), ValueSet("sda2", "sda3", "sda4"), Role("*")), ), - wants: resources( - resource(name("disks"), valueSet("sda1"), role("*")), + wants: Resources( + Resource(Name("disks"), ValueSet("sda1"), Role("*")), ), }, - {r1: resources(disks[0]), r2: resources(disks[1]), wants: resources()}, - {r1: resources(disks[2]), r2: resources(disks[3]), wants: resources(disks[2])}, - {r1: resources(disks[2]), r2: resources(disks[2]), wants: resources()}, - {r1: resources(disks[3]), r2: resources(disks[4]), wants: resources()}, + {r1: Resources(disks[0]), r2: Resources(disks[1]), wants: Resources()}, + {r1: Resources(disks[2]), r2: Resources(disks[3]), wants: Resources(disks[2])}, + {r1: Resources(disks[2]), r2: Resources(disks[2]), wants: Resources()}, + {r1: Resources(disks[3]), r2: Resources(disks[4]), wants: Resources()}, // revocables { - r1: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - r2: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - wants: resources(), + r1: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + r2: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + wants: Resources(), }, { // revocable - non-revocable is a noop - r1: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - r2: resources(resource(name("cpus"), valueScalar(1), role("*"))), - wants: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), + r1: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + r2: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"))), + wants: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), wantsCPU: 1, }, // reserved { - r1: resources( - resource(name("cpus"), valueScalar(8), role("role")), - resource(name("cpus"), valueScalar(8), role("role"), reservation(reservedBy("principal"))), + r1: Resources( + Resource(Name("cpus"), ValueScalar(8), Role("role")), + Resource(Name("cpus"), ValueScalar(8), Role("role"), Reservation(ReservedBy("principal"))), ), - r2: resources( - resource(name("cpus"), valueScalar(2), role("role")), - resource(name("cpus"), valueScalar(4), role("role"), reservation(reservedBy("principal"))), + r2: Resources( + Resource(Name("cpus"), ValueScalar(2), Role("role")), + Resource(Name("cpus"), ValueScalar(4), Role("role"), Reservation(ReservedBy("principal"))), ), - wants: resources( - resource(name("cpus"), valueScalar(6), role("role")), - resource(name("cpus"), valueScalar(4), role("role"), reservation(reservedBy("principal"))), + wants: Resources( + Resource(Name("cpus"), ValueScalar(6), Role("role")), + Resource(Name("cpus"), ValueScalar(4), Role("role"), Reservation(ReservedBy("principal"))), ), wantsCPU: 10, }, @@ -747,14 +636,14 @@ func TestResources_Minus(t *testing.T) { t.Errorf("test case %d failed: wants (%v) != r1 (%v)", i, tc.wants, tc.r1) } - cpus, ok := tc.r1.CPUs() + cpus, ok := rez.CPUs(tc.r1...) if !ok && tc.wantsCPU > 0 { t.Errorf("test case %d failed: failed to obtain total CPU resources", i) } else if cpus != tc.wantsCPU { t.Errorf("test case %d failed: wants cpu (%v) != r1 cpu (%v)", i, tc.wantsCPU, cpus) } - mem, ok := tc.r1.Memory() + mem, ok := rez.Memory(tc.r1...) if !ok && tc.wantsMemory > 0 { t.Errorf("test case %d failed: failed to obtain total memory resources", i) } else if mem != tc.wantsMemory { @@ -771,9 +660,9 @@ func TestResources_Minus(t *testing.T) { func TestResources_Plus(t *testing.T) { disks := mesos.Resources{ - resource(name("disk"), valueScalar(10), role("role"), disk("", "path")), - resource(name("disk"), valueScalar(10), role("role"), disk("", "")), - resource(name("disk"), valueScalar(20), role("role"), disk("", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "path")), + Resource(Name("disk"), ValueScalar(10), Role("role"), Disk("", "")), + Resource(Name("disk"), ValueScalar(20), Role("role"), Disk("", "path")), } for i, tc := range []struct { r1, r2 mesos.Resources @@ -781,120 +670,120 @@ func TestResources_Plus(t *testing.T) { wantsCPU float64 wantsMemory uint64 }{ - {r1: resources(disks[0]), r2: resources(disks[1]), wants: resources(disks[2])}, + {r1: Resources(disks[0]), r2: Resources(disks[1]), wants: Resources(disks[2])}, {r1: nil, r2: nil, wants: nil}, - {r1: resources(), r2: resources(), wants: resources()}, + {r1: Resources(), r2: Resources(), wants: Resources()}, // simple scalars, same roles for everything { - r1: resources( - resource(name("cpus"), valueScalar(1), role("*")), - resource(name("mem"), valueScalar(5), role("*")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("*")), + Resource(Name("mem"), ValueScalar(5), Role("*")), ), - r2: resources( - resource(name("cpus"), valueScalar(2), role("*")), - resource(name("mem"), valueScalar(10), role("*")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(2), Role("*")), + Resource(Name("mem"), ValueScalar(10), Role("*")), ), - wants: resources( - resource(name("cpus"), valueScalar(3), role("*")), - resource(name("mem"), valueScalar(15), role("*")), + wants: Resources( + Resource(Name("cpus"), ValueScalar(3), Role("*")), + Resource(Name("mem"), ValueScalar(15), Role("*")), ), wantsCPU: 3, wantsMemory: 15, }, // simple scalars, differing roles { - r1: resources( - resource(name("cpus"), valueScalar(1), role("role1")), - resource(name("cpus"), valueScalar(3), role("role2")), + r1: Resources( + Resource(Name("cpus"), ValueScalar(1), Role("role1")), + Resource(Name("cpus"), ValueScalar(3), Role("role2")), ), - r2: resources( - resource(name("cpus"), valueScalar(5), role("role1")), + r2: Resources( + Resource(Name("cpus"), ValueScalar(5), Role("role1")), ), - wants: resources( - resource(name("cpus"), valueScalar(6), role("role1")), - resource(name("cpus"), valueScalar(3), role("role2")), + wants: Resources( + Resource(Name("cpus"), ValueScalar(6), Role("role1")), + Resource(Name("cpus"), ValueScalar(3), Role("role2")), ), wantsCPU: 9, }, // ranges addition yields continuous range { - r1: resources( - resource(name("ports"), valueRange(span(20000, 40000)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(20000, 40000)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(30000, 50000), span(10000, 20000)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(30000, 50000), Span(10000, 20000)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(10000, 50000)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(10000, 50000)), Role("*")), ), }, // ranges addition yields a split set of ranges { - r1: resources( - resource(name("ports"), valueRange(span(1, 10), span(5, 30), span(50, 60)), role("*")), - resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(1, 10), Span(5, 30), Span(50, 60)), Role("*")), + Resource(Name("ports"), ValueRange(Span(1, 65), Span(70, 80)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(1, 65), span(70, 80)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(1, 65), Span(70, 80)), Role("*")), ), }, // ranges addition (composite) yields a continuous range { - r1: resources( - resource(name("ports"), valueRange(span(1, 2)), role("*")), - resource(name("ports"), valueRange(span(3, 4)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(1, 2)), Role("*")), + Resource(Name("ports"), ValueRange(Span(3, 4)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(7, 8)), role("*")), - resource(name("ports"), valueRange(span(5, 6)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(7, 8)), Role("*")), + Resource(Name("ports"), ValueRange(Span(5, 6)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(1, 8)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(1, 8)), Role("*")), ), }, // ranges addition yields a split set of ranges { - r1: resources( - resource(name("ports"), valueRange(span(1, 4), span(9, 10), span(20, 22), span(26, 30)), role("*")), + r1: Resources( + Resource(Name("ports"), ValueRange(Span(1, 4), Span(9, 10), Span(20, 22), Span(26, 30)), Role("*")), ), - r2: resources( - resource(name("ports"), valueRange(span(5, 8), span(23, 25)), role("*")), + r2: Resources( + Resource(Name("ports"), ValueRange(Span(5, 8), Span(23, 25)), Role("*")), ), - wants: resources( - resource(name("ports"), valueRange(span(1, 10), span(20, 30)), role("*")), + wants: Resources( + Resource(Name("ports"), ValueRange(Span(1, 10), Span(20, 30)), Role("*")), ), }, // set addition { - r1: resources( - resource(name("disks"), valueSet("sda1", "sda2", "sda3"), role("*")), + r1: Resources( + Resource(Name("disks"), ValueSet("sda1", "sda2", "sda3"), Role("*")), ), - r2: resources( - resource(name("disks"), valueSet("sda1", "sda2", "sda3", "sda4"), role("*")), + r2: Resources( + Resource(Name("disks"), ValueSet("sda1", "sda2", "sda3", "sda4"), Role("*")), ), - wants: resources( - resource(name("disks"), valueSet("sda4", "sda2", "sda1", "sda3"), role("*")), + wants: Resources( + Resource(Name("disks"), ValueSet("sda4", "sda2", "sda1", "sda3"), Role("*")), ), }, // revocables { - r1: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - r2: resources(resource(name("cpus"), valueScalar(1), role("*"), revocable())), - wants: resources(resource(name("cpus"), valueScalar(2), role("*"), revocable())), + r1: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + r2: Resources(Resource(Name("cpus"), ValueScalar(1), Role("*"), Revocable())), + wants: Resources(Resource(Name("cpus"), ValueScalar(2), Role("*"), Revocable())), wantsCPU: 2, }, // statically reserved { - r1: resources(resource(name("cpus"), valueScalar(8), role("role"))), - r2: resources(resource(name("cpus"), valueScalar(4), role("role"))), - wants: resources(resource(name("cpus"), valueScalar(12), role("role"))), + r1: Resources(Resource(Name("cpus"), ValueScalar(8), Role("role"))), + r2: Resources(Resource(Name("cpus"), ValueScalar(4), Role("role"))), + wants: Resources(Resource(Name("cpus"), ValueScalar(12), Role("role"))), wantsCPU: 12, }, // dynamically reserved { - r1: resources(resource(name("cpus"), valueScalar(8), role("role"), reservation(reservedBy("principal")))), - r2: resources(resource(name("cpus"), valueScalar(4), role("role"), reservation(reservedBy("principal")))), - wants: resources(resource(name("cpus"), valueScalar(12), role("role"), reservation(reservedBy("principal")))), + r1: Resources(Resource(Name("cpus"), ValueScalar(8), Role("role"), Reservation(ReservedBy("principal")))), + r2: Resources(Resource(Name("cpus"), ValueScalar(4), Role("role"), Reservation(ReservedBy("principal")))), + wants: Resources(Resource(Name("cpus"), ValueScalar(12), Role("role"), Reservation(ReservedBy("principal")))), wantsCPU: 12, }, } { @@ -915,14 +804,14 @@ func TestResources_Plus(t *testing.T) { t.Errorf("test case %d failed: wants (%v) != r1 (%v)", i, tc.wants, tc.r1) } - cpus, ok := tc.r1.CPUs() + cpus, ok := rez.CPUs(tc.r1...) if !ok && tc.wantsCPU > 0 { t.Errorf("test case %d failed: failed to obtain total CPU resources", i) } else if cpus != tc.wantsCPU { t.Errorf("test case %d failed: wants cpu (%v) != r1 cpu (%v)", i, tc.wantsCPU, cpus) } - mem, ok := tc.r1.Memory() + mem, ok := rez.Memory(tc.r1...) if !ok && tc.wantsMemory > 0 { t.Errorf("test case %d failed: failed to obtain total memory resources", i) } else if mem != tc.wantsMemory { @@ -930,68 +819,3 @@ func TestResources_Plus(t *testing.T) { } } } - -// functional resource modifier -type resourceOpt func(*mesos.Resource) - -func resource(opt ...resourceOpt) (r mesos.Resource) { - if len(opt) == 0 { - return - } - for _, f := range opt { - f(&r) - } - return -} - -func name(x string) resourceOpt { return func(r *mesos.Resource) { r.Name = x } } -func role(x string) resourceOpt { return func(r *mesos.Resource) { r.Role = &x } } - -func revocable() resourceOpt { - return func(r *mesos.Resource) { r.Revocable = &mesos.Resource_RevocableInfo{} } -} - -func valueScalar(x float64) resourceOpt { - return func(r *mesos.Resource) { - r.Type = mesos.SCALAR.Enum() - r.Scalar = &mesos.Value_Scalar{Value: x} - } -} - -func valueSet(x ...string) resourceOpt { - return func(r *mesos.Resource) { - r.Type = mesos.SET.Enum() - r.Set = &mesos.Value_Set{Item: x} - } -} - -type rangeOpt func(*mesos.Ranges) - -// "range" is a keyword, so I called this func "span": it naively appends a range to a Ranges collection -func span(bp, ep uint64) rangeOpt { - return func(rs *mesos.Ranges) { - *rs = append(*rs, mesos.Value_Range{Begin: bp, End: ep}) - } -} - -func valueRange(p ...rangeOpt) resourceOpt { - return func(r *mesos.Resource) { - rs := mesos.Ranges(nil) - for _, f := range p { - f(&rs) - } - r.Type = mesos.RANGES.Enum() - r.Ranges = r.Ranges.Add(&mesos.Value_Ranges{Range: rs}) - } -} - -func resources(r ...mesos.Resource) (result mesos.Resources) { - return result.Add(r...) -} - -func expect(t *testing.T, cond bool, msgformat string, args ...interface{}) bool { - if !cond { - t.Errorf(msgformat, args...) - } - return cond -} diff --git a/api/v1/lib/resourcetest/resourcetest.go b/api/v1/lib/resourcetest/resourcetest.go new file mode 100644 index 00000000..b03ae1f8 --- /dev/null +++ b/api/v1/lib/resourcetest/resourcetest.go @@ -0,0 +1,127 @@ +package resourcetest + +// maybe this could eventually be a resourcedsl package, but resource construction rules aren't that strict here yet + +import ( + "testing" + + "github.com/mesos/mesos-go/api/v1/lib" +) + +// functional resource modifier +type Opt func(*mesos.Resource) + +func Resource(opt ...Opt) (r mesos.Resource) { + if len(opt) == 0 { + return + } + for _, f := range opt { + f(&r) + } + return +} + +func Name(x string) Opt { return func(r *mesos.Resource) { r.Name = x } } +func Role(x string) Opt { return func(r *mesos.Resource) { r.Role = &x } } + +func Revocable() Opt { + return func(r *mesos.Resource) { r.Revocable = &mesos.Resource_RevocableInfo{} } +} + +func ValueScalar(x float64) Opt { + return func(r *mesos.Resource) { + r.Type = mesos.SCALAR.Enum() + r.Scalar = &mesos.Value_Scalar{Value: x} + } +} + +func ValueSet(x ...string) Opt { + return func(r *mesos.Resource) { + r.Type = mesos.SET.Enum() + r.Set = &mesos.Value_Set{Item: x} + } +} + +type RangeOpt func(*mesos.Ranges) + +// "range" is a keyword, so I called this func "span": it naively appends a range to a Ranges collection +func Span(bp, ep uint64) RangeOpt { + return func(rs *mesos.Ranges) { + *rs = append(*rs, mesos.Value_Range{Begin: bp, End: ep}) + } +} + +func ValueRange(p ...RangeOpt) Opt { + return func(r *mesos.Resource) { + rs := mesos.Ranges(nil) + for _, f := range p { + f(&rs) + } + r.Type = mesos.RANGES.Enum() + r.Ranges = r.Ranges.Add(&mesos.Value_Ranges{Range: rs}) + } +} + +func Resources(r ...mesos.Resource) (result mesos.Resources) { + return result.Add(r...) +} + +func Reservation(ri *mesos.Resource_ReservationInfo) Opt { + return func(r *mesos.Resource) { + r.Reservation = ri + } +} + +func Disk(persistenceID, containerPath string) Opt { + return func(r *mesos.Resource) { + r.Disk = &mesos.Resource_DiskInfo{} + if containerPath != "" { + r.Disk.Volume = &mesos.Volume{ContainerPath: containerPath} + } + if persistenceID != "" { + r.Disk.Persistence = &mesos.Resource_DiskInfo_Persistence{ID: persistenceID} + } + } +} + +func ReservedBy(principal string) *mesos.Resource_ReservationInfo { + result := &mesos.Resource_ReservationInfo{} + if principal != "" { + result.Principal = &principal + } + return result +} + +func Reserve(r mesos.Resources) *mesos.Offer_Operation { + return &mesos.Offer_Operation{ + Type: mesos.Offer_Operation_RESERVE.Enum(), + Reserve: &mesos.Offer_Operation_Reserve{ + Resources: r, + }, + } +} + +func Unreserve(r mesos.Resources) *mesos.Offer_Operation { + return &mesos.Offer_Operation{ + Type: mesos.Offer_Operation_UNRESERVE.Enum(), + Unreserve: &mesos.Offer_Operation_Unreserve{ + Resources: r, + }, + } +} + +func Create(r mesos.Resources) *mesos.Offer_Operation { + return &mesos.Offer_Operation{ + Type: mesos.Offer_Operation_CREATE.Enum(), + Create: &mesos.Offer_Operation_Create{ + Volumes: r, + }, + } +} + +func Expect(t *testing.T, cond bool, msgformat string, args ...interface{}) bool { + if !cond { + t.Errorf(msgformat, args...) + } + return cond +} diff --git a/api/v1/lib/scheduler/calls/caller.go b/api/v1/lib/scheduler/calls/caller.go deleted file mode 100644 index 772dc850..00000000 --- a/api/v1/lib/scheduler/calls/caller.go +++ /dev/null @@ -1,116 +0,0 @@ -package calls - -import ( - "github.com/mesos/mesos-go/api/v1/lib" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -// Caller is the public interface this framework scheduler's should consume -type ( - Caller interface { - // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. - Call(*scheduler.Call) (mesos.Response, error) - } - - // CallerFunc is the functional adaptation of the Caller interface - CallerFunc func(*scheduler.Call) (mesos.Response, error) - - // Decorator funcs usually return a Caller whose behavior has been somehow modified - Decorator func(Caller) Caller - - // Decorators is a convenience type that applies multiple Decorator functions to a Caller - Decorators []Decorator -) - -// Call implements the Caller interface for CallerFunc -func (f CallerFunc) Call(c *scheduler.Call) (mesos.Response, error) { return f(c) } - -// Apply is a convenient, nil-safe applicator that returns the result of d(c) iff d != nil; otherwise c -func (d Decorator) Apply(c Caller) (result Caller) { - if d != nil { - result = d(c) - } else { - result = c - } - return -} - -// Apply is a convenience function that applies the combined product of the decorators to the given Caller. -func (ds Decorators) Apply(c Caller) Caller { - return ds.Combine()(c) -} - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Caller that is ultimately returned. -func (ds Decorators) Combine() (result Decorator) { - actual := make(Decorators, 0, len(ds)) - for _, d := range ds { - if d != nil { - actual = append(actual, d) - } - } - if len(actual) == 0 { - result = noopDecorator - } else { - result = Decorator(func(h Caller) Caller { - for _, d := range actual { - h = d(h) - } - return h - }) - } - return -} - -// FrameworkCaller generates and returns a Decorator that applies the given frameworkID to all calls. -// Deprecated in favor of SubscribedCaller; should remove after v0.0.3. -func FrameworkCaller(frameworkID string) Decorator { - return func(h Caller) Caller { - return CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { - c.FrameworkID = &mesos.FrameworkID{Value: frameworkID} - return h.Call(c) - }) - } -} - -// SubscribedCaller returns a Decorator that injects a framework ID to all calls, with the following exceptions: -// - SUBSCRIBE calls are never modified (schedulers should explicitly construct such calls) -// - calls are not modified when the generated framework ID is "" -func SubscribedCaller(frameworkID func() string) Decorator { - return func(h Caller) Caller { - return CallerFunc(func(c *scheduler.Call) (mesos.Response, error) { - // never overwrite framework ID for subscribe calls; the scheduler must do that part - if c.GetType() != scheduler.Call_SUBSCRIBE { - if fid := frameworkID(); fid != "" { - c.FrameworkID = &mesos.FrameworkID{Value: fid} - } - } - return h.Call(c) - }) - } -} - -var noopDecorator = Decorator(func(h Caller) Caller { return h }) - -// CallNoData is a convenience func that executes the given Call using the provided Caller -// and always drops the response data. -func CallNoData(caller Caller, call *scheduler.Call) error { - resp, err := caller.Call(call) - if resp != nil { - resp.Close() - } - return err -} diff --git a/api/v1/lib/scheduler/calls/calls.go b/api/v1/lib/scheduler/calls/calls.go index 3a41b8b7..42fdbfc0 100644 --- a/api/v1/lib/scheduler/calls/calls.go +++ b/api/v1/lib/scheduler/calls/calls.go @@ -36,6 +36,14 @@ func RefuseSecondsWithJitter(r *rand.Rand, d time.Duration) scheduler.CallOpt { }) } +// RefuseSeconds returns a calls.Filters option that sets RefuseSeconds to the given duration +func RefuseSeconds(d time.Duration) scheduler.CallOpt { + asFloat := d.Seconds() + return Filters(func(f *mesos.Filters) { + f.RefuseSeconds = &asFloat + }) +} + // Framework sets a scheduler.Call's FrameworkID func Framework(id string) scheduler.CallOpt { return func(c *scheduler.Call) { diff --git a/api/v1/lib/scheduler/calls/calls_generated.go b/api/v1/lib/scheduler/calls/calls_generated.go new file mode 100644 index 00000000..9645905f --- /dev/null +++ b/api/v1/lib/scheduler/calls/calls_generated.go @@ -0,0 +1,38 @@ +package calls + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type C:*scheduler.Call +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + // Caller is the public interface this framework scheduler's should consume + Caller interface { + // Call issues a call to Mesos and properly manages call-specific HTTP response headers & data. + Call(context.Context, *scheduler.Call) (mesos.Response, error) + } + + // CallerFunc is the functional adaptation of the Caller interface + CallerFunc func(context.Context, *scheduler.Call) (mesos.Response, error) +) + +// Call implements the Caller interface for CallerFunc +func (f CallerFunc) Call(ctx context.Context, c *scheduler.Call) (mesos.Response, error) { + return f(ctx, c) +} + +// CallNoData is a convenience func that executes the given Call using the provided Caller +// and always drops the response data. +func CallNoData(ctx context.Context, caller Caller, call *scheduler.Call) error { + resp, err := caller.Call(ctx, call) + if resp != nil { + resp.Close() + } + return err +} diff --git a/api/v1/lib/scheduler/calls/errors.go b/api/v1/lib/scheduler/calls/errors.go new file mode 100644 index 00000000..3d32635d --- /dev/null +++ b/api/v1/lib/scheduler/calls/errors.go @@ -0,0 +1,13 @@ +package calls + +import ( + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +// AckError wraps a caller-generated error and tracks the call that failed. +type AckError struct { + Ack *scheduler.Call + Cause error +} + +func (err *AckError) Error() string { return err.Cause.Error() } diff --git a/api/v1/lib/scheduler/calls/gen.go b/api/v1/lib/scheduler/calls/gen.go new file mode 100644 index 00000000..1be50f95 --- /dev/null +++ b/api/v1/lib/scheduler/calls/gen.go @@ -0,0 +1,3 @@ +package calls + +//go:generate go run ../../extras/gen/callers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type C:*scheduler.Call diff --git a/api/v1/lib/scheduler/calls/metrics.go b/api/v1/lib/scheduler/calls/metrics.go deleted file mode 100644 index 2319a7bc..00000000 --- a/api/v1/lib/scheduler/calls/metrics.go +++ /dev/null @@ -1,25 +0,0 @@ -package calls - -import ( - "strings" - - "github.com/mesos/mesos-go/api/v1/lib" - xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -func CallerMetrics(harness xmetrics.Harness) Decorator { - return func(caller Caller) (metricsCaller Caller) { - if caller != nil { - metricsCaller = CallerFunc(func(c *scheduler.Call) (res mesos.Response, err error) { - typename := strings.ToLower(c.GetType().String()) - harness(func() error { - res, err = caller.Call(c) - return err // need to count these - }, typename) - return - }) - } - return - } -} diff --git a/api/v1/lib/scheduler/events.go b/api/v1/lib/scheduler/events.go deleted file mode 100644 index c998f003..00000000 --- a/api/v1/lib/scheduler/events.go +++ /dev/null @@ -1,18 +0,0 @@ -package scheduler - -// EventPredicate funcs evaluate scheduler events and determine some pass/fail outcome; useful for -// filtering or conditionally handling events. -type EventPredicate func(*Event) bool - -// Apply returns the result of the predicate function; always true if the predicate is nil. -func (ep EventPredicate) Apply(e *Event) (result bool) { - if ep == nil { - result = true - } else { - result = ep(e) - } - return -} - -// Happens implements scheduler/events.Happens -func (t Event_Type) Happens() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } diff --git a/api/v1/lib/scheduler/events/decorators.go b/api/v1/lib/scheduler/events/decorators.go deleted file mode 100644 index e4699af9..00000000 --- a/api/v1/lib/scheduler/events/decorators.go +++ /dev/null @@ -1,66 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -type ( - // Decorator functions typically modify behavior of the given delegate Handler. - Decorator func(Handler) Handler - - // Decorators aggregates Decorator functions - Decorators []Decorator -) - -var noopDecorator = Decorator(func(h Handler) Handler { return h }) - -// If returns the receiving Decorator if the given bool is true; otherwise returns a no-op -// Decorator instance. -func (d Decorator) If(b bool) Decorator { - if d == nil { - return noopDecorator - } - result := noopDecorator - if b { - result = d - } - return result -} - -// When returns a Decorator that evaluates the bool func every time the Handler is invoked. -// When f returns true, the Decorated Handler is invoked, otherwise the original Handler is. -func (d Decorator) When(f func() bool) Decorator { - if d == nil || f == nil { - return noopDecorator - } - return func(h Handler) Handler { - // generates a new decorator every time the Decorator func is invoked. - // probably OK for now. - decorated := d(h) - return HandlerFunc(func(e *scheduler.Event) (err error) { - if f() { - // let the decorated handler process this - err = decorated.HandleEvent(e) - } else { - err = h.HandleEvent(e) - } - return - }) - } -} - -// Apply applies the Decorators in the order they're listed such that the last Decorator invoked -// generates the final (wrapping) Handler that is ultimately returned. -func (ds Decorators) Apply(h Handler) Handler { - for _, d := range ds { - h = d.Apply(h) - } - return h -} - -func (d Decorator) Apply(h Handler) Handler { - if d != nil { - h = d(h) - } - return h -} diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go deleted file mode 100644 index ed482801..00000000 --- a/api/v1/lib/scheduler/events/events.go +++ /dev/null @@ -1,192 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/scheduler" - "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" -) - -type ( - // Handler is invoked upon the occurrence of some scheduler event that is generated - // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) - Handler interface { - HandleEvent(*scheduler.Event) error - } - - // HandlerFunc is a functional adaptation of the Handler interface - HandlerFunc func(*scheduler.Event) error - - // Mux maps event types to Handlers (only one Handler for each type). A "default" - // Handler implementation may be provided to handle cases in which there is no - // registered Handler for specific event type. - Mux struct { - handlers map[scheduler.Event_Type]Handler - defaultHandler Handler - } - - // Option is a functional configuration option that returns an "undo" option that - // reverts the change made by the option. - Option func(*Mux) Option - - // Handlers aggregates Handler things - Handlers []Handler - - Happens interface { - Happens() scheduler.EventPredicate - } -) - -// HandleEvent implements Handler for HandlerFunc -func (f HandlerFunc) HandleEvent(e *scheduler.Event) error { return f(e) } - -// NewMux generates and returns a new, empty Mux instance. -func NewMux(opts ...Option) *Mux { - m := &Mux{ - handlers: make(map[scheduler.Event_Type]Handler), - } - m.With(opts...) - return m -} - -// With applies the given options to the Mux and returns the result of invoking -// the last Option func. If no options are provided then a no-op Option is returned. -func (m *Mux) With(opts ...Option) Option { - var last Option // defaults to noop - last = Option(func(x *Mux) Option { return last }) - - for _, o := range opts { - if o != nil { - last = o(m) - } - } - return last -} - -// HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *scheduler.Event) (err error) { - h, found := m.handlers[e.GetType()] - if !found { - h = m.defaultHandler - } - if h != nil { - err = h.HandleEvent(e) - } - return -} - -// Handle returns an option that configures a Handler to handle a specific event type. -// If the specified Handler is nil then any currently registered Handler for the given -// event type is deleted upon application of the returned Option. -func Handle(et scheduler.Event_Type, eh Handler) Option { - return func(m *Mux) Option { - old := m.handlers[et] - if eh == nil { - delete(m.handlers, et) - } else { - m.handlers[et] = eh - } - return Handle(et, old) - } -} - -// Map returns an Option that configures multiple Handler objects. -func Map(handlers map[scheduler.Event_Type]Handler) (option Option) { - option = func(m *Mux) Option { - type history struct { - et scheduler.Event_Type - h Handler - } - old := make([]history, len(handlers)) - for et, h := range handlers { - old = append(old, history{et, m.handlers[et]}) - m.handlers[et] = h - } - return func(m *Mux) Option { - for i := range old { - if old[i].h == nil { - delete(m.handlers, old[i].et) - } else { - m.handlers[old[i].et] = old[i].h - } - } - return option - } - } - return -} - -// MapFuncs is the functional adaptation of Map -func MapFuncs(handlers map[scheduler.Event_Type]HandlerFunc) (option Option) { - h := make(map[scheduler.Event_Type]Handler, len(handlers)) - for k, v := range handlers { - h[k] = v - } - return Map(h) -} - -// DefaultHandler returns an option that configures the default handler that's invoked -// in cases where there is no Handler registered for specific event type. -func DefaultHandler(eh Handler) Option { - return func(m *Mux) Option { - old := m.defaultHandler - m.defaultHandler = eh - return DefaultHandler(old) - } -} - -// AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every -// UPDATE event that's received. -func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { - return WhenFunc(scheduler.Event_UPDATE, func(e *scheduler.Event) (err error) { - var ( - s = e.GetUpdate().GetStatus() - uuid = s.GetUUID() - ) - if len(uuid) > 0 { - ack := calls.Acknowledge( - s.GetAgentID().GetValue(), - s.TaskID.Value, - uuid, - ) - err = calls.CallNoData(callerGetter(), ack) - } - return - }) -} - -func Once(h Handler) Handler { - called := false - return HandlerFunc(func(e *scheduler.Event) (err error) { - if !called { - called = true - err = h.HandleEvent(e) - } - return - }) -} - -func OnceFunc(h HandlerFunc) Handler { return Once(h) } - -func When(p Happens, h Handler) Handler { - return HandlerFunc(func(e *scheduler.Event) (err error) { - if p.Happens().Apply(e) { - err = h.HandleEvent(e) - } - return - }) -} - -func WhenFunc(p Happens, h HandlerFunc) Handler { return When(p, h) } - -var _ = Handler(Handlers{}) // Handlers implements Handler - -// HandleEvent implements Handler for Handlers -func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { - for _, h := range hs { - if h != nil { - if err = h.HandleEvent(e); err != nil { - break - } - } - } - return err -} diff --git a/api/v1/lib/scheduler/events/events_generated.go b/api/v1/lib/scheduler/events/events_generated.go new file mode 100644 index 00000000..ff6439bb --- /dev/null +++ b/api/v1/lib/scheduler/events/events_generated.go @@ -0,0 +1,87 @@ +package events + +// go generate -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "context" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + // Handler is invoked upon the occurrence of some scheduler event that is generated + // by some other component in the Mesos ecosystem (e.g. master, agent, executor, etc.) + Handler interface { + HandleEvent(context.Context, *scheduler.Event) error + } + + // HandlerFunc is a functional adaptation of the Handler interface + HandlerFunc func(context.Context, *scheduler.Event) error + + // Handlers executes an event Handler according to the event's type + Handlers map[scheduler.Event_Type]Handler + + // HandlerFuncs executes an event HandlerFunc according to the event's type + HandlerFuncs map[scheduler.Event_Type]HandlerFunc +) + +// HandleEvent implements Handler for HandlerFunc +func (f HandlerFunc) HandleEvent(ctx context.Context, e *scheduler.Event) error { return f(ctx, e) } + +type noopHandler int + +func (_ noopHandler) HandleEvent(_ context.Context, _ *scheduler.Event) error { return nil } + +// NoopHandler is a Handler that does nothing and always returns nil +const NoopHandler = noopHandler(0) + +// HandleEvent implements Handler for Handlers +func (hs Handlers) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// HandleEvent implements Handler for HandlerFuncs +func (hs HandlerFuncs) HandleEvent(ctx context.Context, e *scheduler.Event) (err error) { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return nil +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the Handlers map; unmatched event types are +// processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs Handlers) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *scheduler.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +// Otherwise returns a HandlerFunc that attempts to process an event with the HandlerFuncs map; unmatched event types +// are processed by the given HandlerFunc. A nil HandlerFunc parameter is effecitvely a noop. +func (hs HandlerFuncs) Otherwise(f HandlerFunc) HandlerFunc { + if f == nil { + return hs.HandleEvent + } + return func(ctx context.Context, e *scheduler.Event) error { + if h := hs[e.GetType()]; h != nil { + return h.HandleEvent(ctx, e) + } + return f(ctx, e) + } +} + +var ( + _ = Handler(Handlers(nil)) + _ = Handler(HandlerFunc(nil)) + _ = Handler(HandlerFuncs(nil)) +) diff --git a/api/v1/lib/scheduler/events/gen.go b/api/v1/lib/scheduler/events/gen.go new file mode 100644 index 00000000..259a07cb --- /dev/null +++ b/api/v1/lib/scheduler/events/gen.go @@ -0,0 +1,3 @@ +package events + +//go:generate go run ../../extras/gen/handlers.go ../../extras/gen/gen.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -type E:*scheduler.Event:&scheduler.Event{} -type ET:scheduler.Event_Type diff --git a/api/v1/lib/scheduler/events/metrics.go b/api/v1/lib/scheduler/events/metrics.go deleted file mode 100644 index 6693a00c..00000000 --- a/api/v1/lib/scheduler/events/metrics.go +++ /dev/null @@ -1,20 +0,0 @@ -package events - -import ( - "strings" - - xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -func Metrics(harness xmetrics.Harness) Decorator { - return func(h Handler) Handler { - if h == nil { - return h - } - return HandlerFunc(func(e *scheduler.Event) error { - typename := strings.ToLower(e.GetType().String()) - return harness(func() error { return h.HandleEvent(e) }, typename) - }) - } -} diff --git a/api/v1/lib/scheduler/events/predicates.go b/api/v1/lib/scheduler/events/predicates.go deleted file mode 100644 index fb5be74d..00000000 --- a/api/v1/lib/scheduler/events/predicates.go +++ /dev/null @@ -1,11 +0,0 @@ -package events - -import ( - "github.com/mesos/mesos-go/api/v1/lib/scheduler" -) - -type PredicateBool func() bool - -func (b PredicateBool) Happens() scheduler.EventPredicate { - return func(_ *scheduler.Event) bool { return b() } -} diff --git a/api/v1/vendor/vendor.json b/api/v1/vendor/vendor.json index b32651e1..9e8df969 100644 --- a/api/v1/vendor/vendor.json +++ b/api/v1/vendor/vendor.json @@ -3,706 +3,11 @@ "ignore": "test", "package": [ { - "checksumSHA1": "RIVAETtE3FvCtjbSOh0REf6JgQk=", - "path": "github.com/gogo/protobuf/codec", + "checksumSHA1": "2I4udA/dza74M1bOgQuEPz103gw=", + "path": "github.com/gogo/protobuf", "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "tEjFocOVKUr8IK6jkvegQlfqjOw=", - "path": "github.com/gogo/protobuf/gogoproto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CLfzGRKKVJkecd9WCEtnKI2NPtI=", - "path": "github.com/gogo/protobuf/io", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Ol9P36yJqSe2nMDjKyYmz7i3TRs=", - "path": "github.com/gogo/protobuf/jsonpb", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "paUMgsbzE5tXgsiZ+RTRcEI68I4=", - "path": "github.com/gogo/protobuf/jsonpb/jsonpb_test_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cTbYeqRl+/qhD1JZvZmT5OAjaXA=", - "path": "github.com/gogo/protobuf/plugin/defaultcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Pjc29+wvgMEWJkw9PqVwTWd5R/c=", - "path": "github.com/gogo/protobuf/plugin/description", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NP3xQexLANLm++9RCktWr1ovOY0=", - "path": "github.com/gogo/protobuf/plugin/embedcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yzLXQWgFsBkFkEHScmPkcwdN2iY=", - "path": "github.com/gogo/protobuf/plugin/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Tx1FOos8BwPTDNp1LhAoG8Co6ew=", - "path": "github.com/gogo/protobuf/plugin/equal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xS3aAjAbrgPgnNkTUlTOuAmxXQ8=", - "path": "github.com/gogo/protobuf/plugin/face", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ugvdtinMoMF6MdKu4lN+D+xb0gM=", - "path": "github.com/gogo/protobuf/plugin/gostring", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7VOG8kB8XBzoNkErOWyR3QdpF5I=", - "path": "github.com/gogo/protobuf/plugin/grpc", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "GK139O7VRP+DHzVOG7PIBrzEpYc=", - "path": "github.com/gogo/protobuf/plugin/marshalto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/dztWeNfEY/M2fBB2hjO4vC2idE=", - "path": "github.com/gogo/protobuf/plugin/oneofcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//9AgNP7EaQ42IGI5gp5GBlqq+8=", - "path": "github.com/gogo/protobuf/plugin/populate", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3doyBfoqf4eU3Rkn4nl6MK7I5QM=", - "path": "github.com/gogo/protobuf/plugin/size", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "6C5JEnHpRtsBQ+nhNnDuZtw35Gk=", - "path": "github.com/gogo/protobuf/plugin/stringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5/vombKpWe8yH5q2RH3wZPId0no=", - "path": "github.com/gogo/protobuf/plugin/testgen", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5fuoaG6BK9wOKAyp+AdIowdeIx8=", - "path": "github.com/gogo/protobuf/plugin/union", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zDRKoIeCInauKHmQkTuixhhAjmA=", - "path": "github.com/gogo/protobuf/plugin/unmarshal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hyS3vR96CWcq5guGQXKUWVrlvGA=", - "path": "github.com/gogo/protobuf/proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "WYKtRaKlF90mjEVLg8QuIFEuKH0=", - "path": "github.com/gogo/protobuf/proto/proto3_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "buFKlCd6bVy4ynsgaFZBC/NhhJM=", - "path": "github.com/gogo/protobuf/proto/testdata", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "VkdsPnt/LryLUDkSNuVVIqwq4Fc=", - "path": "github.com/gogo/protobuf/protoc-gen-combo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lYD+Czpx21KQfnMj/VJbD2ll1nI=", - "path": "github.com/gogo/protobuf/protoc-gen-gofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RNSh5A+eMLvpb36czti5+wfm4wA=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "u0+Vvbf2f8FWldJOohemISYPLYo=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/descriptor", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NQbUjvqwqPFFVa8H4ntwuViQtLE=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/generator", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DmPXpIbl/qpcje0w4JAJW4QjEDM=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/plugin", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "X0kogVRUqIRdipIVRbcTwZmODac=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "n6qb61BdGIyL1Sr6+k2L9WvbaQY=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofaster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "igZu+7E154ueL7+91KAoUH3Kb4c=", - "path": "github.com/gogo/protobuf/protoc-gen-gogoslick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "J1bPjNLs1CCZv3pnl2QMYiD9f7Q=", - "path": "github.com/gogo/protobuf/protoc-min-version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6cMbpJj41MpihS5eP4SIliRBK4=", - "path": "github.com/gogo/protobuf/sortkeys", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lbHS2lM7RHQE3xseWie466dKY2I=", - "path": "github.com/gogo/protobuf/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "SxY4AAxSzboRLBc38Yyn1s4okI0=", - "path": "github.com/gogo/protobuf/test/casttype", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CfGm603zYf9bqWAC0Z3eBdrzuGg=", - "path": "github.com/gogo/protobuf/test/casttype/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "FmoK2j/5ncq8T+eHCMPxSeqnzFY=", - "path": "github.com/gogo/protobuf/test/casttype/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "p4kmpuTD71Z3dvhbICrF2xbGosU=", - "path": "github.com/gogo/protobuf/test/casttype/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0c98JonLeFNSc3qdVyoVcfCriN4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mTi59HJocNgdkNGkYRLz2pn8+WE=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "TYBTBsMDW3Fsfs3s0E8CRNcxEu4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sIpq5aF4bvnqOa/4YK2gWQNcOoc=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4XzzMi5+/zaIuy2EWEe0ufCuQtI=", - "path": "github.com/gogo/protobuf/test/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "aq5aEfH8+HPgqKAo3U9EpMjZZUo=", - "path": "github.com/gogo/protobuf/test/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "OCUBqnfbWfraPTB8oW+8/cFiXKw=", - "path": "github.com/gogo/protobuf/test/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RqtUd2ZnGrwwf13KjYQwOK16F3M=", - "path": "github.com/gogo/protobuf/test/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "1d0nW4JvpT3ALIkT5f0y8T5M+xk=", - "path": "github.com/gogo/protobuf/test/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "PCBUEHiaKfjZZfo77m4W32qOeB4=", - "path": "github.com/gogo/protobuf/test/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Q6fiuAzwlM4kVK3PMF9vWnvmays=", - "path": "github.com/gogo/protobuf/test/custom", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2qwk23WTep6Z6DAq2hCMf9OsT/I=", - "path": "github.com/gogo/protobuf/test/custom-dash-type", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "h0LHbcpzexyjfSPu/LbrRM1Zg8c=", - "path": "github.com/gogo/protobuf/test/custombytesnonstruct", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "E2/sHgeCMMpdjoEBb4SY7G8p4Uk=", - "path": "github.com/gogo/protobuf/test/dashfilename", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9yHuzxDG9ZjtDFoJLEomVUFIyz0=", - "path": "github.com/gogo/protobuf/test/defaultconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "S+k0F1XhrDFd+igafb5FLceyUr0=", - "path": "github.com/gogo/protobuf/test/embedconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "75vwg1za0UP2O3YLwPR8HD14SOU=", - "path": "github.com/gogo/protobuf/test/empty-issue70", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ncjf6Xzc4vOfR9YidS6Cc9QTLtw=", - "path": "github.com/gogo/protobuf/test/enumprefix", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fsQ9GD7yVIWT2VC2qI6wHYIdISM=", - "path": "github.com/gogo/protobuf/test/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "UIBxh/5EDEL5DIvVoH4ISmHBDys=", - "path": "github.com/gogo/protobuf/test/example", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YiSMdmPIdyFvXvT/tlc86DjwJLI=", - "path": "github.com/gogo/protobuf/test/fuzztests", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yp7uRKY5gyfh6vSXnVjXIzGAXE4=", - "path": "github.com/gogo/protobuf/test/group", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DeBkH1l+Gub2I4LgrWrnWbLVjpE=", - "path": "github.com/gogo/protobuf/test/importdedup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AVJIC2MdvEh9UfbN40073Z/d0f8=", - "path": "github.com/gogo/protobuf/test/importdedup/subpkg", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0AWSyTxYd97FyMn7gZ95Ej/3m4s=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/t6kWiQ82zTH5KSMafg/uQ16ask=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72/index", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9NEWM8d2HXtilq+/QR+rLmJhOkI=", - "path": "github.com/gogo/protobuf/test/issue34", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "EUSuYKzjDRdtYYSF7r30Xs7FgDg=", - "path": "github.com/gogo/protobuf/test/issue42order", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7fjvhomoxX3rt5gJZZBWp5sKkhs=", - "path": "github.com/gogo/protobuf/test/issue8", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4ZPo2LIaVv3lJZEQYwPLhlvBktc=", - "path": "github.com/gogo/protobuf/test/mapsproto2", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cNiwjpj2RmyJNjiqvJoFQUPsRqM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "vR6wpg/XpCULa9KTONFHXrFLmIQ=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mqF9ePL2nMR0iyJWBwpZl7VSjqA=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "psHlD68ijJjBBHqN/MgG4MGkC3c=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hCfdSModEttRBnDgDBy2Zx0e/uc=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cIszinXlkReEzP4DLVqhFr80loI=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AqEcvNXhMBPX9y4C1JdU5ci6djM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//KmhDh/C/fHRA3oGNMSGYALpuU=", - "path": "github.com/gogo/protobuf/test/mixbench", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0fgRfF8zijkFLXG6OcmgfXv+070=", - "path": "github.com/gogo/protobuf/test/moredefaults", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ahMVz714gNBvYBmO2I+T2EJIeIc=", - "path": "github.com/gogo/protobuf/test/oneof", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/vpBUlSaib5le0urbk9j8ySiRD0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4TGArepiMDOGxLQUnyIFWkCV860=", - "path": "github.com/gogo/protobuf/test/oneof/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sbzC8eZJZ5VWaGwzRxSTPIPQPME=", - "path": "github.com/gogo/protobuf/test/oneof/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YgoWSjBZxO1FAkc1tXUndt4IXS0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "O8WR3D8SHq0gFxa7+rf0MdelAoA=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ebUJvjKLbg5R9F7nTUuNT5s661w=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "HWM1axY96SEoc1bY2edTjlld9c4=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YBnrab1VG14LcgYyVLM2uFswhgc=", - "path": "github.com/gogo/protobuf/test/oneof3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2MOPs9vr5XgDUTtjm8E67tJnXyo=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xfBBM0eq7rW41faUfZh75z+Xj3I=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "T7NdX8Dsv3mfYDXic/YVo1urNSQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zb2LHnn4ql6QQwgVpE1JkZIxcHE=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6bMJP3iB2kyb8e6p3GdoRCBtCk=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "QdexN5eBoxSSga9kn8uw3la1jJA=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fDFnUhPE51bxH9yhMjWpEPu/IQQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "XVtCxPLxfWUo33u2xSRP1hzgd/0=", - "path": "github.com/gogo/protobuf/test/packed", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ZYwqz+QwhSsp/1YKpn1ixA4ahCE=", - "path": "github.com/gogo/protobuf/test/required", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "bCqTYwvGJl6k/7nSxSmMjtIs4VE=", - "path": "github.com/gogo/protobuf/test/sizeunderscore", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9ZhQFwpC3uupLOELXbdLJ2Ss040=", - "path": "github.com/gogo/protobuf/test/tags", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eM0kVXUbz6sinSfxARQe/cTk15k=", - "path": "github.com/gogo/protobuf/test/theproto3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eBxOWTGmgxhWG1U1wcSjuOFl4sU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "BhPGpjdC9375YDejUFVlsfzolXg=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "MdGf5p9TOSeoMj0vGhFHqktACNk=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oSeY/ryVt9ZenUM4xC4ocDGB6Oc=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ivhKH76IrT1EHyWzImjedlTPdCo=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oXfCUeKjTKLTqjmJlDUaz0HZesU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2QEqlqrIIY9Y1On/qVoMu6ABj80=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "IG6H7z0wDBAqcsF7sojaOWTWNI4=", - "path": "github.com/gogo/protobuf/test/unmarshalmerge", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Y5w3d7x5quR19o+iT/HhphY89/I=", - "path": "github.com/gogo/protobuf/test/unrecognized", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "8CjEqnI9sTWfSeGX98CNrrilguw=", - "path": "github.com/gogo/protobuf/test/unrecognizedgroup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3KeNOHQHQb4ZjB3XpigiYUDqLWM=", - "path": "github.com/gogo/protobuf/vanity", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6dohUIo6Qf0ixPXzxH7LExfrDk=", - "path": "github.com/gogo/protobuf/vanity/command", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0/Iq33SFI/UbS8LJeTyOadMxgPE=", - "path": "github.com/gogo/protobuf/vanity/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "jEwyZC+hiPl2avUlrgEDhtCEiT4=", - "path": "github.com/gogo/protobuf/vanity/test/fast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "F3rR7HTm2Wi4jlPedckmvNT7TI8=", - "path": "github.com/gogo/protobuf/vanity/test/faster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "C9/L7Jpst+Unink8JiZuXLHJgv4=", - "path": "github.com/gogo/protobuf/vanity/test/slick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "KLFcaMb5M2MQ7Sl3gLXl70wgKIg=", - "path": "github.com/gogo/protobuf/version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" + "revisionTime": "2015-09-16T11:01:26Z", + "tree": true }, { "checksumSHA1": "ANHyMfOc1XnqDnlxNipBdpFE1qw=",