diff --git a/Makefile b/Makefile index 66629486..d2142b4f 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ $(COVERAGE_TARGETS): .PHONY: vet vet: - go $@ $(PACKAGES) + go $@ $(PACKAGES) $(BINARIES) .PHONY: codecs codecs: protobufs ffjson @@ -77,6 +77,10 @@ sync: (cd ${API_VENDOR}; govendor sync) (cd ${CMD_VENDOR}; govendor sync) +.PHONY: generate +generate: + go generate ./api/v1/lib/extras/scheduler/eventrules + GOPKG := github.com/mesos/mesos-go GOPKG_DIRNAME := $(shell dirname $(GOPKG)) UID ?= $(shell id -u $$USER) diff --git a/api/v1/cmd/example-scheduler/app/app.go b/api/v1/cmd/example-scheduler/app/app.go index 1ecb3f28..d21953f3 100644 --- a/api/v1/cmd/example-scheduler/app/app.go +++ b/api/v1/cmd/example-scheduler/app/app.go @@ -2,7 +2,6 @@ package app import ( "errors" - "fmt" "io" "log" "strconv" @@ -12,6 +11,8 @@ import ( "github.com/mesos/mesos-go/api/v1/lib/backoff" xmetrics "github.com/mesos/mesos-go/api/v1/lib/extras/metrics" "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" @@ -41,108 +42,84 @@ func Run(cfg Config) error { // TODO(jdef) how to track/handle timeout errors that occur for SUBSCRIBE calls? we should // probably tolerate X number of subsequent subscribe failures before bailing. we'll need // to track the lastCallAttempted along with subsequentSubscribeTimeouts. - err = controller.New().Run(buildControllerConfig(state, shutdown)) - if state.err != nil { - err = state.err - } - return err -} -func buildControllerConfig(state *internalState, shutdown <-chan struct{}) controller.Config { - var ( - frameworkIDStore = NewInMemoryIDStore() - controlContext = &controller.ContextAdapter{ - DoneFunc: state.isDone, - FrameworkIDFunc: func() string { return frameworkIDStore.Get() }, - ErrorFunc: func(err error) { - if err != nil { - if err != io.EOF { - log.Println(err) - } - if _, ok := err.(StateError); ok { - state.markDone() - } - return - } - log.Println("disconnected") - }, - } - ) + frameworkIDStore := store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) state.cli = calls.Decorators{ callMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), logCalls(map[scheduler.Call_Type]string{scheduler.Call_SUBSCRIBE: "connecting..."}), - // automatically set the frameworkID for all outgoing calls - calls.SubscribedCaller(frameworkIDStore.Get), + calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), // automatically set the frameworkID for all outgoing calls }.Apply(state.cli) - return controller.Config{ - Context: controlContext, - Framework: buildFrameworkInfo(state.config), - Caller: state.cli, - RegistrationTokens: backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), - - Handler: events.Decorators{ + err = controller.Run( + buildFrameworkInfo(state.config), + state.cli, + controller.WithDone(state.done.Closed), + controller.WithEventHandler( + buildEventHandler(state, frameworkIDStore), eventMetrics(state.metricsAPI, time.Now, state.config.summaryMetrics), events.Decorator(logAllEvents).If(state.config.verbose), - }.Apply(buildEventHandler(state, frameworkIDStore)), + ), + controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithRegistrationTokens( + backoff.Notifier(RegistrationMinBackoff, RegistrationMaxBackoff, shutdown), + ), + controller.WithSubscriptionTerminated(func(err error) { + if err != nil { + if err != io.EOF { + log.Println(err) + } + if _, ok := err.(StateError); ok { + state.done.Close() + } + return + } + log.Println("disconnected") + }), + ) + if state.err != nil { + err = state.err } + return err } // buildEventHandler generates and returns a handler to process events received from the subscription. -func buildEventHandler(state *internalState, frameworkIDStore IDStore) events.Handler { - // TODO(jdef) would be nice to merge this ack handler with the status update handler below; need to - // figure out appropriate error propagation among chained handlers. - ack := events.AcknowledgeUpdates(func() calls.Caller { return state.cli }) - return events.NewMux( - events.DefaultHandler(events.HandlerFunc(controller.DefaultHandler)), - events.MapFuncs(map[scheduler.Event_Type]events.HandlerFunc{ - scheduler.Event_FAILURE: func(e *scheduler.Event) error { - log.Println("received a FAILURE event") - f := e.GetFailure() - failure(f.ExecutorID, f.AgentID, f.Status) - return nil - }, - scheduler.Event_OFFERS: func(e *scheduler.Event) error { +func buildEventHandler(state *internalState, frameworkIDStore store.Singleton) events.Handler { + logger := controller.LogEvents() + return controller.LiftErrors().Handle(events.HandlerSet{ + scheduler.Event_FAILURE: logger.HandleF(func(e *scheduler.Event) error { + f := e.GetFailure() + failure(f.ExecutorID, f.AgentID, f.Status) + return nil + }), + scheduler.Event_OFFERS: trackOffersReceived(state).AndThen().HandleF( + func(e *scheduler.Event) error { if state.config.verbose { log.Println("received an OFFERS event") } - offers := e.GetOffers().GetOffers() - state.metricsAPI.offersReceived.Int(len(offers)) - resourceOffers(state, offers) - return nil - }, - scheduler.Event_UPDATE: func(e *scheduler.Event) error { - if err := ack.HandleEvent(e); err != nil { - log.Printf("failed to ack status update for task: %+v", err) - // TODO(jdef) we don't return the error because that would cause the subscription - // to terminate; is that the right thing to do? - } - statusUpdate(state, e.GetUpdate().GetStatus()) - return nil - }, - scheduler.Event_SUBSCRIBED: func(e *scheduler.Event) error { - log.Println("received a SUBSCRIBED event") - frameworkID := e.GetSubscribed().GetFrameworkID().GetValue() - // order of `if` statements are important: tread carefully w/ respect to future changes - if frameworkID == "" { - // sanity check, should **never** happen - return StateError("mesos gave us an empty frameworkID") - } - if state.frameworkID != "" && state.frameworkID != frameworkID && state.config.checkpoint { - return StateError(fmt.Sprintf( - "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", - state.config.failoverTimeout)) - } - if state.frameworkID != frameworkID { - state.frameworkID = frameworkID - frameworkIDStore.Set(frameworkID) - log.Println("FrameworkID", frameworkID) - } - return nil - }, - }), - ) + return resourceOffers(state, e.GetOffers().GetOffers()) + }), + scheduler.Event_UPDATE: controller.AckStatusUpdates(state.cli).AndThen().HandleF(statusUpdate(state)), + scheduler.Event_SUBSCRIBED: eventrules.Rules{ + logger, + controller.TrackSubscription(frameworkIDStore, state.config.failoverTimeout), + }, + }) +} + +func trackOffersReceived(state *internalState) eventrules.Rule { + return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + if err == nil { + state.metricsAPI.offersReceived.Int(len(e.GetOffers().GetOffers())) + } + return chain(e, nil) + + } } func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { @@ -162,7 +139,7 @@ func failure(eid *mesos.ExecutorID, aid *mesos.AgentID, stat *int32) { } } -func resourceOffers(state *internalState, offers []mesos.Offer) { +func resourceOffers(state *internalState, offers []mesos.Offer) error { callOption := calls.RefuseSecondsWithJitter(state.random, state.config.maxRefuseSeconds) tasksLaunchedThisCycle := 0 offersDeclined := 0 @@ -238,36 +215,41 @@ func resourceOffers(state *internalState, offers []mesos.Offer) { if state.config.summaryMetrics { state.metricsAPI.launchesPerOfferCycle(float64(tasksLaunchedThisCycle)) } + return nil } -func statusUpdate(state *internalState, s mesos.TaskStatus) { - if state.config.verbose { - msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() - if m := s.GetMessage(); m != "" { - msg += " with message '" + m + "'" +func statusUpdate(state *internalState) events.HandlerFunc { + return func(e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() + if state.config.verbose { + msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String() + if m := s.GetMessage(); m != "" { + msg += " with message '" + m + "'" + } + log.Println(msg) } - log.Println(msg) - } - switch st := s.GetState(); st { - case mesos.TASK_FINISHED: - state.tasksFinished++ - state.metricsAPI.tasksFinished() + switch st := s.GetState(); st { + case mesos.TASK_FINISHED: + state.tasksFinished++ + state.metricsAPI.tasksFinished() - if state.tasksFinished == state.totalTasks { - log.Println("mission accomplished, terminating") - state.markDone() - } else { - tryReviveOffers(state) - } + if state.tasksFinished == state.totalTasks { + log.Println("mission accomplished, terminating") + state.done.Close() + } else { + tryReviveOffers(state) + } - case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: - state.err = errors.New("Exiting because task " + s.GetTaskID().Value + - " is in an unexpected state " + st.String() + - " with reason " + s.GetReason().String() + - " from source " + s.GetSource().String() + - " with message '" + s.GetMessage() + "'") - state.markDone() + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + state.err = errors.New("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") + state.done.Close() + } + return nil } } diff --git a/api/v1/cmd/example-scheduler/app/config.go b/api/v1/cmd/example-scheduler/app/config.go index ecd7cbb9..9727dff4 100644 --- a/api/v1/cmd/example-scheduler/app/config.go +++ b/api/v1/cmd/example-scheduler/app/config.go @@ -49,8 +49,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.Var(&cfg.codec, "codec", "Codec to encode/decode scheduler API communications [protobuf, json]") fs.StringVar(&cfg.url, "url", cfg.url, "Mesos scheduler API URL") fs.DurationVar(&cfg.timeout, "timeout", cfg.timeout, "Mesos scheduler API connection timeout") - fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout") - fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable framework checkpointing") + fs.DurationVar(&cfg.failoverTimeout, "failoverTimeout", cfg.failoverTimeout, "Framework failover timeout (recover from scheduler failure)") + fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable agent checkpointing for framework tasks (recover from agent failure)") fs.StringVar(&cfg.principal, "principal", cfg.principal, "Framework principal with which to authenticate") fs.StringVar(&cfg.hostname, "hostname", cfg.hostname, "Framework hostname that is advertised to the master") fs.Var(&cfg.labels, "label", "Framework label, may be specified multiple times") diff --git a/api/v1/cmd/example-scheduler/app/state.go b/api/v1/cmd/example-scheduler/app/state.go index fa484271..de768e88 100644 --- a/api/v1/cmd/example-scheduler/app/state.go +++ b/api/v1/cmd/example-scheduler/app/state.go @@ -8,12 +8,12 @@ import ( "math/rand" "net/http" "os" - "sync" "time" proto "github.com/gogo/protobuf/proto" "github.com/mesos/mesos-go/api/v1/lib" "github.com/mesos/mesos-go/api/v1/lib/backoff" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/httpcli" "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" @@ -87,8 +87,8 @@ func prepareExecutorInfo( func buildWantsTaskResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.taskCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.taskMemory).Resource, + mesos.CPUs(config.taskCPU).Resource, + mesos.Memory(config.taskMemory).Resource, ) log.Println("wants-task-resources = " + r.String()) return @@ -96,8 +96,8 @@ func buildWantsTaskResources(config Config) (r mesos.Resources) { func buildWantsExecutorResources(config Config) (r mesos.Resources) { r.Add( - *mesos.BuildResource().Name("cpus").Scalar(config.execCPU).Resource, - *mesos.BuildResource().Name("mem").Scalar(config.execMemory).Resource, + mesos.CPUs(config.execCPU).Resource, + mesos.Memory(config.execMemory).Resource, ) log.Println("wants-executor-resources = " + r.String()) return @@ -117,13 +117,6 @@ func buildHTTPSched(cfg Config, creds credentials) calls.Caller { httpcli.Do(httpcli.With( authConfigOpt, httpcli.Timeout(cfg.timeout), - httpcli.Transport(func(t *http.Transport) { - // all calls should be ack'd by the server within this interval. - // TODO(jdef) it probably doesn't make sense if this value is larger - // than cfg.timeout. - t.ResponseHeaderTimeout = 15 * time.Second - t.MaxIdleConnsPerHost = 2 // don't depend on go's default - }), )), ) if cfg.compression { @@ -216,31 +209,15 @@ func newInternalState(cfg Config) (*internalState, error) { metricsAPI: metricsAPI, cli: buildHTTPSched(cfg, creds), random: rand.New(rand.NewSource(time.Now().Unix())), - done: make(chan struct{}), + done: latch.New(), } return state, nil } -func (state *internalState) markDone() { - state.doneOnce.Do(func() { - close(state.done) - }) -} - -func (state *internalState) isDone() bool { - select { - case <-state.done: - return true - default: - return false - } -} - type internalState struct { tasksLaunched int tasksFinished int totalTasks int - frameworkID string role string executor *mesos.ExecutorInfo cli calls.Caller @@ -249,7 +226,6 @@ type internalState struct { reviveTokens <-chan struct{} metricsAPI *metricsAPI err error - done chan struct{} - doneOnce sync.Once + done latch.Interface random *rand.Rand } diff --git a/api/v1/cmd/example-scheduler/app/store.go b/api/v1/cmd/example-scheduler/app/store.go deleted file mode 100644 index bc7fa7a2..00000000 --- a/api/v1/cmd/example-scheduler/app/store.go +++ /dev/null @@ -1,43 +0,0 @@ -package app - -import "sync/atomic" - -// IDStore is a thread-safe abstraction to load and store a stringified ID. -type IDStore interface { - Get() string - Set(string) -} - -type IDStoreAdapter struct { - GetFunc func() string - SetFunc func(string) -} - -func (a IDStoreAdapter) Get() string { - if a.GetFunc != nil { - return a.GetFunc() - } - return "" -} - -func (a IDStoreAdapter) Set(s string) { - if a.SetFunc != nil { - a.SetFunc(s) - } -} - -func NewInMemoryIDStore() IDStore { - var frameworkID atomic.Value - return &IDStoreAdapter{ - GetFunc: func() string { - x := frameworkID.Load() - if x == nil { - return "" - } - return x.(string) - }, - SetFunc: func(s string) { - frameworkID.Store(s) - }, - } -} diff --git a/api/v1/cmd/msh/msh.go b/api/v1/cmd/msh/msh.go new file mode 100644 index 00000000..f8856701 --- /dev/null +++ b/api/v1/cmd/msh/msh.go @@ -0,0 +1,223 @@ +// msh is a minimal mesos v1 scheduler; it executes a shell command on a mesos agent. +package main + +// Usage: msh {...command line args...} +// +// For example: +// msh -master 10.2.0.5:5050 -- ls -laF /tmp +// +// TODO: -gpu=1 to enable GPU_RESOURCES caps and request 1 gpu +// + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/mesos/mesos-go/api/v1/lib" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" + "github.com/mesos/mesos-go/api/v1/lib/extras/offers" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller" + "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/httpcli" + "github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +const ( + RFC3339a = "20060102T150405Z0700" +) + +var ( + FrameworkName = "msh" + TaskName = "msh" + MesosMaster = "127.0.0.1:5050" + User = "root" + Role = mesos.RoleName("*") + CPUs = float64(0.010) + Memory = float64(64) + + frameworkIDStore store.Singleton + declineAndSuppress bool + refuseSeconds = calls.RefuseSeconds(5 * time.Second) + wantsResources mesos.Resources + taskPrototype mesos.TaskInfo +) + +func init() { + flag.StringVar(&FrameworkName, "framework_name", FrameworkName, "Name of the framework") + flag.StringVar(&TaskName, "task_name", TaskName, "Name of the msh task") + flag.StringVar(&MesosMaster, "master", MesosMaster, "IP:port of the mesos master") + flag.StringVar(&User, "user", User, "OS user that owns the launched task") + flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command") + flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command") + + frameworkIDStore = store.DecorateSingleton( + store.NewInMemorySingleton(), + store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error { + log.Println("FrameworkID", v) + return nil + })) +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) < 1 { // msh by itself prints usage + flag.Usage() + os.Exit(1) + } + + wantsResources = mesos.Resources{ + mesos.CPUs(CPUs).Resource, + mesos.Memory(Memory).Resource, + } + taskPrototype = mesos.TaskInfo{ + Name: TaskName, + Command: &mesos.CommandInfo{ + Value: proto.String(args[0]), + Shell: proto.Bool(false), + }, + } + if len(args) > 1 { + taskPrototype.Command.Arguments = args[1:] + } + if err := run(); err != nil { + if exitErr, ok := err.(ExitError); ok { + if code := int(exitErr); code != 0 { + log.Println(exitErr) + os.Exit(code) + } + // else, code=0 indicates success, exit normally + } else { + panic(fmt.Sprintf("%#v", err)) + } + } +} + +func run() error { + var ( + done = latch.New() + caller = calls.Decorators{ + calls.SubscribedCaller(store.GetIgnoreErrors(frameworkIDStore)), + }.Apply(buildClient()) + ) + + return controller.Run( + &mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)}, + caller, + controller.WithDone(done.Closed), + controller.WithEventHandler(buildEventHandler(caller)), + controller.WithFrameworkID(store.GetIgnoreErrors(frameworkIDStore)), + controller.WithSubscriptionTerminated(func(err error) { + defer done.Close() + if err == io.EOF { + log.Println("disconnected") + } + }), + ) +} + +func buildClient() calls.Caller { + return httpsched.NewCaller(httpcli.New( + httpcli.Endpoint(fmt.Sprintf("http://%s/api/v1/scheduler", MesosMaster)), + )) +} + +func buildEventHandler(caller calls.Caller) events.Handler { + logger := controller.LogEvents() + return controller.LiftErrors().Handle(events.HandlerSet{ + scheduler.Event_FAILURE: logger, + scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(frameworkIDStore, 0)}, + scheduler.Event_OFFERS: maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)), + scheduler.Event_UPDATE: controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate), + }) +} + +func maybeDeclineOffers(caller calls.Caller) eventrules.Rule { + return func(e *scheduler.Event, err error, chain eventrules.Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) + } + if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress { + return chain(e, err) + } + off := offers.Slice(e.GetOffers().GetOffers()) + err = calls.CallNoData(caller, calls.Decline(off.IDs()...).With(refuseSeconds)) + if err == nil { + // we shouldn't have received offers, maybe the prior suppress call failed? + err = calls.CallNoData(caller, calls.Suppress()) + } + return nil, err // drop + } +} + +func resourceOffers(caller calls.Caller) events.HandlerFunc { + return func(e *scheduler.Event) (err error) { + var ( + off = e.GetOffers().GetOffers() + index = offers.NewIndex(off, nil) + match = index.Find(offers.ContainsResources(wantsResources)) + ) + if match != nil { + task := taskPrototype + task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)} + task.AgentID = match.AgentID + task.Resources = mesos.Resources(match.Resources).Find(wantsResources.Flatten(Role.Assign())) + + err = calls.CallNoData(caller, calls.Accept( + calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID), + )) + if err != nil { + return + } + + declineAndSuppress = true + } else { + log.Println("rejected insufficient offers") + } + // decline all but the possible match + delete(index, match.GetID()) + err = calls.CallNoData(caller, calls.Decline(index.IDs()...).With(refuseSeconds)) + if err != nil { + return + } + if declineAndSuppress { + err = calls.CallNoData(caller, calls.Suppress()) + } + return + } +} + +func statusUpdate(e *scheduler.Event) error { + s := e.GetUpdate().GetStatus() + switch st := s.GetState(); st { + case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING: + log.Printf("status update from agent %q: %v", s.GetAgentID().GetValue(), st) + if st != mesos.TASK_FINISHED { + return nil + } + case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR: + log.Println("Exiting because task " + s.GetTaskID().Value + + " is in an unexpected state " + st.String() + + " with reason " + s.GetReason().String() + + " from source " + s.GetSource().String() + + " with message '" + s.GetMessage() + "'") + return ExitError(3) + default: + log.Println("unexpected task state, aborting", st) + return ExitError(4) + } + return ExitError(0) // kind of ugly, but better than os.Exit(0) +} + +type ExitError int + +func (e ExitError) Error() string { return fmt.Sprintf("exit code %d", int(e)) } diff --git a/api/v1/lib/builders.go b/api/v1/lib/builders.go index 01642b31..2d5ba6ab 100644 --- a/api/v1/lib/builders.go +++ b/api/v1/lib/builders.go @@ -2,49 +2,65 @@ package mesos type ( // ResourceBuilder simplifies construction of Resource objects - ResourceBuilder struct{ *Resource } + ResourceBuilder struct{ Resource } // RangeBuilder simplifies construction of Range objects RangeBuilder struct{ Ranges } ) -func BuildRanges() RangeBuilder { - return RangeBuilder{Ranges: Ranges(nil)} +func CPUs(value float64) *ResourceBuilder { + return BuildResource().Name("cpus").Scalar(value) +} + +func Memory(value float64) *ResourceBuilder { + return BuildResource().Name("mem").Scalar(value) +} + +func Disk(value float64) *ResourceBuilder { + return BuildResource().Name("disk").Scalar(value) +} + +func GPUs(value uint) *ResourceBuilder { + return BuildResource().Name("gpus").Scalar(float64(value)) +} + +func BuildRanges() *RangeBuilder { + return &RangeBuilder{Ranges: Ranges(nil)} } // Span is a functional option for Ranges, defines the begin and end points of a // continuous span within a range -func (rb RangeBuilder) Span(bp, ep uint64) RangeBuilder { +func (rb *RangeBuilder) Span(bp, ep uint64) *RangeBuilder { rb.Ranges = append(rb.Ranges, Value_Range{Begin: bp, End: ep}) return rb } -func BuildResource() ResourceBuilder { - return ResourceBuilder{&Resource{}} +func BuildResource() *ResourceBuilder { + return &ResourceBuilder{} } -func (rb ResourceBuilder) Name(name string) ResourceBuilder { +func (rb *ResourceBuilder) Name(name string) *ResourceBuilder { rb.Resource.Name = name return rb } -func (rb ResourceBuilder) Role(role string) ResourceBuilder { +func (rb *ResourceBuilder) Role(role string) *ResourceBuilder { rb.Resource.Role = &role return rb } -func (rb ResourceBuilder) Scalar(x float64) ResourceBuilder { +func (rb *ResourceBuilder) Scalar(x float64) *ResourceBuilder { rb.Resource.Type = SCALAR.Enum() rb.Resource.Scalar = &Value_Scalar{Value: x} return rb } -func (rb ResourceBuilder) Set(x ...string) ResourceBuilder { +func (rb *ResourceBuilder) Set(x ...string) *ResourceBuilder { rb.Resource.Type = SET.Enum() rb.Resource.Set = &Value_Set{Item: x} return rb } -func (rb ResourceBuilder) Ranges(rs Ranges) ResourceBuilder { +func (rb *ResourceBuilder) Ranges(rs Ranges) *ResourceBuilder { rb.Resource.Type = RANGES.Enum() rb.Resource.Ranges = rb.Resource.Ranges.Add(&Value_Ranges{Range: rs}) return rb } -func (rb ResourceBuilder) Disk(persistenceID, containerPath string) ResourceBuilder { +func (rb *ResourceBuilder) Disk(persistenceID, containerPath string) *ResourceBuilder { rb.Resource.Disk = &Resource_DiskInfo{} if containerPath != "" { rb.Resource.Disk.Volume = &Volume{ContainerPath: containerPath} @@ -54,7 +70,7 @@ func (rb ResourceBuilder) Disk(persistenceID, containerPath string) ResourceBuil } return rb } -func (rb ResourceBuilder) Revocable() ResourceBuilder { +func (rb *ResourceBuilder) Revocable() *ResourceBuilder { rb.Resource.Revocable = &Resource_RevocableInfo{} return rb } diff --git a/api/v1/lib/encoding/codec.go b/api/v1/lib/encoding/codec.go index 1d4255b3..557b7aef 100644 --- a/api/v1/lib/encoding/codec.go +++ b/api/v1/lib/encoding/codec.go @@ -50,6 +50,9 @@ type Codec struct { // String implements the fmt.Stringer interface. func (c *Codec) String() string { return c.Name } +func (c *Codec) RequestContentType() string { return c.MediaTypes[0] } +func (c *Codec) ResponseContentType() string { return c.MediaTypes[1] } + type ( // Marshaler composes the supported marshaling formats. Marshaler interface { diff --git a/api/v1/lib/extras/latch/latch.go b/api/v1/lib/extras/latch/latch.go new file mode 100644 index 00000000..35f4d80b --- /dev/null +++ b/api/v1/lib/extras/latch/latch.go @@ -0,0 +1,48 @@ +package latch + +import "sync/atomic" + +// Interface funcs are safe to invoke concurrently. +type Interface interface { + // Done returns a chan that blocks until Close is called. It never returns data. + Done() <-chan struct{} + // Close closes the latch; all future calls to Closed return true. Safe to invoke multiple times. + Close() + // Closed returns false while the latch is "open" and true after it has been closed via Close. + Closed() bool +} + +type L struct { + value int32 + line chan struct{} +} + +// New returns a new "open" latch such that Closed returns false until Close is invoked. +func New() Interface { + return new(L).Reset() +} + +func (l *L) Done() <-chan struct{} { return l.line } + +// Close may panic for an uninitialized L +func (l *L) Close() { + if atomic.AddInt32(&l.value, 1) == 1 { + close(l.line) + } + <-l.line // concurrent calls to Close block until the latch is actually closed +} + +func (l *L) Closed() (result bool) { + select { + case <-l.line: + result = true + default: + } + return +} + +// Reset clears the state of the latch, not safe to execute concurrently with other L methods. +func (l *L) Reset() *L { + l.line, l.value = make(chan struct{}), 0 + return l +} diff --git a/api/v1/lib/extras/latch/latch_test.go b/api/v1/lib/extras/latch/latch_test.go new file mode 100644 index 00000000..caab35a5 --- /dev/null +++ b/api/v1/lib/extras/latch/latch_test.go @@ -0,0 +1,33 @@ +package latch_test + +import ( + "testing" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/latch" +) + +func TestInterface(t *testing.T) { + l := New() + if l == nil { + t.Fatalf("expected a valid latch, not nil") + } + if l.Closed() { + t.Fatalf("expected new latch to be non-closed") + } + select { + case <-l.Done(): + t.Fatalf("Done chan unexpectedly closed for a new latch") + default: + } + for i := 0; i < 2; i++ { + l.Close() // multiple calls to close should not panic + } + if !l.Closed() { + t.Fatalf("expected closed latch") + } + select { + case <-l.Done(): + default: + t.Fatalf("Done chan unexpectedly non-closed for a closed latch") + } +} diff --git a/api/v1/lib/extras/offers/filters.go b/api/v1/lib/extras/offers/filters.go new file mode 100644 index 00000000..9a872665 --- /dev/null +++ b/api/v1/lib/extras/offers/filters.go @@ -0,0 +1,69 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Filter accepts or rejects a mesos Offer + Filter interface { + Accept(*Offer) bool + } + + // FilterFunc returns true if the given Offer passes the filter + FilterFunc func(*Offer) bool +) + +// Accept implements Filter for FilterFunc +func (f FilterFunc) Accept(o *Offer) bool { return f(o) } + +func nilFilter(_ *Offer) bool { return true } + +func not(f Filter) Filter { + return FilterFunc(func(offer *Offer) bool { return !f.Accept(offer) }) +} + +// ByHostname returns a Filter that accepts offers with a matching Hostname +func ByHostname(hostname string) Filter { + if hostname == "" { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return o.Hostname == hostname + }) +} + +// ByAttributes returns a Filter that accepts offers with an attribute set accepted by +// the provided Attribute filter func. +func ByAttributes(f func(attr []Attribute) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Attributes) + }) +} + +func ByExecutors(f func(exec []ExecutorID) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.ExecutorIDs) + }) +} + +func ByUnavailability(f func(u *Unavailability) bool) Filter { + if f == nil { + return FilterFunc(nilFilter) + } + return FilterFunc(func(o *Offer) bool { + return f(o.Unavailability) + }) +} + +// ContainsResources returns a filter function that returns true if the Resources of an Offer +// contain the wanted Resources. +func ContainsResources(wanted Resources) Filter { + return FilterFunc(func(o *Offer) bool { + return Resources(o.Resources).Flatten().ContainsAll(wanted) + }) +} diff --git a/api/v1/lib/extras/offers/offers.go b/api/v1/lib/extras/offers/offers.go new file mode 100644 index 00000000..75ccdfef --- /dev/null +++ b/api/v1/lib/extras/offers/offers.go @@ -0,0 +1,245 @@ +package offers + +import . "github.com/mesos/mesos-go/api/v1/lib" + +type ( + // Slice is a convenience type wrapper for a slice of mesos Offer messages + Slice []Offer + + // Index is a convenience type wrapper for a dictionary of Offer messages + Index map[interface{}]*Offer + + // KeyFunc generates a key used for indexing offers + KeyFunc func(*Offer) interface{} +) + +// IDs extracts the ID field from a Slice of offers +func (offers Slice) IDs() []OfferID { + ids := make([]OfferID, len(offers)) + for i := range offers { + ids[i] = offers[i].ID + } + return ids +} + +// IDs extracts the ID field from a Index of offers +func (offers Index) IDs() []OfferID { + ids := make([]OfferID, 0, len(offers)) + for _, offer := range offers { + ids = append(ids, offer.GetID()) + } + return ids +} + +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. +func (offers Slice) Find(filter Filter) *Offer { + for i := range offers { + offer := &offers[i] + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Find returns the first Offer that passes the given filter function, or else nil if +// there are no passing offers. +func (offers Index) Find(filter Filter) *Offer { + for _, offer := range offers { + if filter.Accept(offer) { + return offer + } + } + return nil +} + +// Filter returns the subset of the Slice that matches the given filter. +func (offers Slice) Filter(filter Filter) (result Slice) { + if sz := len(result); sz > 0 { + result = make(Slice, 0, sz) + for i := range offers { + if filter.Accept(&offers[i]) { + result = append(result, offers[i]) + } + } + } + return +} + +// Filter returns the subset of the Index that matches the given filter. +func (offers Index) Filter(filter Filter) (result Index) { + if sz := len(result); sz > 0 { + result = make(Index, sz) + for id, offer := range offers { + if filter.Accept(offer) { + result[id] = offer + } + } + } + return +} + +// FilterNot returns the subset of the Slice that does not match the given filter. +func (offers Slice) FilterNot(filter Filter) Slice { return offers.Filter(not(filter)) } + +// FilterNot returns the subset of the Index that does not match the given filter. +func (offers Index) FilterNot(filter Filter) Index { return offers.Filter(not(filter)) } + +// DefaultKeyFunc indexes offers by their OfferID. +var DefaultKeyFunc = KeyFunc(func(o *Offer) interface{} { return o.GetID() }) + +// NewIndex returns a new Index constructed from the list of mesos offers. +// If the KeyFunc is nil then offers are indexed by DefaultKeyFunc. +// The values of the returned Index are pointers to (not copies of) the offers of the slice receiver. +func NewIndex(slice []Offer, kf KeyFunc) Index { + if slice == nil { + return nil + } + if kf == nil { + kf = DefaultKeyFunc + } + index := make(Index, len(slice)) + for i := range slice { + offer := &slice[i] + index[kf(offer)] = offer + } + return index +} + +// ToSlice returns a Slice from the offers in the Index. +// The returned slice will contain shallow copies of the offers from the Index. +func (s Index) ToSlice() (slice Slice) { + if sz := len(s); sz > 0 { + slice = make(Slice, 0, sz) + for _, offer := range s { + slice = append(slice, *offer) + } + } + return +} + +type ( + Reducer interface { + Reduce(_, _ *Offer) *Offer + } + + ReduceFunc func(_, _ *Offer) *Offer +) + +func (f ReduceFunc) Reduce(a, b *Offer) *Offer { return f(a, b) } + +var _ = Reducer(ReduceFunc(func(_, _ *Offer) *Offer { return nil })) // sanity check + +func (slice Slice) Reduce(def Offer, r Reducer) (result Offer) { + result = def + if r != nil { + acc := &result + for i := range slice { + acc = r.Reduce(&result, &slice[i]) + } + if acc == nil { + result = Offer{} + } else { + result = *acc + } + } + return +} + +func (index Index) Reduce(def *Offer, r Reducer) (result *Offer) { + result = def + if r != nil { + for i := range index { + result = r.Reduce(result, index[i]) + } + } + return +} + +func (slice Slice) GroupBy(kf KeyFunc) map[interface{}]Slice { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(slice) == 0 { + return nil + } + result := make(map[interface{}]Slice) + for i := range slice { + groupKey := kf(&slice[i]) + result[groupKey] = append(result[groupKey], slice[i]) + } + return result +} + +func (index Index) GroupBy(kf KeyFunc) map[interface{}]Index { + if kf == nil { + panic("keyFunc must not be nil") + } + if len(index) == 0 { + return nil + } + result := make(map[interface{}]Index) + for i, offer := range index { + groupKey := kf(offer) + group, ok := result[groupKey] + if !ok { + group = make(Index) + result[groupKey] = group + } + group[i] = offer + } + return result +} + +func (index Index) Partition(f Filter) (accepted, rejected Index) { + if f == nil { + return index, nil + } + if len(index) > 0 { + accepted, rejected = make(Index), make(Index) + for id, offer := range index { + if f.Accept(offer) { + accepted[id] = offer + } else { + rejected[id] = offer + } + } + } + return +} + +func (s Slice) Partition(f Filter) (accepted, rejected []int) { + if f == nil { + accepted = make([]int, len(s)) + for i := range s { + accepted[i] = i + } + return + } + if sz := len(s); sz > 0 { + accepted, rejected = make([]int, 0, sz/2), make([]int, 0, sz/2) + for i := range s { + offer := &s[i] + if f.Accept(offer) { + accepted = append(accepted, i) + } else { + rejected = append(rejected, i) + } + } + } + return +} + +func (index Index) Reindex(kf KeyFunc) Index { + sz := len(index) + if kf == nil || sz == 0 { + return index + } + result := make(Index, sz) + for _, offer := range index { + key := kf(offer) + result[key] = offer + } + return result +} diff --git a/api/v1/lib/extras/rules/rules.go b/api/v1/lib/extras/rules/rules.go new file mode 100644 index 00000000..00587709 --- /dev/null +++ b/api/v1/lib/extras/rules/rules.go @@ -0,0 +1,412 @@ +// +build ignore + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + "text/template" +) + +type ( + config struct { + Package string + Imports []string + EventType string + } +) + +func (c *config) String() string { + if c == nil { + return "" + } + return fmt.Sprintf("%#v", ([]string)(c.Imports)) +} + +func (c *config) Set(s string) error { + c.Imports = append(c.Imports, s) + return nil +} + +func main() { + var ( + c = config{ + Package: os.Getenv("GOPACKAGE"), + EventType: "Event", + } + defaultOutput = "foo.go" + ) + if c.Package != "" { + defaultOutput = c.Package + "_generated.go" + } + + output := defaultOutput + + flag.StringVar(&c.Package, "package", c.Package, "destination package") + flag.StringVar(&c.EventType, "event_type", c.EventType, "golang type of the event to be processed") + flag.StringVar(&output, "output", output, "path of the to-be-generated file") + flag.Var(&c, "import", "packages to import") + flag.Parse() + + if c.Package == "" { + c.Package = "foo" + } + if c.EventType == "" { + c.EventType = "Event" + } + if output == "" { + output = defaultOutput + } + + testOutput := output + "_test" + if strings.HasSuffix(output, ".go") { + testOutput = output[:len(output)-3] + "_test.go" + } + + // main template + f, err := os.Create(output) + if err != nil { + log.Fatal(err) + } + defer f.Close() + rulesTemplate.Execute(f, c) + + // unit test template + f, err = os.Create(testOutput) + if err != nil { + log.Fatal(err) + } + testTemplate.Execute(f, c) +} + +var rulesTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "fmt" + "sync" +{{range .Imports}} + {{ printf "%q" . }} +{{ end -}} +) + +type ( + // Rule executes a filter, rule, or decorator function; if the returned event is nil then + // no additional Rule func should be processed for the event. + // Rule implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + Rule func({{.EventType}}, error, Chain) ({{.EventType}}, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func({{.EventType}}, error) ({{.EventType}}, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +// chainIdentity is a Chain that returns the arguments as its results. +var chainIdentity = func(e {{.EventType}}, err error) ({{.EventType}}, error) { + return e, err +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if len(rs) == 0 { + return ch(e, err) // noop + } + // we know there's at least 1 rule in the initial list; start with it and let the chain + // handle the iteration. + return ch(rs[0](e, err, NewChain(rs))) +} + +// Rule adapts Rules to the Rule interface, for convenient call chaining. +func (rs Rules) Rule() Rule { return rs.Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + return b + } + if b == nil { + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + var result ErrorList + for _, err := range es { + if err != nil { + if multi, ok := err.(ErrorList); ok { + // flatten nested error lists + if len(multi) > 0 { + result = append(result, multi...) + } + } else { + result = append(result, err) + } + } + } + return result.Err() +} + +// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + var once sync.Once + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + once.Do(func() { + e, err = r(e, err, ch) + }) + return e, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. +// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + select { + case <-p: + // do something + return r(e, err, ch) + default: + // drop + return ch(nil, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this is a noop. +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if forward() { + return r(e, err, ch) + } + // else, drop + return ch(nil, err) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving +// rule. +func (r Rule) DropOnError() Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if err != nil || e == nil { + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing +// for any of the following cases: +// - there are no more rules to process +// - the event has been zero'ed out (nil) +// Any nil rules in the list are processed as skipped (noop's). +func NewChain(rs Rules) Chain { + sz := len(rs) + if sz == 0 { + return chainIdentity + } + var ( + i = 0 + chain Chain + ) + chain = Chain(func(x {{.EventType}}, y error) ({{.EventType}}, error) { + i++ + if i >= sz || x == nil { + // we're at the end, or DROP was issued (x==nil) + return x, y + } else if rs[i] != nil { + return rs[i](x, y, chain) + } else { + return chain(x, y) + } + + }) + return chain +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + if e != nil && err == nil { + // bypass remainder of chain + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} +`)) + +var testTemplate = template.Must(template.New("").Parse(`package {{.Package}} + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "errors" + "reflect" + "testing" +{{range .Imports}} + {{ printf "%q" . }} +{{ end -}} +) + +func counter(i *int) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + *i++ + return ch(e, err) + } +} + +func returnError(re error) Rule { + return func(e {{.EventType}}, err error, ch Chain) ({{.EventType}}, error) { + return ch(e, Error2(err, re)) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + }{ + {nil, nil, nil}, + {a, nil, a}, + {nil, b, b}, + {a, b, ErrorList{a, b}}, + } { + result := Error2(tc.a, tc.b) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + if !IsErrorList(result) && result == tc.wants { + continue + } + if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { + continue + } + } + t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } +} +`)) diff --git a/api/v1/lib/extras/scheduler/controller/controller.go b/api/v1/lib/extras/scheduler/controller/controller.go index 540c1662..03a732e5 100644 --- a/api/v1/lib/extras/scheduler/controller/controller.go +++ b/api/v1/lib/extras/scheduler/controller/controller.go @@ -9,79 +9,111 @@ import ( ) type ( - Context interface { - // Done returns true when the controller should exit - Done() bool + // Option modifies a Config, returns an Option that acts as an "undo" + Option func(*Config) Option - // FrameworkID returns the current Mesos-assigned framework ID. Frameworks are expected to - // track this ID (that comes from Mesos, in a SUBSCRIBED event). - FrameworkID() string - - // Error is an error handler that is invoked at the end of every subscription cycle; the given - // error may be nil (if no errors occurred). - Error(error) + // Config is a controller configuration. Public fields are REQUIRED. Optional properties are + // configured by applying Option funcs. + Config struct { + doneFunc func() bool + frameworkIDFunc func() string + handler events.Handler + registrationTokens <-chan struct{} + subscriptionTerminated func(error) } +) - ContextAdapter struct { - // FrameworkIDFunc is optional; nil tells the controller to always register as a new framework - // for each subscription attempt. - FrameworkIDFunc func() string - - // Done is optional; nil equates to a func that always returns false - DoneFunc func() bool - - // ErrorFunc is optional; if nil then errors are swallowed - ErrorFunc func(error) +// WithEventHandler sets the consumer of scheduler events. The controller's internal event processing +// loop is aborted if a Handler returns a non-nil error, after which the controller may attempt +// to re-register (subscribe) with Mesos. +func WithEventHandler(handler events.Handler, ds ...events.Decorator) Option { + return func(c *Config) Option { + old := c.handler + c.handler = events.Decorators(ds).Apply(handler) + return WithEventHandler(old) } +} - Config struct { - Context Context // Context is required - Framework *mesos.FrameworkInfo // FrameworkInfo is required - Caller calls.Caller // Caller is required - - // Handler (optional) processes scheduler events. The controller's internal event processing - // loop is aborted if a Handler returns a non-nil error, after which the controller may attempt - // to re-register (subscribe) with Mesos. - Handler events.Handler - - // RegistrationTokens (optional) limits the rate at which a framework (re)registers with Mesos. - // The returned chan should either be non-blocking (nil/closed), or should yield a struct{} in - // order to allow the framework registration process to continue. May be nil. - RegistrationTokens <-chan struct{} +// WithFrameworkID sets a fetcher for the current Mesos-assigned framework ID. Frameworks are expected to +// track this ID (that comes from Mesos, in a SUBSCRIBED event). +// frameworkIDFunc is optional; nil tells the controller to always register as a new framework +// for each subscription attempt. +func WithFrameworkID(frameworkIDFunc func() string) Option { + return func(c *Config) Option { + old := c.frameworkIDFunc + c.frameworkIDFunc = frameworkIDFunc + return WithFrameworkID(old) } +} - Controller interface { - // Run executes the controller using the given Config - Run(Config) error +// WithDone sets a fetcher func that returns true when the controller should exit. +// doneFunc is optional; nil equates to a func that always returns false. +func WithDone(doneFunc func() bool) Option { + return func(c *Config) Option { + old := c.doneFunc + c.doneFunc = doneFunc + return WithDone(old) } +} - // ControllerFunc is a functional adaptation of a Controller - ControllerFunc func(Config) error - - controllerImpl int -) +// WithSubscriptionTerminated sets a handler that is invoked at the end of every subscription cycle; the +// given error may be nil if no error occurred. subscriptionTerminated is optional; if nil then errors are +// swallowed. +func WithSubscriptionTerminated(handler func(error)) Option { + return func(c *Config) Option { + old := c.subscriptionTerminated + c.subscriptionTerminated = handler + return WithSubscriptionTerminated(old) + } +} -// Run implements Controller for ControllerFunc -func (cf ControllerFunc) Run(config Config) error { return cf(config) } +// WithRegistrationTokens limits the rate at which a framework (re)registers with Mesos. +// The chan should either be non-blocking, or should yield a struct{} in order to allow the +// framework registration process to continue. May be nil. +func WithRegistrationTokens(registrationTokens <-chan struct{}) Option { + return func(c *Config) Option { + old := c.registrationTokens + c.registrationTokens = registrationTokens + return WithRegistrationTokens(old) + } +} -func New() Controller { - return new(controllerImpl) +func (c *Config) tryFrameworkID() (result string) { + if c.frameworkIDFunc != nil { + result = c.frameworkIDFunc() + } + return } +func (c *Config) tryDone() (result bool) { return c.doneFunc != nil && c.doneFunc() } + // Run executes a control loop that registers a framework with Mesos and processes the scheduler events -// that flow through the subscription. Upon disconnection, if the given Context reports !Done() then the -// controller will attempt to re-register the framework and continue processing events. -func (_ *controllerImpl) Run(config Config) (lastErr error) { - subscribe := calls.Subscribe(config.Framework) - for !config.Context.Done() { - frameworkID := config.Context.FrameworkID() - if config.Framework.GetFailoverTimeout() > 0 && frameworkID != "" { +// that flow through the subscription. Upon disconnection, if the current configuration reports "not done" +// then the controller will attempt to re-register the framework and continue processing events. +func Run(framework *mesos.FrameworkInfo, caller calls.Caller, options ...Option) (lastErr error) { + var config Config + for _, opt := range options { + if opt != nil { + opt(&config) + } + } + if config.handler == nil { + config.handler = DefaultHandler + } + subscribe := calls.Subscribe(framework) + for !config.tryDone() { + frameworkID := config.tryFrameworkID() + if framework.GetFailoverTimeout() > 0 && frameworkID != "" { subscribe.With(calls.SubscribeTo(frameworkID)) } - <-config.RegistrationTokens - resp, err := config.Caller.Call(subscribe) + if config.registrationTokens != nil { + <-config.registrationTokens + } + resp, err := caller.Call(subscribe) lastErr = processSubscription(config, resp, err) - config.Context.Error(lastErr) + if config.subscriptionTerminated != nil { + config.subscriptionTerminated(lastErr) + } } return } @@ -99,50 +131,14 @@ func processSubscription(config Config, resp mesos.Response, err error) error { // eventLoop returns the framework ID received by mesos (if any); callers should check for a // framework ID regardless of whether error != nil. func eventLoop(config Config, eventDecoder encoding.Decoder) (err error) { - h := config.Handler - if h == nil { - h = events.HandlerFunc(DefaultHandler) - } - for err == nil && !config.Context.Done() { + for err == nil && !config.tryDone() { var e scheduler.Event if err = eventDecoder.Decode(&e); err == nil { - err = h.HandleEvent(&e) + err = config.handler.HandleEvent(&e) } } return err } -var _ = Context(&ContextAdapter{}) // ContextAdapter implements Context - -func (ca *ContextAdapter) Done() bool { - return ca.DoneFunc != nil && ca.DoneFunc() -} -func (ca *ContextAdapter) FrameworkID() (id string) { - if ca.FrameworkIDFunc != nil { - id = ca.FrameworkIDFunc() - } - return -} -func (ca *ContextAdapter) Error(err error) { - if ca.ErrorFunc != nil { - ca.ErrorFunc(err) - } -} - -// ErrEvent errors are generated by the DefaultHandler upon receiving an ERROR event from Mesos. -type ErrEvent string - -func (e ErrEvent) Error() string { - return string(e) -} - -// DefaultHandler provides the minimum implementation required for correct controller behavior. -func DefaultHandler(e *scheduler.Event) (err error) { - if e.GetType() == scheduler.Event_ERROR { - // it's recommended that we abort and re-try subscribing; returning an - // error here will cause the event loop to terminate and the connection - // will be reset. - err = ErrEvent(e.GetError().GetMessage()) - } - return -} +// DefaultHandler defaults to events.NoopHandler +var DefaultHandler = events.NoopHandler() diff --git a/api/v1/lib/extras/scheduler/controller/rules.go b/api/v1/lib/extras/scheduler/controller/rules.go new file mode 100644 index 00000000..a4ff8f86 --- /dev/null +++ b/api/v1/lib/extras/scheduler/controller/rules.go @@ -0,0 +1,125 @@ +package controller + +import ( + "fmt" + "log" + "time" + + . "github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules" + "github.com/mesos/mesos-go/api/v1/lib/extras/store" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +// ErrEvent errors are generated by LiftErrors upon receiving an ERROR event from Mesos. +type ErrEvent string + +func (e ErrEvent) Error() string { + return string(e) +} + +// LiftErrors extract the error message from a scheduler error event and returns it as an ErrEvent +// so that downstream rules/handlers may continue processing. +func LiftErrors() Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) + } + if e.GetType() == scheduler.Event_ERROR { + // it's recommended that we abort and re-try subscribing; returning an + // error here will cause the event loop to terminate and the connection + // will be reset. + return chain(e, ErrEvent(e.GetError().GetMessage())) + } + return chain(e, nil) + } +} + +// StateError is returned when the system encounters an unresolvable state transition error and +// should likely exit. +type StateError string + +func (err StateError) Error() string { return string(err) } + +func TrackSubscription(frameworkIDStore store.Singleton, failoverTimeout time.Duration) Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + if err != nil { + return chain(e, err) + } + if e.GetType() == scheduler.Event_SUBSCRIBED { + var ( + storedFrameworkID, err = frameworkIDStore.Get() + frameworkID = e.GetSubscribed().GetFrameworkID().GetValue() + ) + if err != nil && err != store.ErrNotFound { + return chain(e, err) + } + // order of `if` statements are important: tread carefully w/ respect to future changes + if frameworkID == "" { + // sanity check, should **never** happen + return chain(e, StateError("mesos sent an empty frameworkID?!")) + } + if storedFrameworkID != "" && storedFrameworkID != frameworkID && failoverTimeout > 0 { + return chain(e, StateError(fmt.Sprintf( + "frameworkID changed unexpectedly; failover exceeded timeout? (%s).", failoverTimeout))) + } + if storedFrameworkID != frameworkID { + frameworkIDStore.Set(frameworkID) + } + } + return chain(e, nil) + } +} + +// AckStatusUpdates sends an acknowledgement of a task status update back to mesos and drops the event if +// sending the ack fails. If successful, the specified err param (if any) is forwarded. Acknowledgements +// are only attempted for task status updates tagged with a UUID. +func AckStatusUpdates(caller calls.Caller) Rule { + return AckStatusUpdatesF(func() calls.Caller { return caller }) +} + +// AckStatusUpdatesF is a functional adapter for AckStatusUpdates, useful for cases where the caller may +// change over time. +func AckStatusUpdatesF(callerLookup func() calls.Caller) Rule { + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + // aggressively attempt to ack updates: even if there's pre-existing error state attempt + // to acknowledge all status updates. + origErr := err + if e.GetType() == scheduler.Event_UPDATE { + var ( + s = e.GetUpdate().GetStatus() + uuid = s.GetUUID() + ) + // only ACK non-empty UUID's, as per mesos scheduler spec + if len(uuid) > 0 { + ack := calls.Acknowledge( + s.GetAgentID().GetValue(), + s.TaskID.Value, + uuid, + ) + err = calls.CallNoData(callerLookup(), ack) + if err != nil { + err = &events.AckError{Ack: ack, Cause: err} + return nil, Error2(origErr, err) // drop + } + } + } + return chain(e, origErr) + } +} + +var ( + // EventLabel is, by default, logged as the first argument by EventLogger + EventLabel = "event" + // EventLogger is the logger used by the LogEvents rule generator + EventLogger = func(e *scheduler.Event) { log.Println(EventLabel, e) } +) + +// LogEvents returns a rule that logs scheduler events to the EventLogger +func LogEvents() Rule { + return Rule(func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + EventLogger(e) + return chain(e, err) + }) +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go new file mode 100644 index 00000000..bb4dd636 --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated.go @@ -0,0 +1,252 @@ +package eventrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "fmt" + "sync" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +type ( + // Rule executes a filter, rule, or decorator function; if the returned event is nil then + // no additional Rule func should be processed for the event. + // Rule implementations should not modify the given event parameter (to avoid side effects). + // If changes to the event object are needed, the suggested approach is to make a copy, + // modify the copy, and pass the copy to the chain. + Rule func(*scheduler.Event, error, Chain) (*scheduler.Event, error) + + // Chain is invoked by a Rule to continue processing an event. If the chain is not invoked, + // no additional rules are processed. + Chain func(*scheduler.Event, error) (*scheduler.Event, error) + + // Rules is a list of rules to be processed, in order. + Rules []Rule + + // ErrorList accumulates errors that occur while processing a Chain of Rules. Accumulated + // errors should be appended to the end of the list. An error list should never be empty. + // Callers should use the package Error() func to properly accumulate (and flatten) errors. + ErrorList []error +) + +// chainIdentity is a Chain that returns the arguments as its results. +var chainIdentity = func(e *scheduler.Event, err error) (*scheduler.Event, error) { + return e, err +} + +// Eval is a Rule func that processes the set of all Rules. If there are no rules in the +// set then control is simply passed to the Chain. +func (rs Rules) Eval(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if len(rs) == 0 { + return ch(e, err) // noop + } + // we know there's at least 1 rule in the initial list; start with it and let the chain + // handle the iteration. + return ch(rs[0](e, err, NewChain(rs))) +} + +// Rule adapts Rules to the Rule interface, for convenient call chaining. +func (rs Rules) Rule() Rule { return rs.Eval } + +// Error implements error; returns the message of the first error in the list. +func (es ErrorList) Error() string { + switch len(es) { + case 0: + return "no errors" + case 1: + return es[0].Error() + default: + return fmt.Sprintf("%s (and %d more errors)", es[0], len(es)-1) + } +} + +// Error2 aggregates the given error params, returning nil if both are nil. +// Use Error2 to avoid the overhead of creating a slice when aggregating only 2 errors. +func Error2(a, b error) error { + if a == nil { + if b == nil { + return nil + } + return b + } + if b == nil { + return a + } + return Error(a, b) +} + +// Err reduces an empty or singleton error list +func (es ErrorList) Err() error { + if len(es) == 0 { + return nil + } + if len(es) == 1 { + return es[0] + } + return es +} + +// IsErrorList returns true if err is a non-nil error list +func IsErrorList(err error) bool { + if err != nil { + _, ok := err.(ErrorList) + return ok + } + return false +} + +// Error aggregates, and then (shallowly) flattens, a list of errors accrued during rule processing. +// Returns nil if the given list of errors is empty or contains all nil errors. +func Error(es ...error) error { + var result ErrorList + for _, err := range es { + if err != nil { + if multi, ok := err.(ErrorList); ok { + // flatten nested error lists + if len(multi) > 0 { + result = append(result, multi...) + } + } else { + result = append(result, err) + } + } + } + return result.Err() +} + +// TODO(jdef): other ideas for Rule decorators: If(bool), When(func() bool), Unless(bool) + +// Once returns a Rule that executes the receiver only once. +func (r Rule) Once() Rule { + var once sync.Once + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + once.Do(func() { + e, err = r(e, err, ch) + }) + return e, err + } +} + +// Poll invokes the receiving Rule if the chan is readable (may be closed), otherwise it drops the event. +// A nil chan will drop all events. May be useful, for example, when rate-limiting logged events. +func (r Rule) Poll(p <-chan struct{}) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + select { + case <-p: + // do something + return r(e, err, ch) + default: + // drop + return ch(nil, err) + } + } +} + +// EveryN invokes the receiving rule beginning with the first event seen and then every n'th +// time after that. If nthTime is less then 2 then this is a noop. +func (r Rule) EveryN(nthTime int) Rule { + if nthTime < 2 { + return r + } + var ( + i = 1 // begin with the first event seen + m sync.Mutex + forward = func() bool { + m.Lock() + i-- + if i == 0 { + i = nthTime + m.Unlock() + return true + } + m.Unlock() + return false + } + ) + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if forward() { + return r(e, err, ch) + } + // else, drop + return ch(nil, err) + } +} + +// DropOnError returns a Rule that generates a nil event if the error state != nil +func DropOnError() Rule { + return Rule(nil).DropOnError() +} + +// DropOnError decorates a rule by pre-checking the error state: if the error state != nil then +// the receiver is not invoked and (nil, err) is returned; otherwise control passes to the receiving +// rule. +func (r Rule) DropOnError() Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if err != nil || e == nil { + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +// NewChain returns a Chain that iterates through the given Rules, in order, stopping rule processing +// for any of the following cases: +// - there are no more rules to process +// - the event has been zero'ed out (nil) +// Any nil rules in the list are processed as skipped (noop's). +func NewChain(rs Rules) Chain { + sz := len(rs) + if sz == 0 { + return chainIdentity + } + var ( + i = 0 + chain Chain + ) + chain = Chain(func(x *scheduler.Event, y error) (*scheduler.Event, error) { + i++ + if i >= sz || x == nil { + // we're at the end, or DROP was issued (x==nil) + return x, y + } else if rs[i] != nil { + return rs[i](x, y, chain) + } else { + return chain(x, y) + } + + }) + return chain +} + +// AndThen returns a list of rules, beginning with the receiver, followed by DropOnError, and then +// all of the rules specified by the next parameter. The net effect is: execute the receiver rule +// and only if there is no error state, continue processing the next rules, in order. +func (r Rule) AndThen(next ...Rule) Rule { + return append(Rules{r, DropOnError()}, next...).Eval +} + +func DropOnSuccess() Rule { + return Rule(nil).DropOnSuccess() +} + +func (r Rule) DropOnSuccess() Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + if e != nil && err == nil { + // bypass remainder of chain + return e, err + } + if r != nil { + return r(e, err, ch) + } + return ch(e, err) + } +} + +func (r Rule) OnFailure(next ...Rule) Rule { + return append(Rules{r, DropOnSuccess()}, next...).Eval +} diff --git a/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go new file mode 100644 index 00000000..f5a6f1de --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/eventrules_generated_test.go @@ -0,0 +1,71 @@ +package eventrules + +// go generate +// GENERATED CODE FOLLOWS; DO NOT EDIT. + +import ( + "errors" + "reflect" + "testing" + + "github.com/mesos/mesos-go/api/v1/lib/scheduler" +) + +func counter(i *int) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + *i++ + return ch(e, err) + } +} + +func returnError(re error) Rule { + return func(e *scheduler.Event, err error, ch Chain) (*scheduler.Event, error) { + return ch(e, Error2(err, re)) + } +} + +func TestChainIdentity(t *testing.T) { + var i int + counterRule := counter(&i) + + e, err := Rules{counterRule}.Eval(nil, nil, chainIdentity) + if e != nil { + t.Error("expected nil event instead of", e) + } + if err != nil { + t.Error("expected nil error instead of", err) + } + if i != 1 { + t.Error("expected 1 rule execution instead of", i) + } +} + +func TestError2(t *testing.T) { + var ( + a = errors.New("a") + b = errors.New("b") + ) + for i, tc := range []struct { + a error + b error + wants error + }{ + {nil, nil, nil}, + {a, nil, a}, + {nil, b, b}, + {a, b, ErrorList{a, b}}, + } { + result := Error2(tc.a, tc.b) + // jump through hoops because we can't directly compare two errors with == if + // they're both ErrorList. + if IsErrorList(result) == IsErrorList(tc.wants) { // both are lists or neither + if !IsErrorList(result) && result == tc.wants { + continue + } + if IsErrorList(result) && reflect.DeepEqual(result, tc.wants) { + continue + } + } + t.Errorf("test case %d failed, expected %v instead of %v", i, tc.wants, result) + } +} diff --git a/api/v1/lib/extras/scheduler/eventrules/gen.go b/api/v1/lib/extras/scheduler/eventrules/gen.go new file mode 100644 index 00000000..44a4854c --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/gen.go @@ -0,0 +1,3 @@ +package eventrules + +//go:generate go run ../../rules/rules.go -import github.com/mesos/mesos-go/api/v1/lib/scheduler -event_type *scheduler.Event diff --git a/api/v1/lib/extras/scheduler/eventrules/handlers.go b/api/v1/lib/extras/scheduler/eventrules/handlers.go new file mode 100644 index 00000000..16a08b8d --- /dev/null +++ b/api/v1/lib/extras/scheduler/eventrules/handlers.go @@ -0,0 +1,61 @@ +package eventrules + +import ( + "github.com/mesos/mesos-go/api/v1/lib/scheduler" + "github.com/mesos/mesos-go/api/v1/lib/scheduler/events" +) + +// Handler generates a rule that executes the given handler. +func Handle(h events.Handler) Rule { + if h == nil { + return nil + } + return func(e *scheduler.Event, err error, chain Chain) (*scheduler.Event, error) { + newErr := h.HandleEvent(e) + return chain(e, Error2(err, newErr)) + } +} + +// HandleF is the functional equivalent of Handle +func HandleF(h events.HandlerFunc) Rule { + return Handle(events.Handler(h)) +} + +// Handler returns a rule that invokes the given Handler +func (r Rule) Handle(h events.Handler) Rule { + return Rules{r, Handle(h)}.Eval +} + +// HandleF is the functional equivalent of Handle +func (r Rule) HandleF(h events.HandlerFunc) Rule { + return r.Handle(events.Handler(h)) +} + +// HandleEvent implements events.Handler for Rule +func (r Rule) HandleEvent(e *scheduler.Event) (err error) { + if r == nil { + return nil + } + _, err = r(e, nil, chainIdentity) + return +} + +// HandleEvent implements events.Handler for Rules +func (rs Rules) HandleEvent(e *scheduler.Event) error { + return rs.Rule().HandleEvent(e) +} + +/* +// Apply returns the result of a singleton rule set (the receiver) applied to the given event handler. +func (r Rule) Apply(h events.Handler) events.HandlerFunc { + if r == nil { + return h.HandleEvent + } + return r.Handle(h).HandleEvent +} + +// ApplyF is the functional equivalent of Apply +func (r Rule) ApplyF(h events.HandlerFunc) events.HandlerFunc { + return r.Apply(events.Handler(h)) +} +*/ diff --git a/api/v1/lib/extras/store/singleton.go b/api/v1/lib/extras/store/singleton.go new file mode 100644 index 00000000..fab5bc3b --- /dev/null +++ b/api/v1/lib/extras/store/singleton.go @@ -0,0 +1,157 @@ +package store + +import ( + "errors" + "sync/atomic" +) + +type ( + Getter interface { + Get() (string, error) + } + + GetFunc func() (string, error) + + Setter interface { + Set(string) error + } + + SetFunc func(string) error + + // Singleton is a thread-safe abstraction to load and store a string + Singleton interface { + Getter + Setter + } + + SingletonAdapter struct { + GetFunc + SetFunc + } + + SingletonDecorator interface { + Decorate(Singleton) Singleton + } +) + +func (f GetFunc) Get() (string, error) { return f() } +func (f SetFunc) Set(s string) error { return f(s) } + +var ErrNotFound = errors.New("value not found in store") + +func NewInMemorySingleton() Singleton { + var value atomic.Value + return &SingletonAdapter{ + func() (string, error) { + x := value.Load() + if x == nil { + return "", ErrNotFound + } + return x.(string), nil + }, + func(s string) error { + value.Store(s) + return nil + }, + } +} + +type ( + GetFuncDecorator func(Getter, string, error) (string, error) + SetFuncDecorator func(Setter, string, error) error +) + +func DoSet() SetFuncDecorator { + return func(s Setter, v string, _ error) error { + return s.Set(v) + } +} + +func (f SetFuncDecorator) AndThen(f2 SetFuncDecorator) SetFuncDecorator { + return func(s Setter, v string, err error) error { + err = f(s, v, err) + if err != nil { + return err + } + return f2(s, v, nil) + } +} + +func (f SetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + s.Get, + SetFunc(func(v string) error { + return f(s, v, nil) + }), + } +} + +func DoGet() GetFuncDecorator { + return func(s Getter, _ string, _ error) (string, error) { + return s.Get() + } +} + +func (f GetFuncDecorator) AndThen(f2 GetFuncDecorator) GetFuncDecorator { + return func(s Getter, v string, err error) (string, error) { + v, err = f(s, v, err) + if err != nil { + return v, err + } + return f2(s, v, nil) + } +} + +func (f GetFuncDecorator) Decorate(s Singleton) Singleton { + if f == nil { + return s + } + return &SingletonAdapter{ + GetFunc(func() (string, error) { + return f(s, "", nil) + }), + s.Set, + } +} + +func DecorateSingleton(s Singleton, ds ...SingletonDecorator) Singleton { + for _, d := range ds { + if d != nil { + s = d.Decorate(s) + } + } + return s +} + +// GetOrPanic curries the result of a Getter invocation: the returned func only ever returns the string component when +// the error component of the underlying Get() call is nil. If Get() generates an error then the curried func panics. +func GetOrPanic(g Getter) func() string { + return func() string { + v, err := g.Get() + if err != nil { + panic(err) + } + return v + } +} + +func GetIgnoreErrors(g Getter) func() string { + return func() string { + v, _ := g.Get() + return v + } +} + +// SetOrPanic curries the result of a Setter invocation: the returned func only ever returns normally when the error +// component of the underlying Set() call is nil. If Set() generates an error then the curried func panics. +func SetOrPanic(s Setter) func(v string) { + return func(v string) { + err := s.Set(v) + if err != nil { + panic(err) + } + } +} diff --git a/api/v1/lib/httpcli/http.go b/api/v1/lib/httpcli/http.go index ecc8bc17..0258eb47 100644 --- a/api/v1/lib/httpcli/http.go +++ b/api/v1/lib/httpcli/http.go @@ -25,13 +25,8 @@ type ProtocolError string // Error implements error interface func (pe ProtocolError) Error() string { return string(pe) } -var defaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) - const ( debug = false // TODO(jdef) kill me at some point - - indexRequestContentType = 0 // index into Client.codec.MediaTypes for request content type - indexResponseContentType = 1 // index into Client.codec.MediaTypes for expected response content type ) // DoFunc sends an HTTP request and returns an HTTP response. @@ -77,15 +72,31 @@ type Client struct { handleResponse ResponseHandler } +var ( + DefaultCodec = &encoding.ProtobufCodec + DefaultHeaders = http.Header{} + + // DefaultConfigOpt represents the default client config options. + DefaultConfigOpt = []ConfigOpt{ + Transport(func(t *http.Transport) { + // all calls should be ack'd by the server within this interval. + t.ResponseHeaderTimeout = 15 * time.Second + t.MaxIdleConnsPerHost = 2 // don't depend on go's default + }), + } + + DefaultErrorMapper = ErrorMapperFunc(apierrors.FromResponse) +) + // New returns a new Client with the given Opts applied. // Callers are expected to configure the URL, Do, and Codec options prior to // invoking Do. func New(opts ...Opt) *Client { c := &Client{ - codec: &encoding.ProtobufCodec, + codec: DefaultCodec, do: With(DefaultConfigOpt...), - header: http.Header{}, - errorMapper: defaultErrorMapper, + header: DefaultHeaders, + errorMapper: DefaultErrorMapper, } c.buildRequest = c.BuildRequest c.handleResponse = c.HandleResponse @@ -154,8 +165,8 @@ func (c *Client) BuildRequest(m encoding.Marshaler, opt ...RequestOpt) (*http.Re return helper. withOptions(c.requestOpts, opt). withHeaders(c.header). - withHeader("Content-Type", c.codec.MediaTypes[indexRequestContentType]). - withHeader("Accept", c.codec.MediaTypes[indexResponseContentType]). + withHeader("Content-Type", c.codec.RequestContentType()). + withHeader("Accept", c.codec.ResponseContentType()). Request, nil } @@ -183,7 +194,7 @@ func (c *Client) HandleResponse(res *http.Response, err error) (mesos.Response, log.Println("request OK, decoding response") } ct := res.Header.Get("Content-Type") - if ct != c.codec.MediaTypes[indexResponseContentType] { + if ct != c.codec.ResponseContentType() { res.Body.Close() return nil, ProtocolError(fmt.Sprintf("unexpected content type: %q", ct)) } @@ -313,9 +324,6 @@ type Config struct { type ConfigOpt func(*Config) -// DefaultConfigOpt represents the default client config options. -var DefaultConfigOpt []ConfigOpt - // With returns a DoFunc that executes HTTP round-trips. // The default implementation provides reasonable defaults for timeouts: // keep-alive, connection, request/response read/write, and TLS handshake. diff --git a/api/v1/lib/httpcli/httpsched/state_test.go b/api/v1/lib/httpcli/httpsched/state_test.go index 9210fe35..92a3e1fe 100644 --- a/api/v1/lib/httpcli/httpsched/state_test.go +++ b/api/v1/lib/httpcli/httpsched/state_test.go @@ -5,29 +5,16 @@ import ( "testing" "github.com/mesos/mesos-go/api/v1/lib/encoding" + "github.com/mesos/mesos-go/api/v1/lib/extras/latch" "github.com/mesos/mesos-go/api/v1/lib/scheduler" ) -type latch struct{ line chan struct{} } - -func newLatch() *latch { return &latch{make(chan struct{})} } -func (l *latch) Reset() { l.line = make(chan struct{}) } -func (l *latch) Close() { close(l.line) } -func (l *latch) Closed() (result bool) { - select { - case <-l.line: - result = true - default: - } - return -} - func TestDisconnectionDecoder(t *testing.T) { // invoke disconnect upon decoder errors expected := errors.New("unmarshaler error") decoder := encoding.DecoderFunc(func(_ encoding.Unmarshaler) error { return expected }) - latch := newLatch() + latch := new(latch.L).Reset() d := disconnectionDecoder(decoder, latch.Close) err := d.Decode(nil) diff --git a/api/v1/lib/scheduler/calls/calls.go b/api/v1/lib/scheduler/calls/calls.go index 3a41b8b7..42fdbfc0 100644 --- a/api/v1/lib/scheduler/calls/calls.go +++ b/api/v1/lib/scheduler/calls/calls.go @@ -36,6 +36,14 @@ func RefuseSecondsWithJitter(r *rand.Rand, d time.Duration) scheduler.CallOpt { }) } +// RefuseSeconds returns a calls.Filters option that sets RefuseSeconds to the given duration +func RefuseSeconds(d time.Duration) scheduler.CallOpt { + asFloat := d.Seconds() + return Filters(func(f *mesos.Filters) { + f.RefuseSeconds = &asFloat + }) +} + // Framework sets a scheduler.Call's FrameworkID func Framework(id string) scheduler.CallOpt { return func(c *scheduler.Call) { diff --git a/api/v1/lib/scheduler/events.go b/api/v1/lib/scheduler/events.go index c998f003..26bcea9e 100644 --- a/api/v1/lib/scheduler/events.go +++ b/api/v1/lib/scheduler/events.go @@ -14,5 +14,5 @@ func (ep EventPredicate) Apply(e *Event) (result bool) { return } -// Happens implements scheduler/events.Happens -func (t Event_Type) Happens() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } +// Predicate implements scheduler/events.Predicate +func (t Event_Type) Predicate() EventPredicate { return func(e *Event) bool { return e.GetType() == t } } diff --git a/api/v1/lib/scheduler/events/events.go b/api/v1/lib/scheduler/events/events.go index ed482801..7366280c 100644 --- a/api/v1/lib/scheduler/events/events.go +++ b/api/v1/lib/scheduler/events/events.go @@ -1,6 +1,8 @@ package events import ( + "sync" + "github.com/mesos/mesos-go/api/v1/lib/scheduler" "github.com/mesos/mesos-go/api/v1/lib/scheduler/calls" ) @@ -15,11 +17,14 @@ type ( // HandlerFunc is a functional adaptation of the Handler interface HandlerFunc func(*scheduler.Event) error + HandlerSet map[scheduler.Event_Type]Handler + HandlerFuncSet map[scheduler.Event_Type]HandlerFunc + // Mux maps event types to Handlers (only one Handler for each type). A "default" // Handler implementation may be provided to handle cases in which there is no // registered Handler for specific event type. Mux struct { - handlers map[scheduler.Event_Type]Handler + handlers HandlerSet defaultHandler Handler } @@ -30,18 +35,20 @@ type ( // Handlers aggregates Handler things Handlers []Handler - Happens interface { - Happens() scheduler.EventPredicate + Predicate interface { + Predicate() scheduler.EventPredicate } ) // HandleEvent implements Handler for HandlerFunc func (f HandlerFunc) HandleEvent(e *scheduler.Event) error { return f(e) } +func NoopHandler() HandlerFunc { return func(_ *scheduler.Event) error { return nil } } + // NewMux generates and returns a new, empty Mux instance. func NewMux(opts ...Option) *Mux { m := &Mux{ - handlers: make(map[scheduler.Event_Type]Handler), + handlers: make(HandlerSet), } m.With(opts...) return m @@ -62,15 +69,15 @@ func (m *Mux) With(opts ...Option) Option { } // HandleEvent implements Handler for Mux -func (m *Mux) HandleEvent(e *scheduler.Event) (err error) { - h, found := m.handlers[e.GetType()] - if !found { - h = m.defaultHandler +func (m *Mux) HandleEvent(e *scheduler.Event) error { + ok, err := m.handlers.tryHandleEvent(e) + if ok { + return err } - if h != nil { - err = h.HandleEvent(e) + if m.defaultHandler != nil { + return m.defaultHandler.HandleEvent(e) } - return + return nil } // Handle returns an option that configures a Handler to handle a specific event type. @@ -88,8 +95,22 @@ func Handle(et scheduler.Event_Type, eh Handler) Option { } } +// HandleEvent implements Handler for HandlerSet +func (hs HandlerSet) HandleEvent(e *scheduler.Event) (err error) { + _, err = hs.tryHandleEvent(e) + return +} + +// tryHandleEvent returns true if the event was handled by a member of the HandlerSet +func (hs HandlerSet) tryHandleEvent(e *scheduler.Event) (bool, error) { + if h := hs[e.GetType()]; h != nil { + return true, h.HandleEvent(e) + } + return false, nil +} + // Map returns an Option that configures multiple Handler objects. -func Map(handlers map[scheduler.Event_Type]Handler) (option Option) { +func (handlers HandlerSet) ToOption() (option Option) { option = func(m *Mux) Option { type history struct { et scheduler.Event_Type @@ -114,13 +135,18 @@ func Map(handlers map[scheduler.Event_Type]Handler) (option Option) { return } -// MapFuncs is the functional adaptation of Map -func MapFuncs(handlers map[scheduler.Event_Type]HandlerFunc) (option Option) { - h := make(map[scheduler.Event_Type]Handler, len(handlers)) +// HandlerSet converts a HandlerFuncSet +func (handlers HandlerFuncSet) HandlerSet() HandlerSet { + h := make(HandlerSet, len(handlers)) for k, v := range handlers { h[k] = v } - return Map(h) + return h +} + +// ToOption converts a HandlerFuncSet +func (hs HandlerFuncSet) ToOption() (option Option) { + return hs.HandlerSet().ToOption() } // DefaultHandler returns an option that configures the default handler that's invoked @@ -133,53 +159,71 @@ func DefaultHandler(eh Handler) Option { } } +// AckError wraps a caller-generated error and tracks the call that failed. +type AckError struct { + Ack *scheduler.Call + Cause error +} + +func (err *AckError) Error() string { return err.Cause.Error() } + // AcknowledgeUpdates generates a Handler that sends an Acknowledge call to Mesos for every -// UPDATE event that's received. -func AcknowledgeUpdates(callerGetter func() calls.Caller) Handler { +// UPDATE event that's received (that requests an ACK). +func AcknowledgeUpdates(callerLookup func() calls.Caller) Handler { return WhenFunc(scheduler.Event_UPDATE, func(e *scheduler.Event) (err error) { var ( s = e.GetUpdate().GetStatus() uuid = s.GetUUID() ) + // only ACK non-empty UUID's, as per mesos scheduler spec if len(uuid) > 0 { ack := calls.Acknowledge( s.GetAgentID().GetValue(), s.TaskID.Value, uuid, ) - err = calls.CallNoData(callerGetter(), ack) + err = calls.CallNoData(callerLookup(), ack) + if err != nil { + err = &AckError{ack, err} + } } return }) } +// When +// Deprecated in favor of Rules. func Once(h Handler) Handler { - called := false + var once sync.Once return HandlerFunc(func(e *scheduler.Event) (err error) { - if !called { - called = true + once.Do(func() { err = h.HandleEvent(e) - } + }) return }) } +// When +// Deprecated in favor of Rules. func OnceFunc(h HandlerFunc) Handler { return Once(h) } -func When(p Happens, h Handler) Handler { +// When +// Deprecated in favor of Rules. +func When(p Predicate, h Handler) Handler { return HandlerFunc(func(e *scheduler.Event) (err error) { - if p.Happens().Apply(e) { + if p.Predicate().Apply(e) { err = h.HandleEvent(e) } return }) } -func WhenFunc(p Happens, h HandlerFunc) Handler { return When(p, h) } +// WhenFunc +// Deprecated in favor of Rules. +func WhenFunc(p Predicate, h HandlerFunc) Handler { return When(p, h) } -var _ = Handler(Handlers{}) // Handlers implements Handler - -// HandleEvent implements Handler for Handlers +// HandleEvent implements Handler for Handlers. +// Deprecated in favor of Rules. func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { for _, h := range hs { if h != nil { @@ -188,5 +232,5 @@ func (hs Handlers) HandleEvent(e *scheduler.Event) (err error) { } } } - return err + return } diff --git a/api/v1/lib/scheduler/events/predicates.go b/api/v1/lib/scheduler/events/predicates.go index fb5be74d..6aefb64a 100644 --- a/api/v1/lib/scheduler/events/predicates.go +++ b/api/v1/lib/scheduler/events/predicates.go @@ -6,6 +6,7 @@ import ( type PredicateBool func() bool -func (b PredicateBool) Happens() scheduler.EventPredicate { +// Predicate implements scheduler.events.Predicate +func (b PredicateBool) Predicate() scheduler.EventPredicate { return func(_ *scheduler.Event) bool { return b() } } diff --git a/api/v1/vendor/vendor.json b/api/v1/vendor/vendor.json index b32651e1..9e8df969 100644 --- a/api/v1/vendor/vendor.json +++ b/api/v1/vendor/vendor.json @@ -3,706 +3,11 @@ "ignore": "test", "package": [ { - "checksumSHA1": "RIVAETtE3FvCtjbSOh0REf6JgQk=", - "path": "github.com/gogo/protobuf/codec", + "checksumSHA1": "2I4udA/dza74M1bOgQuEPz103gw=", + "path": "github.com/gogo/protobuf", "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "tEjFocOVKUr8IK6jkvegQlfqjOw=", - "path": "github.com/gogo/protobuf/gogoproto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CLfzGRKKVJkecd9WCEtnKI2NPtI=", - "path": "github.com/gogo/protobuf/io", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Ol9P36yJqSe2nMDjKyYmz7i3TRs=", - "path": "github.com/gogo/protobuf/jsonpb", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "paUMgsbzE5tXgsiZ+RTRcEI68I4=", - "path": "github.com/gogo/protobuf/jsonpb/jsonpb_test_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cTbYeqRl+/qhD1JZvZmT5OAjaXA=", - "path": "github.com/gogo/protobuf/plugin/defaultcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Pjc29+wvgMEWJkw9PqVwTWd5R/c=", - "path": "github.com/gogo/protobuf/plugin/description", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NP3xQexLANLm++9RCktWr1ovOY0=", - "path": "github.com/gogo/protobuf/plugin/embedcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yzLXQWgFsBkFkEHScmPkcwdN2iY=", - "path": "github.com/gogo/protobuf/plugin/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Tx1FOos8BwPTDNp1LhAoG8Co6ew=", - "path": "github.com/gogo/protobuf/plugin/equal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xS3aAjAbrgPgnNkTUlTOuAmxXQ8=", - "path": "github.com/gogo/protobuf/plugin/face", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ugvdtinMoMF6MdKu4lN+D+xb0gM=", - "path": "github.com/gogo/protobuf/plugin/gostring", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7VOG8kB8XBzoNkErOWyR3QdpF5I=", - "path": "github.com/gogo/protobuf/plugin/grpc", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "GK139O7VRP+DHzVOG7PIBrzEpYc=", - "path": "github.com/gogo/protobuf/plugin/marshalto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/dztWeNfEY/M2fBB2hjO4vC2idE=", - "path": "github.com/gogo/protobuf/plugin/oneofcheck", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//9AgNP7EaQ42IGI5gp5GBlqq+8=", - "path": "github.com/gogo/protobuf/plugin/populate", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3doyBfoqf4eU3Rkn4nl6MK7I5QM=", - "path": "github.com/gogo/protobuf/plugin/size", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "6C5JEnHpRtsBQ+nhNnDuZtw35Gk=", - "path": "github.com/gogo/protobuf/plugin/stringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5/vombKpWe8yH5q2RH3wZPId0no=", - "path": "github.com/gogo/protobuf/plugin/testgen", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "5fuoaG6BK9wOKAyp+AdIowdeIx8=", - "path": "github.com/gogo/protobuf/plugin/union", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zDRKoIeCInauKHmQkTuixhhAjmA=", - "path": "github.com/gogo/protobuf/plugin/unmarshal", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hyS3vR96CWcq5guGQXKUWVrlvGA=", - "path": "github.com/gogo/protobuf/proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "WYKtRaKlF90mjEVLg8QuIFEuKH0=", - "path": "github.com/gogo/protobuf/proto/proto3_proto", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "buFKlCd6bVy4ynsgaFZBC/NhhJM=", - "path": "github.com/gogo/protobuf/proto/testdata", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "VkdsPnt/LryLUDkSNuVVIqwq4Fc=", - "path": "github.com/gogo/protobuf/protoc-gen-combo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lYD+Czpx21KQfnMj/VJbD2ll1nI=", - "path": "github.com/gogo/protobuf/protoc-gen-gofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RNSh5A+eMLvpb36czti5+wfm4wA=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "u0+Vvbf2f8FWldJOohemISYPLYo=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/descriptor", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "NQbUjvqwqPFFVa8H4ntwuViQtLE=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/generator", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DmPXpIbl/qpcje0w4JAJW4QjEDM=", - "path": "github.com/gogo/protobuf/protoc-gen-gogo/plugin", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "X0kogVRUqIRdipIVRbcTwZmODac=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "n6qb61BdGIyL1Sr6+k2L9WvbaQY=", - "path": "github.com/gogo/protobuf/protoc-gen-gogofaster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "igZu+7E154ueL7+91KAoUH3Kb4c=", - "path": "github.com/gogo/protobuf/protoc-gen-gogoslick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "J1bPjNLs1CCZv3pnl2QMYiD9f7Q=", - "path": "github.com/gogo/protobuf/protoc-min-version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6cMbpJj41MpihS5eP4SIliRBK4=", - "path": "github.com/gogo/protobuf/sortkeys", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "lbHS2lM7RHQE3xseWie466dKY2I=", - "path": "github.com/gogo/protobuf/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "SxY4AAxSzboRLBc38Yyn1s4okI0=", - "path": "github.com/gogo/protobuf/test/casttype", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "CfGm603zYf9bqWAC0Z3eBdrzuGg=", - "path": "github.com/gogo/protobuf/test/casttype/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "FmoK2j/5ncq8T+eHCMPxSeqnzFY=", - "path": "github.com/gogo/protobuf/test/casttype/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "p4kmpuTD71Z3dvhbICrF2xbGosU=", - "path": "github.com/gogo/protobuf/test/casttype/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0c98JonLeFNSc3qdVyoVcfCriN4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mTi59HJocNgdkNGkYRLz2pn8+WE=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "TYBTBsMDW3Fsfs3s0E8CRNcxEu4=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sIpq5aF4bvnqOa/4YK2gWQNcOoc=", - "path": "github.com/gogo/protobuf/test/casttype/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4XzzMi5+/zaIuy2EWEe0ufCuQtI=", - "path": "github.com/gogo/protobuf/test/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "aq5aEfH8+HPgqKAo3U9EpMjZZUo=", - "path": "github.com/gogo/protobuf/test/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "OCUBqnfbWfraPTB8oW+8/cFiXKw=", - "path": "github.com/gogo/protobuf/test/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "RqtUd2ZnGrwwf13KjYQwOK16F3M=", - "path": "github.com/gogo/protobuf/test/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "1d0nW4JvpT3ALIkT5f0y8T5M+xk=", - "path": "github.com/gogo/protobuf/test/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "PCBUEHiaKfjZZfo77m4W32qOeB4=", - "path": "github.com/gogo/protobuf/test/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Q6fiuAzwlM4kVK3PMF9vWnvmays=", - "path": "github.com/gogo/protobuf/test/custom", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2qwk23WTep6Z6DAq2hCMf9OsT/I=", - "path": "github.com/gogo/protobuf/test/custom-dash-type", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "h0LHbcpzexyjfSPu/LbrRM1Zg8c=", - "path": "github.com/gogo/protobuf/test/custombytesnonstruct", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "E2/sHgeCMMpdjoEBb4SY7G8p4Uk=", - "path": "github.com/gogo/protobuf/test/dashfilename", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9yHuzxDG9ZjtDFoJLEomVUFIyz0=", - "path": "github.com/gogo/protobuf/test/defaultconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "S+k0F1XhrDFd+igafb5FLceyUr0=", - "path": "github.com/gogo/protobuf/test/embedconflict", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "75vwg1za0UP2O3YLwPR8HD14SOU=", - "path": "github.com/gogo/protobuf/test/empty-issue70", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ncjf6Xzc4vOfR9YidS6Cc9QTLtw=", - "path": "github.com/gogo/protobuf/test/enumprefix", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fsQ9GD7yVIWT2VC2qI6wHYIdISM=", - "path": "github.com/gogo/protobuf/test/enumstringer", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "UIBxh/5EDEL5DIvVoH4ISmHBDys=", - "path": "github.com/gogo/protobuf/test/example", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YiSMdmPIdyFvXvT/tlc86DjwJLI=", - "path": "github.com/gogo/protobuf/test/fuzztests", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "yp7uRKY5gyfh6vSXnVjXIzGAXE4=", - "path": "github.com/gogo/protobuf/test/group", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "DeBkH1l+Gub2I4LgrWrnWbLVjpE=", - "path": "github.com/gogo/protobuf/test/importdedup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AVJIC2MdvEh9UfbN40073Z/d0f8=", - "path": "github.com/gogo/protobuf/test/importdedup/subpkg", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0AWSyTxYd97FyMn7gZ95Ej/3m4s=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/t6kWiQ82zTH5KSMafg/uQ16ask=", - "path": "github.com/gogo/protobuf/test/indeximport-issue72/index", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9NEWM8d2HXtilq+/QR+rLmJhOkI=", - "path": "github.com/gogo/protobuf/test/issue34", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "EUSuYKzjDRdtYYSF7r30Xs7FgDg=", - "path": "github.com/gogo/protobuf/test/issue42order", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "7fjvhomoxX3rt5gJZZBWp5sKkhs=", - "path": "github.com/gogo/protobuf/test/issue8", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4ZPo2LIaVv3lJZEQYwPLhlvBktc=", - "path": "github.com/gogo/protobuf/test/mapsproto2", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cNiwjpj2RmyJNjiqvJoFQUPsRqM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "vR6wpg/XpCULa9KTONFHXrFLmIQ=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "mqF9ePL2nMR0iyJWBwpZl7VSjqA=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "psHlD68ijJjBBHqN/MgG4MGkC3c=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "hCfdSModEttRBnDgDBy2Zx0e/uc=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "cIszinXlkReEzP4DLVqhFr80loI=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "AqEcvNXhMBPX9y4C1JdU5ci6djM=", - "path": "github.com/gogo/protobuf/test/mapsproto2/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "//KmhDh/C/fHRA3oGNMSGYALpuU=", - "path": "github.com/gogo/protobuf/test/mixbench", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0fgRfF8zijkFLXG6OcmgfXv+070=", - "path": "github.com/gogo/protobuf/test/moredefaults", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ahMVz714gNBvYBmO2I+T2EJIeIc=", - "path": "github.com/gogo/protobuf/test/oneof", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "/vpBUlSaib5le0urbk9j8ySiRD0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "4TGArepiMDOGxLQUnyIFWkCV860=", - "path": "github.com/gogo/protobuf/test/oneof/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "sbzC8eZJZ5VWaGwzRxSTPIPQPME=", - "path": "github.com/gogo/protobuf/test/oneof/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YgoWSjBZxO1FAkc1tXUndt4IXS0=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "O8WR3D8SHq0gFxa7+rf0MdelAoA=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ebUJvjKLbg5R9F7nTUuNT5s661w=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "HWM1axY96SEoc1bY2edTjlld9c4=", - "path": "github.com/gogo/protobuf/test/oneof/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "YBnrab1VG14LcgYyVLM2uFswhgc=", - "path": "github.com/gogo/protobuf/test/oneof3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2MOPs9vr5XgDUTtjm8E67tJnXyo=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "xfBBM0eq7rW41faUfZh75z+Xj3I=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "T7NdX8Dsv3mfYDXic/YVo1urNSQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "zb2LHnn4ql6QQwgVpE1JkZIxcHE=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6bMJP3iB2kyb8e6p3GdoRCBtCk=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "QdexN5eBoxSSga9kn8uw3la1jJA=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "fDFnUhPE51bxH9yhMjWpEPu/IQQ=", - "path": "github.com/gogo/protobuf/test/oneof3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "XVtCxPLxfWUo33u2xSRP1hzgd/0=", - "path": "github.com/gogo/protobuf/test/packed", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ZYwqz+QwhSsp/1YKpn1ixA4ahCE=", - "path": "github.com/gogo/protobuf/test/required", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "bCqTYwvGJl6k/7nSxSmMjtIs4VE=", - "path": "github.com/gogo/protobuf/test/sizeunderscore", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "9ZhQFwpC3uupLOELXbdLJ2Ss040=", - "path": "github.com/gogo/protobuf/test/tags", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eM0kVXUbz6sinSfxARQe/cTk15k=", - "path": "github.com/gogo/protobuf/test/theproto3", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "eBxOWTGmgxhWG1U1wcSjuOFl4sU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/both", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "BhPGpjdC9375YDejUFVlsfzolXg=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/marshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "MdGf5p9TOSeoMj0vGhFHqktACNk=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/neither", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oSeY/ryVt9ZenUM4xC4ocDGB6Oc=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "ivhKH76IrT1EHyWzImjedlTPdCo=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeboth", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "oXfCUeKjTKLTqjmJlDUaz0HZesU=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafemarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "2QEqlqrIIY9Y1On/qVoMu6ABj80=", - "path": "github.com/gogo/protobuf/test/theproto3/combos/unsafeunmarshaler", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "IG6H7z0wDBAqcsF7sojaOWTWNI4=", - "path": "github.com/gogo/protobuf/test/unmarshalmerge", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "Y5w3d7x5quR19o+iT/HhphY89/I=", - "path": "github.com/gogo/protobuf/test/unrecognized", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "8CjEqnI9sTWfSeGX98CNrrilguw=", - "path": "github.com/gogo/protobuf/test/unrecognizedgroup", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "3KeNOHQHQb4ZjB3XpigiYUDqLWM=", - "path": "github.com/gogo/protobuf/vanity", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "e6dohUIo6Qf0ixPXzxH7LExfrDk=", - "path": "github.com/gogo/protobuf/vanity/command", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "0/Iq33SFI/UbS8LJeTyOadMxgPE=", - "path": "github.com/gogo/protobuf/vanity/test", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "jEwyZC+hiPl2avUlrgEDhtCEiT4=", - "path": "github.com/gogo/protobuf/vanity/test/fast", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "F3rR7HTm2Wi4jlPedckmvNT7TI8=", - "path": "github.com/gogo/protobuf/vanity/test/faster", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "C9/L7Jpst+Unink8JiZuXLHJgv4=", - "path": "github.com/gogo/protobuf/vanity/test/slick", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" - }, - { - "checksumSHA1": "KLFcaMb5M2MQ7Sl3gLXl70wgKIg=", - "path": "github.com/gogo/protobuf/version", - "revision": "2093b57e5ca2ccbee4626814100bc1aada691b18", - "revisionTime": "2015-09-16T11:01:26Z" + "revisionTime": "2015-09-16T11:01:26Z", + "tree": true }, { "checksumSHA1": "ANHyMfOc1XnqDnlxNipBdpFE1qw=",