diff --git a/examples/simple_plugin/go.mod b/examples/simple_plugin/go.mod index 1de8045fdc..df90168a6e 100644 --- a/examples/simple_plugin/go.mod +++ b/examples/simple_plugin/go.mod @@ -14,6 +14,8 @@ replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14 require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cloudquery/plugin-pb-go v1.11.0 // indirect github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect @@ -29,8 +31,10 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.10.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect @@ -40,6 +44,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect github.com/thoas/go-funk v0.9.3 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect diff --git a/examples/simple_plugin/go.sum b/examples/simple_plugin/go.sum index c07bfa0918..a12464d5e1 100644 --- a/examples/simple_plugin/go.sum +++ b/examples/simple_plugin/go.sum @@ -37,8 +37,12 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 h1:mFDZW1FQk9yndPvxScp7RpcOpdSHaqcgBWO7sDlx4S8= github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -163,6 +167,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.10.0 h1:c1ktzNLBun3LyQQhyty5WE3lulbOdIIyOVlkmDLehcE= +github.com/invopop/jsonschema v0.10.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -177,6 +184,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -216,6 +225,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/go.mod b/go.mod index d392a0f5f6..2d0023741c 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 + github.com/invopop/jsonschema v0.10.0 github.com/rs/zerolog v1.29.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spf13/cobra v1.6.1 @@ -32,6 +33,8 @@ replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14 require ( github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect @@ -44,12 +47,14 @@ require ( github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect diff --git a/go.sum b/go.sum index ea743c023d..46ffd94e28 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,12 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 h1:mFDZW1FQk9yndPvxScp7RpcOpdSHaqcgBWO7sDlx4S8= github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -164,6 +168,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.10.0 h1:c1ktzNLBun3LyQQhyty5WE3lulbOdIIyOVlkmDLehcE= +github.com/invopop/jsonschema v0.10.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -179,6 +186,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -222,6 +231,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index aaf80ddcc6..2332500e33 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -1,9 +1,7 @@ package scheduler import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "runtime/debug" @@ -39,83 +37,6 @@ const ( StrategyShuffle ) -type Strategy int - -func (s *Strategy) String() string { - if s == nil { - return "" - } - return AllStrategyNames[*s] -} - -// MarshalJSON implements json.Marshaler. -func (s *Strategy) MarshalJSON() ([]byte, error) { - var b bytes.Buffer - if s == nil { - b.Write([]byte("null")) - return b.Bytes(), nil - } - b.Write([]byte{'"'}) - b.Write([]byte(s.String())) - b.Write([]byte{'"'}) - return b.Bytes(), nil -} - -// UnmarshalJSON implements json.Unmarshaler. -func (s *Strategy) UnmarshalJSON(b []byte) error { - var name string - if err := json.Unmarshal(b, &name); err != nil { - return err - } - strategy, err := StrategyForName(name) - if err != nil { - return err - } - *s = strategy - return nil -} - -func (s *Strategy) Validate() error { - if s == nil { - return errors.New("scheduler strategy is nil") - } - for _, strategy := range AllStrategies { - if strategy == *s { - return nil - } - } - return fmt.Errorf("unknown scheduler strategy: %d", s) -} - -var AllStrategies = Strategies{StrategyDFS, StrategyRoundRobin, StrategyShuffle} -var AllStrategyNames = [...]string{ - StrategyDFS: "dfs", - StrategyRoundRobin: "round-robin", - StrategyShuffle: "shuffle", -} - -func StrategyForName(s string) (Strategy, error) { - for i, name := range AllStrategyNames { - if name == s { - return AllStrategies[i], nil - } - } - return StrategyDFS, fmt.Errorf("unknown scheduler strategy: %s", s) -} - -type Strategies []Strategy - -func (s Strategies) String() string { - var buffer bytes.Buffer - for i, strategy := range s { - if i > 0 { - buffer.WriteString(", ") - } - buffer.WriteString(strategy.String()) - } - return buffer.String() -} - type Option func(*Scheduler) func WithLogger(logger zerolog.Logger) Option { diff --git a/scheduler/strategy.go b/scheduler/strategy.go new file mode 100644 index 0000000000..150a3dc52e --- /dev/null +++ b/scheduler/strategy.go @@ -0,0 +1,101 @@ +package scheduler + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + + "github.com/invopop/jsonschema" +) + +type Strategy int + +func (s *Strategy) String() string { + if s == nil { + return "" + } + return AllStrategyNames[*s] +} + +// MarshalJSON implements json.Marshaler. +func (s *Strategy) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + if s == nil { + b.Write([]byte("null")) + return b.Bytes(), nil + } + b.Write([]byte{'"'}) + b.Write([]byte(s.String())) + b.Write([]byte{'"'}) + return b.Bytes(), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (s *Strategy) UnmarshalJSON(b []byte) error { + var name string + if err := json.Unmarshal(b, &name); err != nil { + return err + } + strategy, err := StrategyForName(name) + if err != nil { + return err + } + *s = strategy + return nil +} + +func (s *Strategy) Validate() error { + if s == nil { + return errors.New("scheduler strategy is nil") + } + for _, strategy := range AllStrategies { + if strategy == *s { + return nil + } + } + return fmt.Errorf("unknown scheduler strategy: %d", s) +} + +// JSONSchema uses value receiver because of https://github.com/invopop/jsonschema/issues/102 +func (Strategy) JSONSchema() *jsonschema.Schema { + enum := make([]any, len(AllStrategyNames)) + for i, s := range AllStrategyNames { + enum[i] = s + } + return &jsonschema.Schema{ + Type: "string", + Enum: enum, + Default: AllStrategyNames[StrategyDFS], + Title: "CloudQuery scheduling strategy", + } +} + +var AllStrategies = Strategies{StrategyDFS, StrategyRoundRobin, StrategyShuffle} +var AllStrategyNames = [...]string{ + StrategyDFS: "dfs", + StrategyRoundRobin: "round-robin", + StrategyShuffle: "shuffle", +} + +func StrategyForName(s string) (Strategy, error) { + for i, name := range AllStrategyNames { + if name == s { + return AllStrategies[i], nil + } + } + return StrategyDFS, fmt.Errorf("unknown scheduler strategy: %s", s) +} + +type Strategies []Strategy + +func (s Strategies) String() string { + var buffer bytes.Buffer + for i, strategy := range s { + if i > 0 { + buffer.WriteString(", ") + } + buffer.WriteString(strategy.String()) + } + return buffer.String() +} diff --git a/scheduler/strategy.json b/scheduler/strategy.json new file mode 100644 index 0000000000..a11a10fb1e --- /dev/null +++ b/scheduler/strategy.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/plugin-sdk/v4/scheduler/strategy", + "$ref": "#/$defs/Strategy", + "$defs": { + "Strategy": { + "type": "string", + "enum": [ + "dfs", + "round-robin", + "shuffle" + ], + "title": "CloudQuery scheduling strategy", + "default": "dfs" + } + } +} diff --git a/scheduler/strategy_test.go b/scheduler/strategy_test.go new file mode 100644 index 0000000000..0218a04581 --- /dev/null +++ b/scheduler/strategy_test.go @@ -0,0 +1,22 @@ +package scheduler_test + +import ( + _ "embed" + "encoding/json" + "reflect" + "testing" + + "github.com/cloudquery/plugin-sdk/v4/scheduler" + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/require" +) + +//go:embed strategy.json +var jsonSchema string + +func TestStrategy_JSONSchema(t *testing.T) { + sc := (&jsonschema.Reflector{RequiredFromJSONSchemaTags: true}).ReflectFromType(reflect.TypeOf(scheduler.StrategyDFS)) + data, err := json.MarshalIndent(sc, "", " ") + require.NoError(t, err) + require.JSONEq(t, string(data)+"\n", jsonSchema) +}