Skip to content

Commit

Permalink
feat(task): add Kafka consumer to create tasks from Kafka topic
Browse files Browse the repository at this point in the history
Signed-off-by: Romain Beuque <556072+rbeuque74@users.noreply.github.com>
Signed-off-by: Thomas Bétrancourt <thomas@betrancourt.net>
  • Loading branch information
rbeuque74 authored and rclsilver committed Apr 28, 2023
1 parent 6148552 commit 75197ca
Show file tree
Hide file tree
Showing 6 changed files with 632 additions and 28 deletions.
26 changes: 26 additions & 0 deletions engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/ovh/utask/models/task"
"github.com/ovh/utask/models/tasktemplate"
"github.com/ovh/utask/pkg/jsonschema"
"github.com/ovh/utask/pkg/kafkaconsumer"
"github.com/ovh/utask/pkg/metadata"
"github.com/ovh/utask/pkg/now"
"github.com/ovh/utask/pkg/taskutils"
Expand Down Expand Up @@ -147,6 +148,31 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err
if err := RetryCollector(ctx); err != nil {
return err
}

// init KafkaConsumer
kcItems, err := configstore.Filter().Store(store).Slice(kafkaconsumer.ConfigKey).Unmarshal(func() interface{} { return &kafkaconsumer.KafkaConfig{} }).Squash().GetItemList()
if err != nil {
return err
}

for _, cfgItem := range kcItems.Items {
i, err := cfgItem.Unmarshaled()
if err != nil {
return err
}

cfg := i.(*kafkaconsumer.KafkaConfig)
consumer, err := kafkaconsumer.StartNewTaskConsumer(ctx, *cfg)
if err != nil {
return err
}

if err := consumer.SetDefaultConsumer(ctx); err != nil {
return err
}

consumer.StartConsumer(ctx)
}
}
return nil
}
Expand Down
33 changes: 22 additions & 11 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.19
require (
github.com/Masterminds/sprig/v3 v3.1.0
github.com/Masterminds/squirrel v1.4.0
github.com/Shopify/sarama v1.37.2
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/fabienm/go-logrus-formatters v1.0.0
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/gin-gonic/gin v1.7.7
github.com/go-gorp/gorp v2.2.0+incompatible
github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020
github.com/go-playground/validator/v10 v10.11.1
github.com/gofrs/uuid v3.3.0+incompatible
github.com/jpillora/backoff v1.0.0
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f
Expand All @@ -29,13 +31,13 @@ require (
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v0.0.6
github.com/spf13/viper v1.4.0
github.com/stretchr/testify v1.5.1
github.com/stretchr/testify v1.8.0
github.com/tidwall/gjson v1.9.3
github.com/wI2L/fizz v0.17.0
github.com/ybriffa/go-http-digest-auth-client v0.6.3
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
golang.org/x/net v0.7.0
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
gopkg.in/mail.v2 v2.3.1
)

Expand All @@ -47,38 +49,44 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/eapache/go-resiliency v1.3.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
github.com/facebookgo/httpcontrol v0.0.0-20150708234001-ccde4420e1fe // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/fsnotify/fsnotify v1.4.7 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.5.1 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.3.1 // indirect
github.com/imdario/mergo v0.3.9 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/gokrb5/v8 v8.3.0 // indirect
github.com/jcmturner/rpc/v2 v2.0.2 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jinzhu/now v1.0.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
Expand All @@ -89,12 +97,14 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml v1.4.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/poy/onpar v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/cast v1.3.1 // indirect
Expand All @@ -112,4 +122,5 @@ require (
gopkg.in/ini.v1 v1.46.0 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

0 comments on commit 75197ca

Please sign in to comment.