From 81ce56c132cac4ef84ba9e85db82829802f31c50 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Thu, 6 Jun 2019 10:42:47 +0200 Subject: [PATCH] refactor(contrib): remove cds2es uService It's replaced by engine start elasticsearch Signed-off-by: Yvonnick Esnault --- contrib/README.md | 2 +- contrib/uservices/cds2es/.gitignore | 1 - contrib/uservices/cds2es/Dockerfile | 7 --- contrib/uservices/cds2es/README.md | 34 ------------ contrib/uservices/cds2es/config.go | 38 ------------- contrib/uservices/cds2es/main.go | 85 ----------------------------- contrib/uservices/cds2es/process.go | 77 -------------------------- 7 files changed, 1 insertion(+), 243 deletions(-) delete mode 100644 contrib/uservices/cds2es/.gitignore delete mode 100644 contrib/uservices/cds2es/Dockerfile delete mode 100644 contrib/uservices/cds2es/README.md delete mode 100644 contrib/uservices/cds2es/config.go delete mode 100644 contrib/uservices/cds2es/main.go delete mode 100644 contrib/uservices/cds2es/process.go diff --git a/contrib/README.md b/contrib/README.md index 162b717e6e..2324244771 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -25,6 +25,6 @@ See [Workflow Templates Documentation](https://ovh.github.io/cds/docs/concepts/t ## µServices -- [cds2es](https://github.com/ovh/cds/tree/master/contrib/uservices/cds2es) +- [badge](https://github.com/ovh/cds/tree/master/contrib/uservices/badge) - [cds2http](https://github.com/ovh/cds/tree/master/contrib/uservices/cds2http) - [Hubot XMPP](https://github.com/ovh/cds/tree/master/contrib/uservices/hubot-xmpp) diff --git a/contrib/uservices/cds2es/.gitignore b/contrib/uservices/cds2es/.gitignore deleted file mode 100644 index 8b13789179..0000000000 --- a/contrib/uservices/cds2es/.gitignore +++ /dev/null @@ -1 +0,0 @@ - diff --git a/contrib/uservices/cds2es/Dockerfile b/contrib/uservices/cds2es/Dockerfile deleted file mode 100644 index 6ac8a32ae9..0000000000 --- a/contrib/uservices/cds2es/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM debian:jessie -RUN apt-get update && apt-get install -y ca-certificates -COPY ./cds2er /app/cds2es -COPY ./config.toml /app/config.toml -RUN chmod +x /app/cds2es && chown nobody:nogroup /app/cds2es -USER nobody -CMD ["/app/cds2es -f config.toml"] diff --git a/contrib/uservices/cds2es/README.md b/contrib/uservices/cds2es/README.md deleted file mode 100644 index 5db4213cdc..0000000000 --- a/contrib/uservices/cds2es/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# CDS to ES - -## Configuration file - -```toml - -[Debug] - log_level = "info" - -[ElasticSearch] - domain = "your-es" - index = "your-index-cds" - password = "your-token" - port = "9200" - protocol = "https" - username = "your-username" - -[Kafka] - brokers = "your-broker:9093" - group = "your-group" - password = "your-kafka-password" - topic = "your-topic" - user = "your-kafka-user" - -[Http] - port = 9085 - -``` - -## Run it - -```bash -cds2es -f config.yml -``` \ No newline at end of file diff --git a/contrib/uservices/cds2es/config.go b/contrib/uservices/cds2es/config.go deleted file mode 100644 index e23028e059..0000000000 --- a/contrib/uservices/cds2es/config.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -// Configuration is the configuraton structure for CDS API -type Configuration struct { - Kafka KafkaConf - ElasticSearch ElasticSearchConf - Debug DebugConf - Http HttpConf -} - -// HttpConf represents http configuration -type HttpConf struct { - Port int `toml:"port"` -} - -// KafkaConf represents kafka configuration -type KafkaConf struct { - Brokers string `toml:"brokers"` - Topic string `toml:"topic"` - User string `toml:"user"` - Password string `toml:"password"` - Group string `toml:"group"` -} - -// ElasticSearchConf represents elastic search configuration -type ElasticSearchConf struct { - Protocol string `toml:"protocol"` - Domain string `toml:"domain"` - Port string `toml:"port"` - Username string `toml:"username"` - Password string `toml:"password"` - Index string `toml:"index"` -} - -// DebugConf reprents log configuration -type DebugConf struct { - LogLevel string `toml:"log_level"` -} diff --git a/contrib/uservices/cds2es/main.go b/contrib/uservices/cds2es/main.go deleted file mode 100644 index 488be21724..0000000000 --- a/contrib/uservices/cds2es/main.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "net/http" - "os" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/itsjamie/gin-cors" - "github.com/pelletier/go-toml" - log "github.com/sirupsen/logrus" - - "github.com/ovh/cds/sdk" -) - -// VERSION ... -const VERSION = "0.1.0" - -func main() { - configPath := flag.String("f", "", "path to the config file") - flag.Parse() - - if *configPath == "" { - fmt.Println("Usage: cds2es -f configFile.yml") - os.Exit(1) - } - - configData, err := ioutil.ReadFile(*configPath) - if err != nil { - fmt.Printf("Unable to read configuration: %s\n", err) - os.Exit(2) - } - var conf Configuration - if err := toml.Unmarshal(configData, &conf); err != nil { - fmt.Printf("Unable to unmarshal configuration: %s", err) - os.Exit(3) - } - - switch conf.Debug.LogLevel { - case "debug": - log.SetLevel(log.DebugLevel) - case "info": - log.SetLevel(log.InfoLevel) - case "error": - log.SetLevel(log.WarnLevel) - gin.SetMode(gin.ReleaseMode) - default: - log.SetLevel(log.DebugLevel) - } - router := gin.New() - router.Use(gin.Recovery()) - - router.Use(cors.Middleware(cors.Config{ - Origins: "*", - Methods: "GET, PUT, POST, DELETE", - RequestHeaders: "Origin, Authorization, Content-Type, Accept", - MaxAge: 50 * time.Second, - Credentials: true, - ValidateHeaders: false, - })) - - router.GET("/mon/ping", func(ctx *gin.Context) { - ctx.JSON(http.StatusOK, gin.H{"version": VERSION}) - }) - - s := &http.Server{ - Addr: ":" + strconv.Itoa(conf.Http.Port), - Handler: router, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - MaxHeaderBytes: 1 << 20, - } - - c := make(chan sdk.Event) - go consume(conf, c) - go sendToES(conf, c) - log.Infof("Running cds2es on %d", conf.Http.Port) - if err := s.ListenAndServe(); err != nil { - log.Errorf("Error while running ListenAndServe: %s", err.Error()) - } -} diff --git a/contrib/uservices/cds2es/process.go b/contrib/uservices/cds2es/process.go deleted file mode 100644 index 4da8b58f3c..0000000000 --- a/contrib/uservices/cds2es/process.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "crypto/md5" - "encoding/hex" - "encoding/json" - "time" - - "github.com/mattbaird/elastigo/lib" - log "github.com/sirupsen/logrus" - "github.com/spf13/viper" - - "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/event" -) - -var esConn *elastigo.Conn - -func consume(config Configuration, c chan<- sdk.Event) { - log.Infof("Consuming kafka message") - if err := event.ConsumeKafka(config.Kafka.Brokers, config.Kafka.Topic, config.Kafka.Group, config.Kafka.User, config.Kafka.Password, - func(e sdk.Event) error { - c <- e - return nil - }, - log.Errorf, - ); err != nil { - log.Fatalf("Cannot consume kafka %s", err) - } -} - -func sendToES(config Configuration, c <-chan sdk.Event) { - //Only one ES Connection - esConn = elastigo.NewConn() - - esConn.Protocol = config.ElasticSearch.Protocol - esConn.Domain = config.ElasticSearch.Domain - esConn.Port = config.ElasticSearch.Port - esConn.Username = config.ElasticSearch.Username - esConn.Password = config.ElasticSearch.Password - - esIndex := config.ElasticSearch.Index - - for event := range c { - jsonPayload, errM := json.Marshal(event.Payload) - if errM != nil { - log.Errorf("cannot marshal payload :%s", errM) - continue - } - dataES := map[string]interface{}{ - "Username": event.Username, - "Email": event.UserMail, - "CDSName": event.CDSName, - "EventType": event.EventType, - "Hostname": event.Hostname, - "Attempts": event.Attempts, - "Timestamp": event.Timestamp, - "Event": string(jsonPayload), - "ProjectKey": event.ProjectKey, - "ApplicationName": event.ApplicationName, - "PipelineName": event.PipelineName, - "EnvironmentName": event.EnvironmentName, - "WorkflowName": event.WorkflowName, - } - _, err := esConn.IndexWithParameters(esIndex, event.EventType, getMD5Hash(string(jsonPayload)), "", 0, "", "", event.Timestamp.Format(time.RFC3339), 0, "", "", false, nil, dataES) - time.Sleep(time.Duration(viper.GetInt("pause_es")) * time.Millisecond) - if err != nil { - log.Errorf("cannot index message %+v in :%s", dataES, err) - } - } -} - -func getMD5Hash(text string) string { - hasher := md5.New() - hasher.Write([]byte(text)) - return hex.EncodeToString(hasher.Sum(nil)) -}