Skip to content

Commit

Permalink
add: openobserve log support
Browse files Browse the repository at this point in the history
  • Loading branch information
minoic committed Jun 25, 2024
1 parent 3e1dd65 commit f17ec1b
Show file tree
Hide file tree
Showing 8 changed files with 173 additions and 20 deletions.
10 changes: 5 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
FROM golang:1.22.0-alpine as builder
FROM golang:1.22.3-alpine as builder

WORKDIR /workspace

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories \
&& apk add --no-cache upx tzdata && rm -rf /var/cache/apk/*
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache upx tzdata && rm -rf /var/cache/apk/*

ENV GO111MODULE=on
ENV GOPROXY=https://goproxy.cn
Expand Down Expand Up @@ -34,8 +34,8 @@ EXPOSE $PORT
ENV TZ=Asia/Shanghai
#RUN DEBIAN_FRONTEND=noninteractive TZ=Asia/Shanghai apt-get -qq update \
# && apt-get -qq install -y --no-install-recommends ca-certificates curl tzdata
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories \
&& apk upgrade --no-cache --available \
#RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk upgrade --no-cache --available \
&& apk --no-cache add tzdata ca-certificates libc6-compat libgcc libstdc++ curl

COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert
Expand Down
132 changes: 132 additions & 0 deletions common/openobserve/openobserve.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package openobserve

import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"

"github.com/bellis-daemon/bellis/common"
"github.com/bellis-daemon/bellis/common/storage"
"github.com/minoic/glgf"
)

type openObserveWriter struct {
level string
c chan []byte
}

func (this *openObserveWriter) Write(p []byte) (n int, err error) {
this.c <- p
return len(p), nil
}

type openObserve struct {
org string
username string
password string
client *http.Client
writers []openObserveWriter
}

func (this *openObserve) send(logs []map[string]any) error {
var buf bytes.Buffer
json.NewEncoder(&buf).Encode(logs)
req, err := http.NewRequest("POST", fmt.Sprintf("https://api.openobserve.ai/api/%s/default/_json", storage.Config().OpenObserveOrg), &buf)
if err != nil {
return err
}
req.SetBasicAuth(storage.Config().OpenObserveUsername, storage.Config().OpenObservePassword)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")
_, err = this.client.Do(req)
if err != nil {
return err
}
return nil
}

func (this *openObserve) run() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := make(chan map[string]any, 16)
for i := range this.writers {
go func() {
for {
select {
case <-ctx.Done():
return
case msg := <-this.writers[i].c:
c <- map[string]any{
"level": this.writers[i].level,
"hostname": common.Hostname(),
"log": string(msg),
}
}
}
}()
}
list := []map[string]any{}
ticker := time.NewTicker(30 * time.Second)
for {
select {
case <-ticker.C:
if len(list) != 0 {
err := this.send(list)
if err != nil {
fmt.Println(err)
} else {
list = []map[string]any{}
}
}
case msg := <-c:
list = append(list, msg)
if len(list) >= 10 {
err := this.send(list)
if err != nil {
fmt.Println(err)
} else {
list = []map[string]any{}
}
}
}
}
}

func RegisterGlgf() {
instance := &openObserve{
org: storage.Config().OpenObserveOrg,
username: storage.Config().OpenObserveUsername,
password: storage.Config().OpenObservePassword,
client: http.DefaultClient,
writers: []openObserveWriter{
{
glgf.DEBG.String(),
make(chan []byte),
},
{
glgf.INFO.String(),
make(chan []byte),
},
{
glgf.WARN.String(),
make(chan []byte),
},
{
glgf.ERR.String(),
make(chan []byte),
},
{
glgf.OK.String(),
make(chan []byte),
},
},
}
glgf.Get().AddLevelWriter(glgf.DEBG, &instance.writers[0])
glgf.Get().AddLevelWriter(glgf.INFO, &instance.writers[1])
glgf.Get().AddLevelWriter(glgf.WARN, &instance.writers[2])
glgf.Get().AddLevelWriter(glgf.ERR, &instance.writers[3])
glgf.Get().AddLevelWriter(glgf.OK, &instance.writers[4])
}
25 changes: 15 additions & 10 deletions common/storage/etcd.go → common/storage/config.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package storage

import (
"github.com/minoic/glgf"
"gopkg.in/yaml.v3"
"net/http"
"os"
"strings"
"sync"

"github.com/minoic/glgf"
"gopkg.in/yaml.v3"
)

var etcdOnce sync.Once
var etcdConfig *ConfigInfo
var configOnce sync.Once
var config *ConfigInfo

type ConfigInfo struct {
InfluxDBToken string `yaml:"influxdb_token"`
Expand All @@ -26,24 +27,28 @@ type ConfigInfo struct {
RedisAddrs []string `yaml:"redis_addrs"`
RedisUsername string `yaml:"redis_username"`
RedisPassword string `yaml:"redis_password"`
OpenObserveEnabled bool `yaml:"openobserve_enabled"`
OpenObserveOrg string `yaml:"openobserve_org"`
OpenObserveUsername string `yaml:"openobserve_username"`
OpenObservePassword string `yaml:"openobserve_password"`
}

func Config() *ConfigInfo {
etcdOnce.Do(func() {
configOnce.Do(func() {
url := os.ExpandEnv("$CONFIG_URL")
etcdConfig = new(ConfigInfo)
config = new(ConfigInfo)
resp, err := http.Get(url)
if err != nil {
glgf.Error(url)
panic(err)
}
defer resp.Body.Close()
err = yaml.NewDecoder(resp.Body).Decode(etcdConfig)
err = yaml.NewDecoder(resp.Body).Decode(config)
if err != nil {
panic(err)
}
etcdConfig.WebEndpoint = strings.TrimRight(etcdConfig.WebEndpoint, "/")
etcdConfig.TelegramBotApiEndpoint = strings.TrimRight(etcdConfig.TelegramBotApiEndpoint, "/")
config.WebEndpoint = strings.TrimRight(config.WebEndpoint, "/")
config.TelegramBotApiEndpoint = strings.TrimRight(config.TelegramBotApiEndpoint, "/")
})
return etcdConfig
return config
}
6 changes: 3 additions & 3 deletions modules/backend/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
FROM golang:1.22.0-alpine as builder
FROM golang:1.22.3-alpine as builder

WORKDIR /workspace

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories \
&& apk add --no-cache upx tzdata && rm -rf /var/cache/apk/*
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache upx tzdata && rm -rf /var/cache/apk/*

ENV GO111MODULE=on
ENV GOPROXY=https://goproxy.cn
Expand Down
7 changes: 6 additions & 1 deletion modules/backend/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,19 @@ package main

import (
"context"
"github.com/bellis-daemon/bellis/modules/backend/jobs"
"net"
"os"
"os/exec"
"os/signal"
"syscall"
"time"

"github.com/bellis-daemon/bellis/modules/backend/jobs"

"github.com/bellis-daemon/bellis/common"
_ "github.com/bellis-daemon/bellis/common/models"
"github.com/bellis-daemon/bellis/common/models/index"
"github.com/bellis-daemon/bellis/common/openobserve"
"github.com/bellis-daemon/bellis/common/storage"
"github.com/bellis-daemon/bellis/modules/backend/app/mobile"
_ "github.com/bellis-daemon/bellis/modules/backend/app/mobile/auth"
Expand All @@ -32,6 +34,9 @@ func init() {
common.BuildTime = BuildTime
common.GoVersion = GoVersion
glgf.Infof("BuildTime: %s, GoVersion: %s", BuildTime, GoVersion)
if storage.Config().OpenObserveEnabled{
openobserve.RegisterGlgf()
}
}

func main() {
Expand Down
4 changes: 4 additions & 0 deletions modules/dispatcher/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"github.com/bellis-daemon/bellis/common"
"github.com/bellis-daemon/bellis/common/storage"
"github.com/bellis-daemon/bellis/common/openobserve"
"github.com/bellis-daemon/bellis/modules/dispatcher/consumer"
"github.com/bellis-daemon/bellis/modules/dispatcher/dispatch"
"github.com/minoic/glgf"
Expand All @@ -17,6 +18,9 @@ func init() {
common.BuildTime = BuildTime
common.GoVersion = GoVersion
glgf.Infof("BuildTime: %s, GoVersion: %s", BuildTime, GoVersion)
if storage.Config().OpenObserveEnabled{
openobserve.RegisterGlgf()
}
}

func main() {
Expand Down
5 changes: 4 additions & 1 deletion modules/envoy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package main

import (
"github.com/bellis-daemon/bellis/common"

"github.com/bellis-daemon/bellis/common/openobserve"
"github.com/bellis-daemon/bellis/common/storage"
"github.com/bellis-daemon/bellis/modules/envoy/consumer"
"github.com/minoic/glgf"
Expand All @@ -17,6 +17,9 @@ func init() {
common.BuildTime = BuildTime
common.GoVersion = GoVersion
glgf.Infof("BuildTime: %s, GoVersion: %s", BuildTime, GoVersion)
if storage.Config().OpenObserveEnabled {
openobserve.RegisterGlgf()
}
}

func main() {
Expand Down
4 changes: 4 additions & 0 deletions modules/sentry/cmd/cluster/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"github.com/bellis-daemon/bellis/common"
"github.com/bellis-daemon/bellis/common/storage"
"github.com/bellis-daemon/bellis/common/openobserve"
_ "github.com/bellis-daemon/bellis/modules/sentry/apps/implements/all"
"github.com/bellis-daemon/bellis/modules/sentry/consumer"
"github.com/minoic/glgf"
Expand All @@ -17,6 +18,9 @@ func init() {
common.BuildTime = BuildTime
common.GoVersion = GoVersion
glgf.Infof("BuildTime: %s, GoVersion: %s", BuildTime, GoVersion)
if storage.Config().OpenObserveEnabled{
openobserve.RegisterGlgf()
}
}

func main() {
Expand Down

0 comments on commit f17ec1b

Please sign in to comment.