Merge pull request #984 from kl7sn/main

feat: alarm push content log link use short link #982

.github/ISSUE_TEMPLATE/ask-a-question.md

@@ -1,6 +1,6 @@
 ---
 name: Ask a question
-about: Ask anything of ilogtail. It is recommended to ask a question in discussions.
+about: Ask anything of clickvisual. It is recommended to ask a question in discussions.
 title: "[QUESTION]:"
 labels: question
 assignees: ''
@@ -15,7 +15,7 @@ Before asking a question, please first consider:
 
 -------
 
-提问之前，建议先在 iLogtail 社区寻找答案。
+提问之前，建议先在 ClickVisual 社区寻找答案。
 
 - 查看 [存在的 discussions](https://github.com/clickvisual/clickvisual/discussions/)
 - 查看 [存在的 issues](https://github.com/clickvisual/clickvisual/issues/)

api/internal/api/apiv2/base/shorturl.go

@@ -2,16 +2,13 @@ package base
 
 import (
 	"context"
-	"fmt"
-	"net/url"
 	"strings"
 	"time"
 
-	"github.com/gotomicro/ego/core/econf"
 	"github.com/gotomicro/ego/core/elog"
 
 	"github.com/clickvisual/clickvisual/api/internal/invoker"
-	"github.com/clickvisual/clickvisual/api/internal/service"
+	"github.com/clickvisual/clickvisual/api/internal/service/shorturl"
 	"github.com/clickvisual/clickvisual/api/pkg/component/core"
 	"github.com/clickvisual/clickvisual/api/pkg/model/db"
 )
@@ -52,36 +49,10 @@ func ShortURLCreate(c *core.Context) {
 		c.JSONE(1, "invalid parameter: "+err.Error(), err)
 		return
 	}
-	u, err := url.Parse(req.OriginUrl)
+	res, err := shorturl.GenShortURL(req.OriginUrl)
 	if err != nil {
-		c.JSONE(1, "invalid parameter: "+err.Error(), err)
-		return
-	}
-	v := u.Query()
-	v.Set("tab", "custom")
-	u2 := fmt.Sprintf("%s://%s%s?%s", u.Scheme, u.Host, u.Path, v.Encode())
-	shortUrl := db.BaseShortURL{
-		OriginUrl: u2,
-		SCode:     "",
-		CallCnt:   0,
-	}
-	tx := invoker.Db.Begin()
-	if err = db.ShortURLCreate(tx, &shortUrl); err != nil {
-		tx.Rollback()
-		c.JSONE(core.CodeErr, err.Error(), nil)
-		return
-	}
-	sCode := service.HashIDGenCode(shortUrl.ID)
-	if err = db.ShortURLUpdate(tx, shortUrl.ID, map[string]interface{}{"s_code": sCode}); err != nil {
-		tx.Rollback()
-		c.JSONE(core.CodeErr, err.Error(), nil)
-		return
-	}
-	if err = tx.Commit().Error; err != nil {
-		c.JSONE(core.CodeErr, err.Error(), nil)
+		c.JSONE(1, "gen short url error: "+err.Error(), err)
 		return
 	}
-	rootUrl := strings.TrimSuffix(econf.GetString("app.rootURL"), "/")
-	res := fmt.Sprintf("%s/api/share/%s", rootUrl, sCode)
 	c.JSONOK(res)
 }

api/internal/api/apiv2/storage/storage.go

@@ -250,7 +250,6 @@ func Update(c *core.Context) {
 // @Router       /api/v2/storage/{template} [post]
 func CreateStorageByTemplate(c *core.Context) {
 	tpl := strings.TrimSpace(c.Param("template"))
-
 	switch tpl {
 	case "ego":
 		createStorageByTemplateEgo(c)

api/internal/api/apiv2/storage/storage_test.go

@@ -73,3 +73,24 @@ func TestCreateJSONEachRow(t *testing.T) {
 	}, gintest.WithRoutePath("/storage"), gintest.WithRouteMiddleware(middlewares.SetMockUser()))
 	_ = objTest1.Run()
 }
+
+func TestCreateStorageByTemplate(t *testing.T) {
+	config.InitCfg()
+	_ = invoker.Init()
+	_ = service.Init()
+	objTest1 := gintest.Init()
+	// prometheus file
+	objTest1.POST(core.Handle(CreateStorageByTemplate), func(m *gintest.Mock) error {
+		byteInfo := m.Exec(
+			gintest.WithUri("/storage/ilogtail"),
+			gintest.WithJsonBody(view.ReqCreateStorageByTemplateILogtail{
+				Name:       "demo_0201_v3",
+				Brokers:    "127.0.0.1:9092",
+				DatabaseId: 14,
+				Topic:      "otlp_spans",
+			}))
+		assert.Equal(t, `{"code":0,"msg":"succ","data":""}`, string(byteInfo))
+		return nil
+	}, gintest.WithRoutePath("/storage/:template"), gintest.WithRouteMiddleware(middlewares.SetMockUser()))
+	_ = objTest1.Run()
+}

api/internal/service/alarm/pusher/factory.go

@@ -9,8 +9,10 @@ import (
 	"time"
 
 	"github.com/gotomicro/ego/core/econf"
+	"github.com/gotomicro/ego/core/elog"
 
 	"github.com/clickvisual/clickvisual/api/internal/invoker"
+	"github.com/clickvisual/clickvisual/api/internal/service/shorturl"
 	"github.com/clickvisual/clickvisual/api/pkg/model/db"
 )
 
@@ -49,29 +51,28 @@ func BuildAlarmMsg(notification db.Notification, table *db.BaseTable, alarm *db.
 	var buffer bytes.Buffer
 	// base info
 	if notification.GetStatus() == db.AlarmStatusNormal {
-		buffer.WriteString("###  <font color=#008000>您的告警已恢复</font>\n")
+		buffer.WriteString("<font color=#008000>您的告警已恢复</font>\n\n")
 	} else {
-		buffer.WriteString("###  <font color=#FF0000>您有待处理的告警</font>\n")
+		buffer.WriteString("<font color=#FF0000>您有待处理的告警</font>\n\n")
 	}
-	buffer.WriteString(fmt.Sprintf("##### 告警名称: %s\n", alarm.Name))
+	buffer.WriteString(fmt.Sprintf("【告警名称】: %s\n\n", alarm.Name))
 	if alarm.Desc != "" {
-		buffer.WriteString(fmt.Sprintf("##### 告警描述: %s\n", alarm.Desc))
+		buffer.WriteString(fmt.Sprintf("【告警描述】: %s\n\n", alarm.Desc))
 	}
 	users, phones := dutyOffices(alarm)
 	instance, _ := db.InstanceInfo(invoker.Db, table.Database.Iid)
 	statusText := "告警中"
 	for _, alert := range notification.Alerts {
 		end := alert.StartsAt.Add(time.Minute).Unix()
 		start := alert.StartsAt.Add(-alarm.GetInterval() - time.Minute).Unix()
-		annotations := alert.Annotations
-		buffer.WriteString(fmt.Sprintf("##### 触发时间: %s\n", alert.StartsAt.Add(time.Hour*8).Format("2006-01-02 15:04:05")))
-		buffer.WriteString(fmt.Sprintf("##### 相关实例: %s %s\n", instance.Name, instance.Desc))
-		buffer.WriteString(fmt.Sprintf("##### 日志库: %s %s\n", table.Name, table.Desc))
+		buffer.WriteString(fmt.Sprintf("【触发时间】: %s\n\n", alert.StartsAt.Add(time.Hour*8).Format("2006-01-02 15:04:05")))
+		buffer.WriteString(fmt.Sprintf("【相关实例】: %s %s\n\n", instance.Name, instance.Desc))
+		buffer.WriteString(fmt.Sprintf("【日志库表】: %s %s\n\n", table.Name, table.Desc))
 		if notification.GetStatus() == db.AlarmStatusNormal {
 			statusText = "已恢复"
-			buffer.WriteString("##### 状态: <font color=#008000>已恢复</font>\n")
+			buffer.WriteString("【告警状态】: <font color=#008000>已恢复</font>\n\n")
 		} else {
-			buffer.WriteString("##### 状态: <font color=red>告警中</font>\n")
+			buffer.WriteString("【告警状态】: <font color=red>告警中</font>\n\n")
 		}
 		dutyOfficesStr := ""
 		for _, user := range users {
@@ -83,21 +84,26 @@ func BuildAlarmMsg(notification db.Notification, table *db.BaseTable, alarm *db.
 		}
 		if dutyOfficesStr == "" {
 			user, _ := db.UserInfo(alarm.Uid)
-			buffer.WriteString(fmt.Sprintf("##### 创建人: %s \n", user.Nickname))
+			buffer.WriteString(fmt.Sprintf("【告警创建】: %s\n\n", user.Nickname))
 		} else {
-			buffer.WriteString(fmt.Sprintf("##### 责任人: %s \n", dutyOfficesStr))
+			buffer.WriteString(fmt.Sprintf("【告警责任】: %s\n\n", dutyOfficesStr))
 		}
-
-		buffer.WriteString(fmt.Sprintf("##### %s\n\n", annotations["description"]))
-		buffer.WriteString(fmt.Sprintf("##### clickvisual 跳转: %s/share?mode=0&tab=custom&tid=%d&kw=%s&start=%d&end=%d\n\n",
+		jumpURL := fmt.Sprintf("%s/share?mode=0&tab=custom&tid=%d&kw=%s&start=%d&end=%d",
 			strings.TrimRight(econf.GetString("app.rootURL"), "/"), filter.Tid, url.QueryEscape(filter.When), start, end,
-		))
+		)
+		shortURL, err := shorturl.GenShortURL(jumpURL)
+		if err != nil {
+			elog.Error("shorturl.GenShortURL", elog.FieldErr(err), elog.String("jumpURL", jumpURL))
+			buffer.WriteString(fmt.Sprintf("【链接跳转】: %s\n\n", jumpURL))
+		} else {
+			buffer.WriteString(fmt.Sprintf("【链接跳转】: %s\n\n", shortURL))
+		}
 		if partialLog != "" {
 			partialLog = strings.Replace(partialLog, "\"", "", -1)
 			if len(partialLog) > 600 {
-				buffer.WriteString(fmt.Sprintf("##### 日志: %s ...", partialLog[0:599]))
+				buffer.WriteString(fmt.Sprintf("【告警详情】: %s ...more", partialLog[0:599]))
 			} else {
-				buffer.WriteString(fmt.Sprintf("##### 日志: %s", partialLog))
+				buffer.WriteString(fmt.Sprintf("【告警详情】: %s", partialLog))
 			}
 		}
 	}

api/internal/service/init.go

@@ -12,6 +12,7 @@ import (
 	"github.com/clickvisual/clickvisual/api/internal/service/event"
 	"github.com/clickvisual/clickvisual/api/internal/service/kube"
 	"github.com/clickvisual/clickvisual/api/internal/service/permission"
+	"github.com/clickvisual/clickvisual/api/internal/service/shorturl"
 	"github.com/clickvisual/clickvisual/api/internal/service/user"
 	"github.com/clickvisual/clickvisual/api/pkg/preempt"
 )
@@ -46,7 +47,7 @@ func Init() error {
 		Dependence.Sync()
 	})
 	xgo.Go(func() {
-		ShortURLClean()
+		shorturl.ShortURLClean()
 	})
 
 	Node = NewNode()

api/internal/service/inquiry/clickhouse.go

@@ -1005,7 +1005,6 @@ func (c *ClickHouseX) isReplica() bool {
 
 // CreateStorage create default stream data table and view
 func (c *ClickHouseX) CreateStorage(did int, database db.BaseDatabase, ct view.ReqStorageCreate) (dStreamSQL, dDataSQL, dViewSQL, dDistributedSQL string, err error) {
-
 	if ct.CreateType == constx.TableCreateTypeJSONAsString {
 		// 采用 core 的新流程
 		// 创建 storer -> reader -> switcher

api/internal/service/shorturl/shorturl.go

@@ -0,0 +1,60 @@
+package shorturl
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/gotomicro/ego/core/econf"
+	"github.com/gotomicro/ego/core/elog"
+	"github.com/pkg/errors"
+
+	"github.com/clickvisual/clickvisual/api/internal/invoker"
+	"github.com/clickvisual/clickvisual/api/pkg/model/db"
+)
+
+func hashIDGenCode(id int) string {
+	ret, err := invoker.HashId.EncodeInt64([]int64{int64(id)})
+	if err != nil {
+		elog.Error("gen error", elog.FieldErr(err))
+	}
+	return ret
+}
+
+func ShortURLClean() {
+	for {
+		time.Sleep(time.Minute * 10)
+		db.ShortURLDelete30Days()
+	}
+}
+
+func GenShortURL(ur string) (string, error) {
+	u, err := url.Parse(ur)
+	if err != nil {
+		return "", errors.New(err.Error())
+	}
+	v := u.Query()
+	v.Set("tab", "custom")
+	u2 := fmt.Sprintf("%s://%s%s?%s", u.Scheme, u.Host, u.Path, v.Encode())
+	shortUrl := db.BaseShortURL{
+		OriginUrl: u2,
+		SCode:     "",
+		CallCnt:   0,
+	}
+	tx := invoker.Db.Begin()
+	if err = db.ShortURLCreate(tx, &shortUrl); err != nil {
+		tx.Rollback()
+		return "", errors.Wrap(err, "ShortURLCreate short url error")
+	}
+	sCode := hashIDGenCode(shortUrl.ID)
+	if err = db.ShortURLUpdate(tx, shortUrl.ID, map[string]interface{}{"s_code": sCode}); err != nil {
+		tx.Rollback()
+		return "", errors.Wrap(err, "ShortURLUpdate short url error")
+	}
+	if err = tx.Commit().Error; err != nil {
+		return "", errors.Wrap(err, "tx commit error")
+	}
+	rootUrl := strings.TrimSuffix(econf.GetString("app.rootURL"), "/")
+	return fmt.Sprintf("%s/api/share/%s", rootUrl, sCode), nil
+}

api/internal/service/storage.go

@@ -142,7 +142,7 @@ func (s *srvStorage) CreateByILogtailTemplate(uid int, databaseInfo db.BaseDatab
 	}
 	cp.Topics = param.Topic
 	cp.TableName = param.Name
-	if err = s.createByEgoTemplateItem(uid, databaseInfo, cp); err != nil {
+	if err = s.createByILogtailTemplateItem(uid, databaseInfo, cp); err != nil {
 		return err
 	}
 	return
@@ -185,6 +185,26 @@ func (s *srvStorage) CreateByEgoTemplate(uid int, databaseInfo db.BaseDatabase,
 	return
 }
 
+func (s *srvStorage) createByILogtailTemplateItem(uid int, databaseInfo db.BaseDatabase, param view.ReqStorageCreate) (err error) {
+	// Detection is whether it has been created
+	conds := egorm.Conds{}
+	conds["did"] = databaseInfo.ID
+	conds["name"] = param.TableName
+	tableInfo, _ := db.TableInfoX(invoker.Db, conds)
+	if tableInfo.ID != 0 {
+		return nil
+	}
+	table, err := StorageCreate(uid, databaseInfo, param)
+	if err != nil {
+		return
+	}
+	err = AnalysisFieldsUpdate(table.ID, templateTableAnalysisField[table.Name])
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (s *srvStorage) createByEgoTemplateItem(uid int, databaseInfo db.BaseDatabase, param view.ReqStorageCreate) (err error) {
 	// Detection is whether it has been created
 	conds := egorm.Conds{}

api/internal/service/table.go

@@ -82,7 +82,7 @@ func StorageCreate(uid int, databaseInfo db.BaseDatabase, param view.ReqStorageC
 	}
 	s, d, v, a, err := op.CreateStorage(databaseInfo.ID, databaseInfo, param)
 	if err != nil {
-		err = errors.Wrap(err, "create failed 01:")
+		err = errors.Wrap(err, "storage create failed")
 		return
 	}
 	tableInfo = db.BaseTable{

api/pkg/model/view/storage.go

@@ -46,9 +46,8 @@ type ReqCreateStorageByTemplateEgo struct {
 type ReqCreateStorageByTemplateILogtail struct {
 	Brokers    string `form:"brokers" binding:"required"`
 	DatabaseId int    `form:"databaseId" binding:"required"`
-
-	Name  string `form:"name" binding:"required"`
-	Topic string `form:"topic" binding:"required"`
+	Name       string `form:"name" binding:"required"`
+	Topic      string `form:"topic" binding:"required"`
 }
 
 func (r *ReqStorageCreate) GetRawLogField() string {