clickvisual · kl7sn · Mar 3, 2022 · Feb 25, 2022 · Feb 28, 2022 · Feb 28, 2022
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,8 @@ api/internal/ui/dist/*
 .umi/
 .umi-production/
 package-lock.json
+
+data/all-in-one/clickhouse/database/
+data/all-in-one/kafka/data/
+data/all-in-one/zookeeper/data/
+data/all-in-one/zookeeper/datalog/
diff --git a/api/config/resource.yaml b/api/config/resource.yaml
@@ -13,6 +13,11 @@ permission:
         path: /api/query/tables
   - name: alarm
     path: /alarm
+    children:
+      - name: rules
+        path: /alarm/rules
+      - name: notifications
+        path: /alarm/notifications
   - name: configure
     path: /configure
   - name: systemSettings

diff --git a/api/internal/apiv1/alarm/alarm.go b/api/internal/apiv1/alarm/alarm.go
@@ -1,13 +1,10 @@
 package alarm
 
 import (
-	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/google/uuid"
 	"github.com/gotomicro/ego-component/egorm"
-	"github.com/gotomicro/ego/core/elog"
 	"github.com/spf13/cast"
 
 	"github.com/shimohq/mogo/api/internal/invoker"
@@ -50,60 +47,12 @@ func Create(c *core.Context) {
 		c.JSONE(1, "alarm create failed 01: "+err.Error(), nil)
 		return
 	}
-	filtersDB, err := service.Alarm.FilterCreate(tx, obj.ID, req.Filters)
+	err = service.Alarm.CreateOrUpdate(tx, obj, req)
 	if err != nil {
 		tx.Rollback()
-		c.JSONE(1, "alarm create failed 02: "+err.Error(), nil)
-		return
-	}
-	exp, err := service.Alarm.ConditionCreate(tx, obj, req.Conditions)
-	if err != nil {
-		tx.Rollback()
-		c.JSONE(1, "alarm create failed 03: "+err.Error(), nil)
-		return
-	}
-	// table info
-	tableInfo, err := db.TableInfo(invoker.Db, tid)
-	if err != nil {
-		tx.Rollback()
-		c.JSONE(core.CodeErr, err.Error(), nil)
-		return
-	}
-	// prometheus set
-	instance, err := db.InstanceInfo(tx, tableInfo.Database.Iid)
-	if err != nil {
-		tx.Rollback()
-		c.JSONE(1, "you need to configure alarms related to the instance first: "+err.Error(), nil)
-		return
-	}
-	op, err := service.InstanceManager.Load(tableInfo.Database.Iid)
-	if err != nil {
-		tx.Rollback()
-		c.JSONE(1, "alarm create failed 04: "+err.Error(), nil)
-		return
-	}
-	// view set
-	viewSQL, err := op.AlertViewCreate(obj, filtersDB)
-	ups := make(map[string]interface{}, 0)
-	ups["view"] = viewSQL
-	err = db.AlarmUpdate(tx, obj.ID, ups)
-
-	// rule store
-	err = service.Alarm.RuleStore(instance, obj, exp)
-	if err != nil {
-		tx.Rollback()
-		c.JSONE(1, "alarm create failed 05: "+err.Error(), nil)
-		return
-	}
-	resp, errReload := http.Post(strings.TrimSuffix(instance.PrometheusTarget, "/")+"/-/reload", "text/html;charset=utf-8", nil)
-	if errReload != nil {
-		tx.Rollback()
-		elog.Error("reload", elog.Any("reload", instance.PrometheusTarget+"/-/reload"), elog.Any("err", errReload.Error()))
-		c.JSONE(1, "create failed: prometheus reload failed", nil)
+		c.JSONE(1, "alarm create failed 01: "+err.Error(), nil)
 		return
 	}
-	defer func() { _ = resp.Body.Close() }()
-
 	if err = tx.Commit().Error; err != nil {
 		tx.Rollback()
 		c.JSONE(1, "alarm create failed 06: "+err.Error(), nil)
@@ -133,52 +82,35 @@ func Update(c *core.Context) {
 	ups["uid"] = c.Uid()
 	if err := db.AlarmUpdate(tx, id, ups); err != nil {
 		tx.Rollback()
-		c.JSONE(1, "update failed: "+err.Error(), nil)
+		c.JSONE(1, "update failed 01: "+err.Error(), nil)
 		return
 	}
 	// filter
 	if err := db.AlarmFilterDeleteBatch(tx, id); err != nil {
 		tx.Rollback()
-		c.JSONE(1, "update failed: "+err.Error(), nil)
+		c.JSONE(1, "update failed 02: "+err.Error(), nil)
 		return
 	}
-	for _, filter := range req.Filters {
-		filterObj := &db.AlarmFilter{
-			AlarmId:        id,
-			When:           filter.When,
-			SetOperatorTyp: filter.SetOperatorTyp,
-			SetOperatorExp: filter.SetOperatorExp,
-		}
-		if err := db.AlarmFilterCreate(tx, filterObj); err != nil {
-			tx.Rollback()
-			c.JSONE(1, "create failed: "+err.Error(), nil)
-			return
-		}
-	}
 	// condition
 	if err := db.AlarmConditionDeleteBatch(tx, id); err != nil {
 		tx.Rollback()
-		c.JSONE(1, "update failed: "+err.Error(), nil)
+		c.JSONE(1, "update failed 03: "+err.Error(), nil)
 		return
 	}
-	for _, condition := range req.Conditions {
-		conditionObj := &db.AlarmCondition{
-			AlarmId:        id,
-			SetOperatorTyp: condition.SetOperatorTyp,
-			SetOperatorExp: condition.SetOperatorExp,
-			Cond:           condition.Cond,
-			Val1:           condition.Val1,
-			Val2:           condition.Val2,
-		}
-		if err := db.AlarmConditionCreate(tx, conditionObj); err != nil {
-			tx.Rollback()
-			c.JSONE(1, "create failed: "+err.Error(), nil)
-			return
-		}
+	obj, err := db.AlarmInfo(tx, id)
+	if err != nil {
+		tx.Rollback()
+		c.JSONE(1, "update failed 04: "+err.Error(), nil)
+		return
 	}
-	if err := tx.Commit().Error; err != nil {
+	if err = service.Alarm.CreateOrUpdate(tx, &obj, req); err != nil {
 		tx.Rollback()
-		c.JSONE(1, "create failed: "+err.Error(), nil)
+		c.JSONE(1, "update failed 05:"+err.Error(), nil)
+		return
+	}
+	if err = tx.Commit().Error; err != nil {
+		tx.Rollback()
+		c.JSONE(1, "update failed 06:"+err.Error(), nil)
 		return
 	}
 	c.JSONOK()
@@ -245,11 +177,14 @@ func Info(c *core.Context) {
 		c.JSONE(core.CodeErr, err.Error(), nil)
 		return
 	}
+	user, _ := db.UserInfo(alarmInfo.Uid)
 	res := view.ReqAlarmInfo{
 		Alarm:      alarmInfo,
 		Filters:    filters,
 		Conditions: conditions,
+		User:       user,
 	}
+	res.Password = "*"
 	c.JSONE(core.CodeOK, "succ", res)
 	return
 }

diff --git a/api/internal/apiv1/base/table.go b/api/internal/apiv1/base/table.go
@@ -295,6 +295,10 @@ func TableLogs(c *core.Context) {
 		c.JSONE(core.CodeErr, "invalid parameter: "+err.Error(), nil)
 		return
 	}
+	if param.Query == "" {
+		c.JSONE(core.CodeErr, "Query parameter error. Refer to the ClickHouse WHERE syntax. https://clickhouse.com/docs/zh/sql-reference/statements/select/where/", nil)
+		return
+	}
 	res, err := op.GET(param, tableInfo.ID)
 	if err != nil {
 		c.JSONE(core.CodeErr, "query failed: "+err.Error(), nil)

diff --git a/api/internal/service/alarm.go b/api/internal/service/alarm.go
@@ -4,12 +4,14 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"sort"
 	"strings"
 
 	"github.com/gotomicro/ego/core/elog"
 	"gorm.io/gorm"
 
+	"github.com/shimohq/mogo/api/internal/invoker"
 	"github.com/shimohq/mogo/api/internal/service/inquiry"
 	"github.com/shimohq/mogo/api/internal/service/kube"
 	"github.com/shimohq/mogo/api/internal/service/kube/resource"
@@ -101,7 +103,7 @@ func (i *alarm) ConditionCreate(tx *gorm.DB, obj *db.Alarm, conditions []view.Re
 	return
 }
 
-func (i *alarm) RuleStore(instance db.Instance, obj *db.Alarm, exp string) (err error) {
+func (i *alarm) RuleStore(instance db.Instance, obj *db.Alarm, exp string) (rule string, err error) {
 	template := `groups:
 - name: default
   rules:
@@ -113,30 +115,83 @@ func (i *alarm) RuleStore(instance db.Instance, obj *db.Alarm, exp string) (err
     annotations:
       summary: "告警 {{ $labels.name }}"
       description: "{{ $labels.desc }}  (当前值: {{ $value }})"`
-	newRule := fmt.Sprintf(template, obj.Name, exp, obj.AlertInterval())
+	rule = fmt.Sprintf(template, obj.Name, exp, obj.AlertInterval())
 	switch instance.RuleStoreType {
 	case RuleStoreTypeK8s:
 		elog.Debug("alert", elog.Any("instance", instance))
 		client, errCluster := kube.ClusterManager.GetClusterManager(instance.ClusterId)
 		if errCluster != nil {
-			return errCluster
+			return rule, errCluster
 		}
-		rule := make(map[string]string)
-		rule[obj.AlertRuleName()] = newRule
-		elog.Debug("alert", elog.Any("rule", rule))
-		err = resource.ConfigmapCreateOrUpdate(client, instance.Namespace, instance.Configmap, rule)
+		rules := make(map[string]string)
+		rules[obj.AlertRuleName()] = rule
+		elog.Debug("alert", elog.Any("rules", rules))
+		err = resource.ConfigmapCreateOrUpdate(client, instance.Namespace, instance.Configmap, rules)
 		if err != nil {
 			return
 		}
 	case RuleStoreTypeFile:
-		content := []byte(newRule)
+		content := []byte(rule)
 		path := strings.TrimSuffix(instance.FilePath, "/")
 		err = ioutil.WriteFile(path+"/"+obj.AlertRuleName(), content, 0644)
 		if err != nil {
 			return
 		}
 	default:
-		return constx.ErrAlarmRuleStoreIsClosed
+		return rule, constx.ErrAlarmRuleStoreIsClosed
 	}
+	return rule, nil
+}
+
+func (i *alarm) CreateOrUpdate(tx *gorm.DB, obj *db.Alarm, req view.ReqAlarmCreate) (err error) {
+	filtersDB, err := i.FilterCreate(tx, obj.ID, req.Filters)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm create failed 02"), elog.String("err", err.Error()))
+		return
+	}
+	exp, err := i.ConditionCreate(tx, obj, req.Conditions)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm create failed 03"), elog.String("err", err.Error()))
+		return
+	}
+	// table info
+	tableInfo, err := db.TableInfo(invoker.Db, obj.Tid)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm table info"), elog.String("err", err.Error()))
+		return
+	}
+	// prometheus set
+	instance, err := db.InstanceInfo(tx, tableInfo.Database.Iid)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "you need to configure alarms related to the instance first:"), elog.String("err", err.Error()))
+		return
+	}
+	op, err := InstanceManager.Load(tableInfo.Database.Iid)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm create failed 04"), elog.String("err", err.Error()))
+		return
+	}
+	// view set
+	viewSQL, err := op.AlertViewCreate(obj, filtersDB)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm create failed 05"), elog.String("err", err.Error()))
+		return
+	}
+	// rule store
+	rule, err := i.RuleStore(instance, obj, exp)
+	if err != nil {
+		elog.Error("alarm", elog.String("step", "alarm create failed 06"), elog.String("err", err.Error()))
+		return
+	}
+	ups := make(map[string]interface{}, 0)
+	ups["view"] = viewSQL
+	ups["alert_rule"] = rule
+	err = db.AlarmUpdate(tx, obj.ID, ups)
+	resp, errReload := http.Post(strings.TrimSuffix(instance.PrometheusTarget, "/")+"/-/reload", "text/html;charset=utf-8", nil)
+	if errReload != nil {
+		elog.Error("reload", elog.Any("reload", instance.PrometheusTarget+"/-/reload"), elog.Any("err", errReload.Error()))
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
 	return nil
 }
diff --git a/api/internal/service/inquiry/clickhouse.go b/api/internal/service/inquiry/clickhouse.go
@@ -36,9 +36,9 @@ var typORM = map[int]string{
 }
 
 var jsonExtractORM = map[int]string{
-	0: "JSONExtractString",
-	1: "JSONExtractInt",
-	2: "JSONExtractFloat",
+	0: "toString",
+	1: "toInt64OrZero",
+	2: "toFloat64OrZero",
 }
 
 type ClickHouse struct {
@@ -65,7 +65,7 @@ func (c *ClickHouse) genJsonExtractSQL(indexes map[string]*db.Index) (string, er
 	var jsonExtractSQL string
 	jsonExtractSQL = ","
 	for _, obj := range indexes {
-		jsonExtractSQL += fmt.Sprintf("%s(_log_, '%s') AS %s,", jsonExtractORM[obj.Typ], obj.Field, obj.Field)
+		jsonExtractSQL += fmt.Sprintf("%s(JSONExtractRaw(_log_, '%s')) AS %s,", jsonExtractORM[obj.Typ], obj.Field, obj.Field)
 	}
 	jsonExtractSQL = strings.TrimSuffix(jsonExtractSQL, ",")
 	return jsonExtractSQL, nil

diff --git a/api/pkg/model/db/alarm.go b/api/pkg/model/db/alarm.go
@@ -78,7 +78,7 @@ func AlarmListPage(conds egorm.Conds, reqList *ReqPage) (total int64, respList [
 		reqList.Current = 1
 	}
 	sql, binds := egorm.BuildQuery(conds)
-	db := invoker.Db.Model(Alarm{}).Where(sql, binds...)
+	db := invoker.Db.Model(Alarm{}).Where(sql, binds...).Order("utime desc")
 	db.Count(&total)
 	db.Offset((reqList.Current - 1) * reqList.PageSize).Limit(reqList.PageSize).Find(&respList)
 	return

diff --git a/api/pkg/model/view/alarm.go b/api/pkg/model/view/alarm.go
@@ -35,4 +35,5 @@ type ReqAlarmInfo struct {
 	db.Alarm
 	Filters    []*db.AlarmFilter    `json:"filters" from:"filters"`
 	Conditions []*db.AlarmCondition `json:"conditions" from:"conditions"`
+	db.User
 }
diff --git a/...ckhouse/conf/config.d/graphite_rollup.xml → ...lickhouse/conf/conf.d/graphite_rollup.xml b/...ckhouse/conf/config.d/graphite_rollup.xml → ...lickhouse/conf/conf.d/graphite_rollup.xml
@@ -1,3 +1,4 @@
+<?xml version="1.0" ?>
 <yandex>
     <graphite_rollup>
         <path_column_name>tags</path_column_name>

diff --git a/data/k8s/prometheus/prom2click-deployment.yaml b/data/k8s/prometheus/prom2click-deployment.yaml
@@ -25,7 +25,7 @@ spec:
           image: fhalim/prom2click:latest
           imagePullPolicy: IfNotPresent
           args:
-            - '-ch.dsn=tcp://127.0.0.1:8123'
+            - '-ch.dsn=tcp://127.0.0.1:9000'
             - '-ch.db=metrics'
             - '-ch.table=samples'
           command: