feat: user info metrics

controllers/pkg/database/interface.go

@@ -14,6 +14,7 @@ type Interface interface {
 	GetBillingLastUpdateTime(owner string, _type accountv1.Type) (bool, time.Time, error)
 	SaveBillingsWithAccountBalance(accountBalanceSpec *accountv1.AccountBalanceSpec) error
 	QueryBillingRecords(billingRecordQuery *accountv1.BillingRecordQuery, owner string) error
+	GetBillingCount(accountType accountv1.Type) (count, amount int64, err error)
 	GetUpdateTimeForCategoryAndPropertyFromMetering(category string, property string) (time.Time, error)
 	GetAllPricesMap() (map[string]common.Price, error)
 	GenerateMeteringData(startTime, endTime time.Time, prices map[string]common.Price) error

controllers/pkg/database/mongodb.go

@@ -489,6 +489,33 @@ func (m *MongoDB) QueryBillingRecords(billingRecordQuery *accountv1.BillingRecor
 	return nil
 }
 
+func (m *MongoDB) GetBillingCount(accountType accountv1.Type) (count, amount int64, err error) {
+	filter := bson.M{"type": accountType}
+	cursor, err := m.getBillingCollection().Find(context.Background(), filter)
+	if err != nil {
+		return 0, 0, err
+	}
+	defer cursor.Close(context.Background())
+	var accountBalanceList []AccountBalanceSpecBSON
+	err = cursor.All(context.Background(), &accountBalanceList)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to decode all billing record: %w", err)
+	}
+	for i := range accountBalanceList {
+		count++
+		amount += accountBalanceList[i].Amount
+	}
+	//for cursor.Next(context.Background()) {
+	//	var accountBalance AccountBalanceSpecBSON
+	//	if err := cursor.Decode(&accountBalance); err != nil {
+	//		return 0, 0, err
+	//	}
+	//	count++
+	//	amount += accountBalance.Amount
+	//}
+	return
+}
+
 func (m *MongoDB) getMeteringCollection() *mongo.Collection {
 	return m.Client.Database(m.DBName).Collection(m.MeteringConn)
 }

metrics/Dockerfile

@@ -0,0 +1,7 @@
+FROM gcr.io/distroless/static:nonroot
+
+WORKDIR /
+USER 65532:65532
+
+COPY crd_exporter /crd_exporter
+ENTRYPOINT ["/crd_exporter"]

metrics/Makefile

@@ -0,0 +1,45 @@
+# 定义变量
+DOCKER_IMAGE_NAME = registry.cn-hangzhou.aliyuncs.com/bxy4543/user-exporter
+BUILD_TOOL ?= docker
+DOCKER_TAG ?= dev
+#KUBECONFIG = ${HOME}/.kube/config
+
+# 默认目标：help
+.DEFAULT_GOAL := help
+
+# 显示帮助信息
+.PHONY: help
+help:
+	@echo "Makefile for CRD Exporter"
+	@echo "Available commands:"
+	@echo "  make build         Build the exporter binary and Docker image"
+	@echo "  make push          Push the Docker image to the repository"
+	@echo "  make deploy        Deploy the exporter to a Kubernetes cluster"
+	@echo "  make undeploy      Remove the exporter from the Kubernetes cluster"
+	@echo "  make clean         Clean the build artifacts"
+
+# 构建 exporter 二进制文件和 Docker 镜像
+.PHONY: build
+build:
+	CGO_ENABLED=0 GOOS=linux go build -o crd_exporter .
+	$(BUILD_TOOL) build -t $(DOCKER_IMAGE_NAME):$(DOCKER_TAG) .
+
+# 将 Docker 镜像推送到存储库
+.PHONY: push
+push:
+	$(BUILD_TOOL) push $(DOCKER_IMAGE_NAME):$(DOCKER_TAG)
+
+# 部署 exporter 到 Kubernetes 集群
+.PHONY: deploy
+deploy:
+	sed "s/<YOUR_DOCKER_IMAGE>/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG)/" deploy.yaml | kubectl --kubeconfig=$(KUBECONFIG) apply -f -
+
+# 从 Kubernetes 集群中删除 exporter
+.PHONY: undeploy
+undeploy:
+	sed "s/<YOUR_DOCKER_IMAGE>/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG)/" deploy.yaml | kubectl --kubeconfig=$(KUBECONFIG) delete -f -
+
+# 清理构建产物
+.PHONY: clean
+clean:
+	rm -f crd_exporter

metrics/deploy.yaml

@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crd-exporter
+  namespace: sealos
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: crd-exporter
+  template:
+    metadata:
+      labels:
+        app: crd-exporter
+    spec:
+      serviceAccountName: crd-exporter-sa
+      containers:
+        - name: crd-exporter
+          image: registry.cn-hangzhou.aliyuncs.com/bxy4543/user-exporter:v0.0.1
+          imagePullPolicy: Always
+          ports:
+            - name: metrics
+              containerPort: 8000
+          envFrom:
+            - secretRef:
+                name: metrics-secret
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: crd-exporter
+  namespace: sealos
+  labels:
+    app: crd-exporter
+spec:
+  selector:
+    app: crd-exporter
+  ports:
+    - protocol: TCP
+      port: 8000
+      targetPort: metrics
+      name: metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: crd-exporter
+  namespace: sealos
+  labels:
+    release: prometheus
+spec:
+  selector:
+    matchLabels:
+      app: crd-exporter
+  endpoints:
+    - port: metrics
+      interval: 300s
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crd-exporter-sa
+  namespace: sealos
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: crd-exporter-clusterrole
+rules:
+  - apiGroups:
+      - user.sealos.io
+    resources:
+      - users
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: crd-exporter-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: crd-exporter-clusterrole
+subjects:
+  - kind: ServiceAccount
+    name: crd-exporter-sa
+    namespace: sealos

metrics/metrics-secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metrics-secret
+  namespace: sealos
+stringData:
+  MONGO_URI: "METRICS_MONGO_URI"

metrics/user.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	v12 "github.com/labring/sealos/controllers/account/api/v1"
+	"github.com/labring/sealos/controllers/pkg/database"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"log"
+	"net/http"
+	"os"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	userCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "user_info_total",
+		Help: "Total number of User CRs",
+	})
+	userPodCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "user_pod_total",
+		Help: "Total number of User Pods",
+	})
+	userRechargeCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "user_recharge_count",
+		Help: "Total number of user recharge transactions",
+	})
+
+	userRechargeAmount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "user_recharge_amount",
+		Help: "Total amount of user recharge transactions",
+	})
+)
+
+var UserPodCountInterval = 600
+var UserCountInterval = 600
+
+func init() {
+	prometheus.MustRegister(userCount)
+	prometheus.MustRegister(userPodCount)
+	prometheus.MustRegister(userRechargeCount)
+	prometheus.MustRegister(userRechargeAmount)
+	var err error
+	if os.Getenv("USER_COUNT_INTERVAL") != "" {
+		UserCountInterval, err = strconv.Atoi(os.Getenv("USER_COUNT_INTERVAL"))
+		if err != nil {
+			log.Fatalf("USER_COUNT_INTERVAL must be a number")
+		}
+	}
+	if os.Getenv("USER_POD_COUNT_INTERVAL") != "" {
+		UserPodCountInterval, err = strconv.Atoi(os.Getenv("USER_POD_COUNT_INTERVAL"))
+		if err != nil {
+			log.Fatalf("USER_POD_COUNT_INTERVAL must be a number")
+		}
+	}
+}
+
+func main() {
+	http.Handle("/metrics", promhttp.Handler())
+
+	go func() {
+		for {
+			userCount.Set(getUserCount())
+			count, amount := getUserRechargeCountAndAmount()
+			userRechargeCount.Set(float64(count))
+			log.Println("userRechargeAmount", amount)
+			log.Println("userRechargeAmount", float64(amount)/1_000_000)
+			userRechargeAmount.Set(float64(amount) / 1_000_000)
+			userPodCount.Set(getUserPodCount())
+			time.Sleep(time.Duration(UserCountInterval) * time.Second)
+		}
+	}()
+
+	log.Fatal(http.ListenAndServe(":8000", nil))
+}
+
+func getUserCount() float64 {
+	clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie())
+	if err != nil {
+		log.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+	group := "user.sealos.io"
+	version := "v1"
+	plural := "users"
+
+	userList, err := clientset.RESTClient().Get().
+		AbsPath("/apis", group, version, plural).DoRaw(context.Background())
+
+	if err != nil {
+		log.Printf("Failed to get User CRD list: %v", err)
+		return 0
+	}
+
+	var userCRDList map[string]interface{}
+	if err := json.Unmarshal(userList, &userCRDList); err != nil {
+		log.Printf("Failed to unmarshal User CRD list: %v", err)
+		return 0
+	}
+
+	items, ok := userCRDList["items"].([]interface{})
+	if !ok {
+		log.Printf("Failed to extract items from User CRD list")
+		return 0
+	}
+	return float64(len(items))
+}
+
+func getUserPodCount() float64 {
+	clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie())
+	if err != nil {
+		log.Fatalf("Failed to create Kubernetes clientset: %v", err)
+	}
+
+	podList, err := clientset.CoreV1().Pods("").List(context.Background(), v1.ListOptions{})
+	if err != nil {
+		log.Printf("Failed to get pod list: %v", err)
+		return 0
+	}
+
+	var totalPods int64
+	for _, pod := range podList.Items {
+		if strings.HasPrefix(pod.Namespace, "ns-") {
+			totalPods++
+		}
+	}
+
+	return float64(totalPods)
+}
+
+func getUserRechargeCountAndAmount() (int64, int64) {
+	dbCtx := context.Background()
+	dbClient, err := database.NewMongoDB(dbCtx, os.Getenv(database.MongoURL))
+	if err != nil {
+		log.Fatalf("connect mongo client failed: %v", err)
+		return 0, 0
+	}
+	defer func() {
+		err := dbClient.Disconnect(dbCtx)
+		if err != nil {
+			log.Fatalf("disconnect mongo client failed: %v", err)
+		}
+	}()
+
+	count, amount, err := dbClient.GetBillingCount(v12.Recharge)
+	if err != nil {
+		log.Fatalf("get billing count failed: %v", err)
+	}
+	return count, amount
+}