diff --git a/controllers/pkg/database/interface.go b/controllers/pkg/database/interface.go index 5e32035daee..d1edbeccb26 100644 --- a/controllers/pkg/database/interface.go +++ b/controllers/pkg/database/interface.go @@ -14,6 +14,7 @@ type Interface interface { GetBillingLastUpdateTime(owner string, _type accountv1.Type) (bool, time.Time, error) SaveBillingsWithAccountBalance(accountBalanceSpec *accountv1.AccountBalanceSpec) error QueryBillingRecords(billingRecordQuery *accountv1.BillingRecordQuery, owner string) error + GetBillingCount(accountType accountv1.Type) (count, amount int64, err error) GetUpdateTimeForCategoryAndPropertyFromMetering(category string, property string) (time.Time, error) GetAllPricesMap() (map[string]common.Price, error) GenerateMeteringData(startTime, endTime time.Time, prices map[string]common.Price) error diff --git a/controllers/pkg/database/mongodb.go b/controllers/pkg/database/mongodb.go index bb8d9770efb..cc4bdddcb72 100644 --- a/controllers/pkg/database/mongodb.go +++ b/controllers/pkg/database/mongodb.go @@ -489,6 +489,33 @@ func (m *MongoDB) QueryBillingRecords(billingRecordQuery *accountv1.BillingRecor return nil } +func (m *MongoDB) GetBillingCount(accountType accountv1.Type) (count, amount int64, err error) { + filter := bson.M{"type": accountType} + cursor, err := m.getBillingCollection().Find(context.Background(), filter) + if err != nil { + return 0, 0, err + } + defer cursor.Close(context.Background()) + var accountBalanceList []AccountBalanceSpecBSON + err = cursor.All(context.Background(), &accountBalanceList) + if err != nil { + return 0, 0, fmt.Errorf("failed to decode all billing record: %w", err) + } + for i := range accountBalanceList { + count++ + amount += accountBalanceList[i].Amount + } + //for cursor.Next(context.Background()) { + // var accountBalance AccountBalanceSpecBSON + // if err := cursor.Decode(&accountBalance); err != nil { + // return 0, 0, err + // } + // count++ + // amount += accountBalance.Amount + //} + return +} + func (m *MongoDB) getMeteringCollection() *mongo.Collection { return m.Client.Database(m.DBName).Collection(m.MeteringConn) } diff --git a/metrics/Dockerfile b/metrics/Dockerfile new file mode 100644 index 00000000000..38dead528ce --- /dev/null +++ b/metrics/Dockerfile @@ -0,0 +1,7 @@ +FROM gcr.io/distroless/static:nonroot + +WORKDIR / +USER 65532:65532 + +COPY crd_exporter /crd_exporter +ENTRYPOINT ["/crd_exporter"] \ No newline at end of file diff --git a/metrics/Makefile b/metrics/Makefile new file mode 100644 index 00000000000..5a1c2e5a04e --- /dev/null +++ b/metrics/Makefile @@ -0,0 +1,45 @@ +# 定义变量 +DOCKER_IMAGE_NAME = registry.cn-hangzhou.aliyuncs.com/bxy4543/user-exporter +BUILD_TOOL ?= docker +DOCKER_TAG ?= dev +#KUBECONFIG = ${HOME}/.kube/config + +# 默认目标：help +.DEFAULT_GOAL := help + +# 显示帮助信息 +.PHONY: help +help: + @echo "Makefile for CRD Exporter" + @echo "Available commands:" + @echo " make build Build the exporter binary and Docker image" + @echo " make push Push the Docker image to the repository" + @echo " make deploy Deploy the exporter to a Kubernetes cluster" + @echo " make undeploy Remove the exporter from the Kubernetes cluster" + @echo " make clean Clean the build artifacts" + +# 构建 exporter 二进制文件和 Docker 镜像 +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux go build -o crd_exporter . + $(BUILD_TOOL) build -t $(DOCKER_IMAGE_NAME):$(DOCKER_TAG) . + +# 将 Docker 镜像推送到存储库 +.PHONY: push +push: + $(BUILD_TOOL) push $(DOCKER_IMAGE_NAME):$(DOCKER_TAG) + +# 部署 exporter 到 Kubernetes 集群 +.PHONY: deploy +deploy: + sed "s//$(DOCKER_IMAGE_NAME):$(DOCKER_TAG)/" deploy.yaml | kubectl --kubeconfig=$(KUBECONFIG) apply -f - + +# 从 Kubernetes 集群中删除 exporter +.PHONY: undeploy +undeploy: + sed "s//$(DOCKER_IMAGE_NAME):$(DOCKER_TAG)/" deploy.yaml | kubectl --kubeconfig=$(KUBECONFIG) delete -f - + +# 清理构建产物 +.PHONY: clean +clean: + rm -f crd_exporter \ No newline at end of file diff --git a/metrics/deploy.yaml b/metrics/deploy.yaml new file mode 100644 index 00000000000..a2e839295ae --- /dev/null +++ b/metrics/deploy.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crd-exporter + namespace: sealos +spec: + replicas: 1 + selector: + matchLabels: + app: crd-exporter + template: + metadata: + labels: + app: crd-exporter + spec: + serviceAccountName: crd-exporter-sa + containers: + - name: crd-exporter + image: registry.cn-hangzhou.aliyuncs.com/bxy4543/user-exporter:v0.0.1 + imagePullPolicy: Always + ports: + - name: metrics + containerPort: 8000 + envFrom: + - secretRef: + name: metrics-secret +--- +apiVersion: v1 +kind: Service +metadata: + name: crd-exporter + namespace: sealos + labels: + app: crd-exporter +spec: + selector: + app: crd-exporter + ports: + - protocol: TCP + port: 8000 + targetPort: metrics + name: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: crd-exporter + namespace: sealos + labels: + release: prometheus +spec: + selector: + matchLabels: + app: crd-exporter + endpoints: + - port: metrics + interval: 300s +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: crd-exporter-sa + namespace: sealos +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: crd-exporter-clusterrole +rules: + - apiGroups: + - user.sealos.io + resources: + - users + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: crd-exporter-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: crd-exporter-clusterrole +subjects: + - kind: ServiceAccount + name: crd-exporter-sa + namespace: sealos \ No newline at end of file diff --git a/metrics/metrics-secret.yaml b/metrics/metrics-secret.yaml new file mode 100644 index 00000000000..7f6b87ffa2a --- /dev/null +++ b/metrics/metrics-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: metrics-secret + namespace: sealos +stringData: + MONGO_URI: "METRICS_MONGO_URI" \ No newline at end of file diff --git a/metrics/user.go b/metrics/user.go new file mode 100644 index 00000000000..1a920720241 --- /dev/null +++ b/metrics/user.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "encoding/json" + v12 "github.com/labring/sealos/controllers/account/api/v1" + "github.com/labring/sealos/controllers/pkg/database" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "log" + "net/http" + "os" + ctrl "sigs.k8s.io/controller-runtime" + "strconv" + "strings" + "time" +) + +var ( + userCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "user_info_total", + Help: "Total number of User CRs", + }) + userPodCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "user_pod_total", + Help: "Total number of User Pods", + }) + userRechargeCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "user_recharge_count", + Help: "Total number of user recharge transactions", + }) + + userRechargeAmount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "user_recharge_amount", + Help: "Total amount of user recharge transactions", + }) +) + +var UserPodCountInterval = 600 +var UserCountInterval = 600 + +func init() { + prometheus.MustRegister(userCount) + prometheus.MustRegister(userPodCount) + prometheus.MustRegister(userRechargeCount) + prometheus.MustRegister(userRechargeAmount) + var err error + if os.Getenv("USER_COUNT_INTERVAL") != "" { + UserCountInterval, err = strconv.Atoi(os.Getenv("USER_COUNT_INTERVAL")) + if err != nil { + log.Fatalf("USER_COUNT_INTERVAL must be a number") + } + } + if os.Getenv("USER_POD_COUNT_INTERVAL") != "" { + UserPodCountInterval, err = strconv.Atoi(os.Getenv("USER_POD_COUNT_INTERVAL")) + if err != nil { + log.Fatalf("USER_POD_COUNT_INTERVAL must be a number") + } + } +} + +func main() { + http.Handle("/metrics", promhttp.Handler()) + + go func() { + for { + userCount.Set(getUserCount()) + count, amount := getUserRechargeCountAndAmount() + userRechargeCount.Set(float64(count)) + log.Println("userRechargeAmount", amount) + log.Println("userRechargeAmount", float64(amount)/1_000_000) + userRechargeAmount.Set(float64(amount) / 1_000_000) + userPodCount.Set(getUserPodCount()) + time.Sleep(time.Duration(UserCountInterval) * time.Second) + } + }() + + log.Fatal(http.ListenAndServe(":8000", nil)) +} + +func getUserCount() float64 { + clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if err != nil { + log.Fatalf("Failed to create Kubernetes clientset: %v", err) + } + group := "user.sealos.io" + version := "v1" + plural := "users" + + userList, err := clientset.RESTClient().Get(). + AbsPath("/apis", group, version, plural).DoRaw(context.Background()) + + if err != nil { + log.Printf("Failed to get User CRD list: %v", err) + return 0 + } + + var userCRDList map[string]interface{} + if err := json.Unmarshal(userList, &userCRDList); err != nil { + log.Printf("Failed to unmarshal User CRD list: %v", err) + return 0 + } + + items, ok := userCRDList["items"].([]interface{}) + if !ok { + log.Printf("Failed to extract items from User CRD list") + return 0 + } + return float64(len(items)) +} + +func getUserPodCount() float64 { + clientset, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if err != nil { + log.Fatalf("Failed to create Kubernetes clientset: %v", err) + } + + podList, err := clientset.CoreV1().Pods("").List(context.Background(), v1.ListOptions{}) + if err != nil { + log.Printf("Failed to get pod list: %v", err) + return 0 + } + + var totalPods int64 + for _, pod := range podList.Items { + if strings.HasPrefix(pod.Namespace, "ns-") { + totalPods++ + } + } + + return float64(totalPods) +} + +func getUserRechargeCountAndAmount() (int64, int64) { + dbCtx := context.Background() + dbClient, err := database.NewMongoDB(dbCtx, os.Getenv(database.MongoURL)) + if err != nil { + log.Fatalf("connect mongo client failed: %v", err) + return 0, 0 + } + defer func() { + err := dbClient.Disconnect(dbCtx) + if err != nil { + log.Fatalf("disconnect mongo client failed: %v", err) + } + }() + + count, amount, err := dbClient.GetBillingCount(v12.Recharge) + if err != nil { + log.Fatalf("get billing count failed: %v", err) + } + return count, amount +}