diff --git a/.github/workflows/controllers.yml b/.github/workflows/controllers.yml index 97adce75396..35aea28200d 100644 --- a/.github/workflows/controllers.yml +++ b/.github/workflows/controllers.yml @@ -97,7 +97,6 @@ jobs: - { name: license, path: license } - { name: job-init, path: job/init } - { name: resources, path: resources } - - { name: resources-metering, path: resources/metering } - { name: node, path: node } - { name: admission, path: admission } steps: @@ -199,7 +198,6 @@ jobs: - { name: license, path: license } - { name: job-init, path: job/init } - { name: resources, path: resources } - - { name: resources-metering, path: resources/metering } - { name: node, path: node } - { name: admission, path: admission } steps: diff --git a/controllers/pkg/database/interface.go b/controllers/pkg/database/interface.go index 1258cf5cb07..2e5245d811c 100644 --- a/controllers/pkg/database/interface.go +++ b/controllers/pkg/database/interface.go @@ -24,9 +24,7 @@ import ( type Interface interface { //InitDB() error - GetMeteringOwnerTimeResult(queryTime time.Time, queryCategories, queryProperties []string) (*MeteringOwnerTimeResult, error) GetBillingLastUpdateTime(owner string, _type accountv1.Type) (bool, time.Time, error) - //TODO will delete this GetBillingHistoryNamespaceList(ns *accountv1.NamespaceBillingHistorySpec, owner string) ([]string, error) GetBillingHistoryNamespaces(startTime, endTime *time.Time, billType int, owner string) ([]string, error) SaveBillings(billing ...*resources.Billing) error @@ -38,8 +36,6 @@ type Interface interface { InitDefaultPropertyTypeLS() error SavePropertyTypes(types []resources.PropertyType) error GetBillingCount(accountType accountv1.Type, startTime, endTime time.Time) (count, amount int64, err error) - //TODO delete - GenerateMeteringData(startTime, endTime time.Time, prices map[string]resources.Price) error GenerateBillingData(startTime, endTime time.Time, prols *resources.PropertyTypeLS, namespaces []string, owner string) (orderID []string, amount int64, err error) InsertMonitor(ctx context.Context, monitors ...*resources.Monitor) error DropMonitorCollectionsOlderThan(days int) error @@ -51,7 +47,6 @@ type Creator interface { CreateBillingIfNotExist() error //suffix by day, eg: monitor_20200101 CreateMonitorTimeSeriesIfNotExist(collTime time.Time) error - CreateMeteringTimeSeriesIfNotExist() error } type MeteringOwnerTimeResult struct { diff --git a/controllers/pkg/database/mongodb.go b/controllers/pkg/database/mongodb.go index 66625f6052e..921af925c01 100644 --- a/controllers/pkg/database/mongodb.go +++ b/controllers/pkg/database/mongodb.go @@ -235,64 +235,6 @@ func (m *MongoDB) SaveBillings(billing ...*resources.Billing) error { return err } -func (m *MongoDB) GetMeteringOwnerTimeResult(queryTime time.Time, queryCategories, queryProperties []string) (*MeteringOwnerTimeResult, error) { - matchValue := bson.M{ - "time": queryTime, - "category": bson.M{"$in": queryCategories}, - } - if len(queryProperties) > 0 { - matchValue["property"] = bson.M{"$in": queryProperties} - } - pipeline := bson.A{ - bson.D{{Key: "$match", Value: matchValue}}, - bson.D{{Key: "$group", Value: bson.M{ - "_id": bson.M{"property": "$property"}, - "propertyTotal": bson.M{"$sum": "$amount"}, - }}}, - bson.D{{Key: "$project", Value: bson.M{ - "_id": 0, - "property": "$_id.property", - "propertyTotal": 1, - }}}, - bson.D{{Key: "$group", Value: bson.M{ - "_id": nil, - "amountTotal": bson.M{"$sum": "$propertyTotal"}, - "costs": bson.M{"$push": bson.M{"k": "$property", "v": "$propertyTotal"}}, - }}}, - bson.D{{Key: "$addFields", Value: bson.M{ - //"owner": queryOwner, - "time": queryTime, - "amount": "$amountTotal", - "costs": bson.M{"$arrayToObject": "$costs"}, - }}}, - } - - /* - db.metering.aggregate([ - { $match: - { time: queryTime, category: - { $in: ["ns-gxqoxr8s"] }, property: { $in: ["cpu", "memory", "storage"] } } }, - { $group: { _id: { property: "$property" }, propertyTotal: { $sum: "$amount" } } }, - { $project: { _id: 0, property: "$_id.property", propertyTotal: 1 } }, - { $group: { _id: null, amountTotal: { $sum: "$propertyTotal" }, costs: { $push: { k: "$property", v: "$propertyTotal" } } } }, - { $addFields: { orderId: "111111111", own: queryOwn, time: queryTime, type: 0, amount: "$amountTotal", costs: { $arrayToObject: "$costs" } } }, - { $out: "results1" }]); - */ - cursor, err := m.getMeteringCollection().Aggregate(context.Background(), pipeline) - if err != nil { - return nil, err - } - defer cursor.Close(context.Background()) - if cursor.Next(context.Background()) { - var result MeteringOwnerTimeResult - if err = cursor.Decode(&result); err != nil { - return nil, err - } - return &result, nil - } - return nil, nil -} - // InsertMonitor insert monitor data to mongodb collection monitor + time (eg: monitor_20200101) // The monitor data is saved daily 2020-12-01 00:00:00 - 2020-12-01 23:59:59 => monitor_20201201 func (m *MongoDB) InsertMonitor(ctx context.Context, monitors ...*resources.Monitor) error { @@ -990,11 +932,6 @@ func (m *MongoDB) CreateMonitorTimeSeriesIfNotExist(collTime time.Time) error { return m.CreateTimeSeriesIfNotExist(m.DBName, m.getMonitorCollectionName(collTime)) } -// CreateMeteringTimeSeriesIfNotExist creates the time series table for metering -func (m *MongoDB) CreateMeteringTimeSeriesIfNotExist() error { - return m.CreateTimeSeriesIfNotExist(m.DBName, m.MeteringConn) -} - func (m *MongoDB) CreateTimeSeriesIfNotExist(dbName, collectionName string) error { // Check if the collection already exists if exist, err := m.collectionExist(dbName, collectionName); exist || err != nil { diff --git a/controllers/pkg/database/mongodb_test.go b/controllers/pkg/database/mongodb_test.go index fd474597351..ec8d43cdc49 100644 --- a/controllers/pkg/database/mongodb_test.go +++ b/controllers/pkg/database/mongodb_test.go @@ -33,30 +33,6 @@ import ( accountv1 "github.com/labring/sealos/controllers/account/api/v1" ) -func TestMongoDB_GetMeteringOwnerTimeResult(t *testing.T) { - dbCTX := context.Background() - - //"mongodb://192.168.64.21:27017/" - m, err := NewMongoDB(dbCTX, os.Getenv("MONGODB_URI")) - if err != nil { - t.Errorf("failed to connect mongo: error = %v", err) - } - defer func() { - if err = m.Disconnect(dbCTX); err != nil { - t.Errorf("failed to disconnect mongo: error = %v", err) - } - }() - - now := time.Now() - queryTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.Local).UTC().Add(-time.Hour) - - got, err := m.GetMeteringOwnerTimeResult(queryTime, []string{"ns-0nfm8mkr"}, nil) - if err != nil { - t.Errorf("failed to get metering owner time result: error = %v", err) - } - t.Logf("all properties got: %v", got) -} - func TestMongoDB_QueryBillingRecords(t *testing.T) { dbCTX := context.Background() diff --git a/controllers/resources/metering/Dockerfile b/controllers/resources/metering/Dockerfile deleted file mode 100644 index 9903dafd741..00000000000 --- a/controllers/resources/metering/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright © 2023 sealos. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM gcr.io/distroless/static:nonroot -ARG TARGETARCH - -WORKDIR / -USER 65532:65532 - -COPY bin/controller-resources-metering-$TARGETARCH /manager -ENTRYPOINT ["/manager", "start"] \ No newline at end of file diff --git a/controllers/resources/metering/Makefile b/controllers/resources/metering/Makefile deleted file mode 100644 index 41fd5d68c23..00000000000 --- a/controllers/resources/metering/Makefile +++ /dev/null @@ -1,50 +0,0 @@ - -# Image URL to use all building/pushing image targets -IMG ?= ghcr.io/labring/sealos-resources-controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.23 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# This is a requirement for 'setup-envtest.sh' in the test target. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ Build - -.PHONY: build -build: ## Build manager binary. - CGO_ENABLED=0 GOOS=linux go build $(shell [ -n "${CRYPTOKEY}" ] && echo "-ldflags '-X github.com/labring/sealos/controllers/pkg/crypto.encryptionKey=${CRYPTOKEY} -X github.com/labring/sealos/controllers/pkg/database.cryptoKey=${CRYPTOKEY}'") -o bin/manager main.go - -.PHONY: run -run: ## Run a controller from your host. - go run ./main.go - -.PHONY: docker-build -docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - docker push ${IMG} - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) diff --git a/controllers/resources/metering/deploy/Kubefile b/controllers/resources/metering/deploy/Kubefile deleted file mode 100644 index 98b2f0b289e..00000000000 --- a/controllers/resources/metering/deploy/Kubefile +++ /dev/null @@ -1,12 +0,0 @@ -FROM scratch - -USER 65532:65532 - -COPY registry registry -COPY manifests manifests - -ENV DEFAULT_NAMESPACE resources-system -ENV MONGO_URI "mongodb://mongo:27017/resources" - - -CMD ["kubectl apply -f manifests/deploy.yaml -n $DEFAULT_NAMESPACE && ( kubectl create -f manifests/mongo-secret.yaml -n $DEFAULT_NAMESPACE || true )"] \ No newline at end of file diff --git a/controllers/resources/metering/deploy/manifests/deploy.yaml b/controllers/resources/metering/deploy/manifests/deploy.yaml deleted file mode 100644 index e587008c4da..00000000000 --- a/controllers/resources/metering/deploy/manifests/deploy.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright © 2023 sealos. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: resources-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - control-plane: metering-manager - name: resources-metering-manager - namespace: resources-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: metering-manager - template: - metadata: - labels: - control-plane: metering-manager - spec: - containers: - - image: ghcr.io/labring/sealos-resources-metering-controller:latest - name: resource-metering - command: - - "/manager" - - "start" - - "--show-path" - - "--debug" - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - env: - - name: MONGO_URI - valueFrom: - secretKeyRef: - name: mongo-secret - key: MONGO_URI - imagePullPolicy: Always diff --git a/controllers/resources/metering/deploy/manifests/mongo-secret.yaml.tmpl b/controllers/resources/metering/deploy/manifests/mongo-secret.yaml.tmpl deleted file mode 100644 index 1d3e2b640c8..00000000000 --- a/controllers/resources/metering/deploy/manifests/mongo-secret.yaml.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: mongo-secret - namespace: {{ .DEFAULT_NAMESPACE }} -stringData: - MONGO_URI: "{{ .MONGO_URI }}" \ No newline at end of file diff --git a/controllers/resources/metering/main.go b/controllers/resources/metering/main.go deleted file mode 100644 index e0acd01b931..00000000000 --- a/controllers/resources/metering/main.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright © 2023 sealos. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "syscall" - "time" - - "github.com/spf13/cobra" - - "github.com/labring/sealos/controllers/pkg/database" - "github.com/labring/sealos/controllers/pkg/resources" - "github.com/labring/sealos/controllers/pkg/utils/env" - "github.com/labring/sealos/controllers/pkg/utils/flags" - "github.com/labring/sealos/controllers/pkg/utils/logger" - "github.com/labring/sealos/controllers/pkg/utils/retry" -) - -type Config struct { - // mongodb connect url - MongoConnectURI string - MongoUsername string - MongoPassword string - RetentionDay int64 - PermanentRetention bool - // interval of metering resources - Interval time.Duration -} - -var config *Config - -func ResourcesMetering() { - ctx, cancel := context.WithCancel(context.Background()) - go func() { - // 信号监听 - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - <-sigCh - - logger.Info("Receiving signal, canceling...") - cancel() - }() - - // 在启动时执行任务 - logger.Info("start to execute metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05")) - if err := PreApply(); err != nil { - logger.Error(err.Error()) - os.Exit(1) - } - logger.Info("success first metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05")) - - go func() { - for { - // start execution beginning of every hour - durationToNextHour := time.Until(time.Now().Truncate(time.Hour).Add(time.Hour)) - - select { - case <-time.After(durationToNextHour): - go func() { - logger.Debug("Start to execute metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05")) - if err := retry.Retry(5, time.Minute, executeTask); err != nil { - logger.Error("Failed to execute task: %v", err) - return - } - logger.Debug("Success metering resources, time: %v", time.Now().Format("2006-01-02 15:04:05")) - }() - case <-ctx.Done(): - return - } - } - }() - if !config.PermanentRetention { - ticker := time.NewTicker(24 * time.Hour) - go func() { - for range ticker.C { - if err := DropMonitorCollectionOlder(); err != nil { - logger.Error("Failed to drop monitor collection older than %d days: %v", config.RetentionDay, err) - } - } - }() - } - - <-ctx.Done() - logger.Info("program exit") -} - -func DropMonitorCollectionOlder() error { - dbCtx := context.Background() - dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI) - if err != nil { - return fmt.Errorf("connect mongo client failed: %v", err) - } - defer func() { - err := dbClient.Disconnect(dbCtx) - if err != nil { - logger.Error("disconnect mongo client failed: %v", err) - } - }() - return dbClient.DropMonitorCollectionsOlderThan(int(config.RetentionDay)) -} - -func PreApply() error { - switch { - case config.MongoConnectURI == "": - return fmt.Errorf("mongo connect url is empty") - } - dbCtx := context.Background() - dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI) - if err != nil { - return fmt.Errorf("connect mongo client failed: %v", err) - } - defer func() { - err := dbClient.Disconnect(dbCtx) - if err != nil { - logger.Error("disconnect mongo client failed: %v", err) - } - }() - - if err = dbClient.CreateMeteringTimeSeriesIfNotExist(); err != nil { - logger.Warn("create metering time series failed: %v", err) - } - if err = CreateMonitorTimeSeries(dbClient, time.Now().UTC()); err != nil { - logger.Warn("create monitor time series failed: %v", err) - } - return executeTask() -} - -func executeTask() error { - //opts := options.Client().ApplyURI("mongodb://192.168.64.17:27017") - dbCtx := context.Background() - dbClient, err := database.NewMongoDB(dbCtx, config.MongoConnectURI) - if err != nil { - return fmt.Errorf("connect mongo client failed: %v", err) - } - defer func() { - err := dbClient.Disconnect(dbCtx) - if err != nil { - logger.Warn("disconnect mongo client failed: %v", err) - } - }() - prices, err := dbClient.GetAllPricesMap() - if err != nil { - logger.Error("failed to get all prices map: %v", err) - } - //prices is empty, use default price - if len(prices) == 0 || err != nil { - prices = resources.DefaultPrices - } - now := time.Now().UTC() - startTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour()-1, 0, 0, 0, time.UTC) - endTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC) - if err := dbClient.GenerateMeteringData(startTime, endTime, prices); err != nil { - return fmt.Errorf("failed to generate metering data: %v", err) - } - // create tomorrow monitor time series - if err := CreateMonitorTimeSeries(dbClient, now.Add(24*time.Hour)); err != nil { - logger.Debug("failed to create monitor time series: %v", err) - } - return nil -} - -func CreateMonitorTimeSeries(dbClient database.Interface, collTime time.Time) error { - return dbClient.CreateMonitorTimeSeriesIfNotExist(collTime) -} - -func main() { - Execute() -} - -var ( - debug bool - showPath bool -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "metering", - Short: "program for sealos resources metering.", - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func newStartCmd() *cobra.Command { - //type startFlag struct { - //} - //var flag startFlag - - startCmd := &cobra.Command{ - Use: "start", - Short: "start metering", - Long: `start metering`, - Run: func(cmd *cobra.Command, args []string) { - flags.PrintFlags(cmd.Flags()) - ResourcesMetering() - }, - } - - return startCmd -} - -func init() { - cobra.OnInitialize(func() { - logger.CfgConsoleLogger(debug, showPath) - }) - - rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug logger") - rootCmd.PersistentFlags().BoolVar(&showPath, "show-path", false, "enable show code path") - rootCmd.AddCommand(newStartCmd()) - config = &Config{ - MongoConnectURI: os.Getenv(database.MongoURI), - MongoUsername: os.Getenv(database.MongoUsername), - MongoPassword: os.Getenv(database.MongoPassword), - RetentionDay: env.GetInt64EnvWithDefault(database.RetentionDay, database.DefaultRetentionDay), - PermanentRetention: os.Getenv(database.PermanentRetention) == "true", - } -}