feat: implements all datastore methods

Signed-off-by: Neeraj Gartia <neerajgartia211002@gmail.com>

.github/workflows/server-test.yml

@@ -66,6 +66,12 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y golang-ginkgo-dev
 
+      - name : Set up MySQL
+        uses: mirromutth/mysql-action@v1.1
+        with:
+          mysql database: 'kubevela'
+          mysql root password: 'kubevelaSQL123'
+
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@d26215f71b2ce60420a2a3776a25893d11a65f85 # 1.9.0
         with:

pkg/server/domain/model/user.go

@@ -40,12 +40,12 @@ const RoleAdmin = "admin"
 // User is the model of user
 type User struct {
 	BaseModel
-	Name          string    `json:"name"`
+	Name          string    `json:"name" gorm:"primaryKey"`
 	Email         string    `json:"email"`
 	Alias         string    `json:"alias,omitempty"`
 	Password      string    `json:"password,omitempty"`
 	Disabled      bool      `json:"disabled"`
-	LastLoginTime time.Time `json:"lastLoginTime,omitempty"`
+	LastLoginTime time.Time `json:"lastLoginTime,omitempty" gorm:"default:'2020-01-01'"`
 	// UserRoles binding the platform level roles
 	UserRoles []string `json:"userRoles" gorm:"serializer:json"`
 	DexSub    string   `json:"dexSub,omitempty"`
@@ -94,8 +94,8 @@ func (u *User) IsAdmin() bool {
 // ProjectUser is the model of user in project
 type ProjectUser struct {
 	BaseModel
-	Username    string `json:"username"`
-	ProjectName string `json:"projectName"`
+	Username    string `json:"username" gorm:"primaryKey"`
+	ProjectName string `json:"projectName" gorm:"primaryKey"`
 	// UserRoles binding the project level roles
 	UserRoles []string `json:"userRoles" gorm:"serializer:json"`
 }
@@ -137,7 +137,7 @@ type CustomClaims struct {
 // Role is a model for a new RBAC mode.
 type Role struct {
 	BaseModel
-	Name        string   `json:"name"`
+	Name        string   `json:"name" gorm:"primaryKey"`
 	Alias       string   `json:"alias"`
 	Project     string   `json:"project,omitempty"`
 	Permissions []string `json:"permissions" gorm:"serializer:json"`
@@ -146,9 +146,9 @@ type Role struct {
 // Permission is a model for a new RBAC mode.
 type Permission struct {
 	BaseModel
-	Name      string   `json:"name"`
+	Name      string   `json:"name" gorm:"primaryKey"`
 	Alias     string   `json:"alias"`
-	Project   string   `json:"project,omitempty"`
+	Project   string   `json:"project,omitempty" gorm:"primaryKey"`
 	Resources []string `json:"resources" gorm:"serializer:json"`
 	Actions   []string `json:"actions" gorm:"serializer:json"`
 	// Effect option values: Allow,Deny

pkg/server/domain/model/workflow.go

@@ -116,8 +116,8 @@ type WorkflowRecord struct {
 	RevisionPrimaryKey string               `json:"revisionPrimaryKey"`
 	Name               string               `json:"name" gorm:"primaryKey"`
 	Namespace          string               `json:"namespace"`
-	StartTime          time.Time            `json:"startTime,omitempty"`
-	EndTime            time.Time            `json:"endTime,omitempty"`
+	StartTime          time.Time            `json:"startTime,omitempty" gorm:"default:'2020-01-01'"`
+	EndTime            time.Time            `json:"endTime,omitempty" gorm:"default:'2020-01-01'"`
 	Finished           string               `json:"finished"`
 	Steps              []WorkflowStepStatus `json:"steps,omitempty" gorm:"serializer:json"`
 	Status             string               `json:"status"`

pkg/server/infrastructure/datastore/mysql/mysql.go

@@ -18,13 +18,20 @@ package mysql
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
+	"time"
 
 	mysqlgorm "gorm.io/driver/mysql"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"gorm.io/gorm/logger"
+	"k8s.io/klog/v2"
 
 	"github.com/kubevela/velaux/pkg/server/domain/model"
 	"github.com/kubevela/velaux/pkg/server/infrastructure/datastore"
+	"github.com/kubevela/velaux/pkg/server/infrastructure/datastore/sqlnamer"
 )
 
 type mysql struct {
@@ -34,7 +41,10 @@ type mysql struct {
 
 // New new mongodb datastore instance
 func New(ctx context.Context, cfg datastore.Config) (datastore.DataStore, error) {
-	db, err := gorm.Open(mysqlgorm.Open(cfg.URL), &gorm.Config{})
+	db, err := gorm.Open(mysqlgorm.Open(cfg.URL), &gorm.Config{
+		NamingStrategy: sqlnamer.SQLNamer{},
+		Logger:         logger.Default.LogMode(logger.Silent),
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -54,40 +64,242 @@ func New(ctx context.Context, cfg datastore.Config) (datastore.DataStore, error)
 
 // Add add data model
 func (m *mysql) Add(ctx context.Context, entity datastore.Entity) error {
-	return fmt.Errorf("method to be implemented")
+	if entity.PrimaryKey() == "" {
+		return datastore.ErrPrimaryEmpty
+	}
+	if entity.TableName() == "" {
+		return datastore.ErrTableNameEmpty
+	}
+	entity.SetCreateTime(time.Now())
+	entity.SetUpdateTime(time.Now())
+
+	if dbAdd := m.client.Create(entity); dbAdd.Error != nil {
+		if match := errors.Is(dbAdd.Error, gorm.ErrDuplicatedKey); match {
+			return datastore.ErrRecordExist
+		}
+		return datastore.NewDBError(dbAdd.Error)
+	}
+	return nil
 }
 
 // BatchAdd batch add entity, this operation has some atomicity.
 func (m *mysql) BatchAdd(ctx context.Context, entities []datastore.Entity) error {
-	return fmt.Errorf("method to be implemented")
+	notRollback := make(map[string]int)
+	for i, saveEntity := range entities {
+		if err := m.Add(ctx, saveEntity); err != nil {
+			if errors.Is(err, datastore.ErrRecordExist) {
+				notRollback[saveEntity.PrimaryKey()] = 1
+			}
+			for _, deleteEntity := range entities[:i] {
+				if _, exit := notRollback[deleteEntity.PrimaryKey()]; !exit {
+					if err := m.Delete(ctx, deleteEntity); err != nil {
+						if !errors.Is(err, datastore.ErrRecordNotExist) {
+							klog.Errorf("rollback delete entity failure %w", err)
+						}
+					}
+				}
+			}
+			return datastore.NewDBError(fmt.Errorf("save entities occur error, %w", err))
+		}
+	}
+	return nil
 }
 
 // Get get data model
 func (m *mysql) Get(ctx context.Context, entity datastore.Entity) error {
-	return fmt.Errorf("method to be implemented")
+	if entity.PrimaryKey() == "" {
+		return datastore.ErrPrimaryEmpty
+	}
+	if entity.TableName() == "" {
+		return datastore.ErrTableNameEmpty
+	}
+
+	if dbGet := m.client.First(entity); dbGet.Error != nil {
+		if errors.Is(dbGet.Error, gorm.ErrRecordNotFound) {
+			return datastore.ErrRecordNotExist
+		}
+		return datastore.NewDBError(dbGet.Error)
+	}
+	return nil
 }
 
 // Put update data model
 func (m *mysql) Put(ctx context.Context, entity datastore.Entity) error {
-	return fmt.Errorf("method to be implemented")
+	if entity.PrimaryKey() == "" {
+		return datastore.ErrPrimaryEmpty
+	}
+	if entity.TableName() == "" {
+		return datastore.ErrTableNameEmpty
+	}
+	entity.SetUpdateTime(time.Now())
+	if dbPut := m.client.Model(entity).Updates(entity); dbPut.Error != nil {
+		if errors.Is(dbPut.Error, gorm.ErrRecordNotFound) {
+			return datastore.ErrRecordNotExist
+		}
+		return datastore.NewDBError(dbPut.Error)
+	}
+	return nil
 }
 
 // IsExist determine whether data exists.
 func (m *mysql) IsExist(ctx context.Context, entity datastore.Entity) (bool, error) {
-	return true, fmt.Errorf("method to be implemented")
+	if entity.PrimaryKey() == "" {
+		return false, datastore.ErrPrimaryEmpty
+	}
+	if entity.TableName() == "" {
+		return false, datastore.ErrTableNameEmpty
+	}
+
+	if dbExist := m.client.First(entity); dbExist.Error != nil {
+		if errors.Is(dbExist.Error, gorm.ErrRecordNotFound) {
+			return false, nil
+		}
+		return false, datastore.NewDBError(dbExist.Error)
+	}
+
+	return true, nil
 }
 
 // Delete delete data
 func (m *mysql) Delete(ctx context.Context, entity datastore.Entity) error {
-	return fmt.Errorf("method to be implemented")
+	if entity.PrimaryKey() == "" {
+		return datastore.ErrPrimaryEmpty
+	}
+	if entity.TableName() == "" {
+		return datastore.ErrTableNameEmpty
+	}
+	// check entity is exist
+	if err := m.Get(ctx, entity); err != nil {
+		return err
+	}
+
+	if dbDelete := m.client.Unscoped().Model(entity).Delete(entity); dbDelete.Error != nil {
+		klog.Errorf("delete document failure %w", dbDelete.Error)
+		return datastore.NewDBError(dbDelete.Error)
+	}
+
+	return nil
+}
+
+func _applyFilterOptions(clauses []clause.Expression, filterOptions datastore.FilterOptions) []clause.Expression {
+	for _, queryOp := range filterOptions.Queries {
+		clauses = append(clauses, clause.Like{
+			Column: strings.ToLower(queryOp.Key),
+			Value:  fmt.Sprintf("%%%s%%", queryOp.Query),
+		})
+	}
+	for _, queryOp := range filterOptions.In {
+		values := make([]interface{}, len(queryOp.Values))
+		for i, v := range queryOp.Values {
+			values[i] = v
+		}
+		clauses = append(clauses, clause.IN{
+			Column: strings.ToLower(queryOp.Key),
+			Values: values,
+		})
+	}
+	for _, queryOp := range filterOptions.IsNotExist {
+		clauses = append(clauses, clause.Eq{
+			Column: strings.ToLower(queryOp.Key),
+			Value:  "",
+		})
+	}
+	return clauses
 }
 
 // List list entity function
 func (m *mysql) List(ctx context.Context, entity datastore.Entity, op *datastore.ListOptions) ([]datastore.Entity, error) {
-	return nil, fmt.Errorf("method to be implemented")
+	if entity.TableName() == "" {
+		return nil, datastore.ErrTableNameEmpty
+	}
+	var (
+		clauses []clause.Expression
+		exprs   []clause.Expression
+		limit   int
+		offset  int
+	)
+	if op != nil && op.PageSize > 0 && op.Page > 0 {
+		limit = int(op.PageSize)
+		offset = int(op.PageSize * (op.Page - 1))
+		clauses = append(clauses, clause.Limit{
+			Limit:  &limit,
+			Offset: offset,
+		})
+	}
+	for k, v := range entity.Index() {
+		exprs = append(exprs, clause.Eq{
+			Column: strings.ToLower(k),
+			Value:  v,
+		})
+	}
+	if op != nil {
+		exprs = _applyFilterOptions(exprs, op.FilterOptions)
+	}
+	if len(exprs) > 0 {
+		clauses = append(clauses, clause.Where{
+			Exprs: exprs,
+		})
+	}
+	if op != nil && op.SortBy != nil {
+		var sortOption []clause.OrderByColumn
+		for _, v := range op.SortBy {
+			sortOption = append(sortOption, clause.OrderByColumn{
+				Column: clause.Column{
+					Name: strings.ToLower(v.Key),
+				},
+				Desc: (v.Order == -1),
+			})
+		}
+		clauses = append(clauses, clause.OrderBy{
+			Columns: sortOption,
+		})
+	}
+	var list []datastore.Entity
+	rows, err := m.client.Model(entity).Clauses(clauses...).Rows()
+	defer rows.Close()
+	if err != nil {
+		return nil, datastore.NewDBError(err)
+	}
+	for rows.Next() {
+		item, err := datastore.NewEntity(entity)
+		if err != nil {
+			return nil, datastore.NewDBError(err)
+		}
+		m.client.ScanRows(rows, &item)
+		list = append(list, item)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, datastore.NewDBError(err)
+	}
+	return list, nil
 }
 
 // Count counts entities
 func (m *mysql) Count(ctx context.Context, entity datastore.Entity, filterOptions *datastore.FilterOptions) (int64, error) {
-	return 0, fmt.Errorf("method to be implemented")
+	if entity.TableName() == "" {
+		return 0, datastore.ErrTableNameEmpty
+	}
+	var (
+		count   int64
+		exprs   []clause.Expression
+		clauses []clause.Expression
+	)
+	for k, v := range entity.Index() {
+		exprs = append(exprs, clause.Eq{
+			Column: strings.ToLower(k),
+			Value:  v,
+		})
+	}
+	if filterOptions != nil {
+		exprs = _applyFilterOptions(exprs, *filterOptions)
+	}
+	if len(exprs) > 0 {
+		clauses = append(clauses, clause.Where{
+			Exprs: exprs,
+		})
+	}
+	if dbCount := m.client.Model(entity).Clauses(clauses...).Count(&count); dbCount.Error != nil {
+		return 0, datastore.NewDBError(dbCount.Error)
+	}
+	return count, nil
 }

pkg/server/infrastructure/datastore/mysql/mysql_suite_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/kubevela/velaux/pkg/server/infrastructure/datastore"
+)
+
+func TestMySQL(t *testing.T) {
+	_, err := New(context.TODO(), datastore.Config{
+		URL:      "root:kubevelaSQL123@tcp(127.0.0.1:3306)/kubevela?parseTime=True",
+		Database: "kubevela",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "MySQL Suite")
+}