implement generic service informer

cmd/climc/main.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/climc/promputils/complete_zsh.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package promputils
 
 import (

cmd/climc/shell/k8s/event.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package k8s
 
 import (

cmd/climc/shell/watch.go

@@ -0,0 +1,94 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/sets"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/informer"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+)
+
+type eventHandler struct {
+	man informer.IResourceManager
+}
+
+func (e eventHandler) keyword() string {
+	return e.man.GetKeyword()
+}
+
+func (e eventHandler) OnAdd(obj *jsonutils.JSONDict) {
+	log.Infof("%s [CREATED]: \n%s", e.keyword(), obj.String())
+}
+
+func (e eventHandler) OnUpdate(oldObj, newObj *jsonutils.JSONDict) {
+	log.Infof("%s [UPDATED]: \n[NEW]: %s\n[OLD]: %s", e.keyword(), newObj.String(), oldObj.String())
+}
+
+func (e eventHandler) OnDelete(obj *jsonutils.JSONDict) {
+	log.Infof("%s [DELETED]: \n%s", e.keyword(), obj.String())
+}
+
+func init() {
+	type WatchOptions struct {
+		Resource []string `help:"Resource manager plural keyword, e.g.'servers, disks, guestdisks'" short-token:"s"`
+		All      bool     `help:"Watch all resources"`
+	}
+
+	R(&WatchOptions{}, "watch", "Watch resources", func(s *mcclient.ClientSession, opts *WatchOptions) error {
+		watchMan, err := informer.NewWatchManagerBySession(s, nil)
+		if err != nil {
+			return err
+		}
+		resources := opts.Resource
+		if opts.All {
+			resSets := sets.NewString()
+			mods, _ := modulebase.GetRegisterdModules()
+			for _, ress := range mods {
+				resSets.Insert(ress...)
+			}
+			resources = resSets.List()
+		}
+		if len(resources) == 0 {
+			return errors.Errorf("no watch resources specified")
+		}
+		for _, res := range resources {
+			var resMan informer.IResourceManager
+			if modMan, _ := modulebase.GetModule(s, res); modMan != nil {
+				resMan = modMan
+			}
+			if resMan == nil {
+				if jointModMan, _ := modulebase.GetJointModule(s, res); jointModMan != nil {
+					resMan = jointModMan
+				}
+			}
+			if resMan == nil {
+				//return errors.Errorf("Not found %q resource manager", res)
+				log.Warningf("Not found %q resource manager", res)
+				continue
+			}
+			if err := watchMan.For(resMan).AddEventHandler(context.Background(), eventHandler{resMan}); err != nil {
+				return errors.Wrapf(err, "watch resource %s", res)
+			}
+		}
+		select {}
+	})
+}

pkg/apis/compute/vpcs_ovn.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 import (

pkg/apis/identity/endpoint.go

@@ -38,5 +38,5 @@ type CertificateDetails struct {
 }
 
 const (
-	ENDPOINT_ETCD_INTERNAL = "etcd-internal"
+	SERVICE_TYPE_ETCD = "etcd"
 )

pkg/cloudcommon/app/auth.go

@@ -19,17 +19,14 @@ import (
 	"time"
 
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
-	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompletedCallback) {
@@ -63,7 +60,7 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 	if options.SessionEndpointType != "" {
 		if !utils.IsInStringArray(options.SessionEndpointType,
-			[]string{auth.PublicEndpointType, auth.InternalEndpointType}) {
+			[]string{identity.EndpointInterfacePublic, identity.EndpointInterfaceInternal}) {
 			log.Fatalf("Invalid session endpoint type %s", options.SessionEndpointType)
 		}
 		auth.SetEndpointType(options.SessionEndpointType)
@@ -97,16 +94,5 @@ func InitBaseAuth(options *common_options.BaseOptions) {
 
 func FetchEtcdServiceInfo() (*identity.EndpointDetails, error) {
 	s := auth.GetAdminSession(context.Background(), "", "")
-	ret, err := modules.EndpointsV3.GetByName(s, identity.ENDPOINT_ETCD_INTERNAL, nil)
-	if err != nil && errors.Cause(err) == httperrors.ErrNotFound {
-		return nil, nil
-	} else if err != nil {
-		return nil, err
-	}
-	endpoint := new(identity.EndpointDetails)
-	err = ret.Unmarshal(endpoint)
-	if err != nil {
-		return nil, errors.Wrap(err, "unmarshal endpoint")
-	}
-	return endpoint, nil
+	return s.GetCommonEtcdEndpoint()
 }

pkg/cloudcommon/database.go

@@ -27,6 +27,8 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
+	"yunion.io/x/onecloud/pkg/cloudcommon/informer"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 )
 
@@ -77,6 +79,26 @@ func InitDB(options *common_options.DBOptions) {
 		lockman.Init(lm)
 	}
 	// lm := lockman.NewNoopLockManager()
+
+	if len(options.EtcdEndpoints) != 0 {
+		log.Infof("using etcd as resource informer backend")
+		tlsCfg, err := options.GetEtcdTLSConfig()
+		if err != nil {
+			log.Fatalf("get etcd informer backend tls config err: %v", err)
+		}
+		informerBackend, err := informer.NewEtcdBackend(&etcd.SEtcdOptions{
+			EtcdEndpoint:              options.EtcdEndpoints,
+			EtcdTimeoutSeconds:        5,
+			EtcdRequestTimeoutSeconds: 2,
+			EtcdLeaseExpireSeconds:    5,
+			EtcdEnabldSsl:             options.EtcdUseTLS,
+			TLSConfig:                 tlsCfg,
+		}, nil)
+		if err != nil {
+			log.Fatalf("new etcd informer backend error: %v", err)
+		}
+		informer.Init(informerBackend)
+	}
 }
 
 func CloseDB() {

pkg/cloudcommon/db/certificateresource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package db
 
 import (

pkg/cloudcommon/db/interface.go

@@ -40,7 +40,7 @@ type IModelManager interface {
 	GetIModelManager() IModelManager
 
 	// Table() *sqlchemy.STable
-	TableSpec() *sqlchemy.STableSpec
+	TableSpec() ITableSpec
 
 	// Keyword() string
 	KeywordPlural() string

pkg/cloudcommon/db/modelbase.go

@@ -44,15 +44,15 @@ type SModelBase struct {
 type SModelBaseManager struct {
 	object.SObject
 
-	tableSpec     *sqlchemy.STableSpec
+	tableSpec     ITableSpec
 	keyword       string
 	keywordPlural string
 	alias         string
 	aliasPlural   string
 }
 
 func NewModelBaseManager(model interface{}, tableName string, keyword string, keywordPlural string) SModelBaseManager {
-	ts := sqlchemy.NewTableSpecFromStruct(model, tableName)
+	ts := newTableSpec(model, tableName)
 	modelMan := SModelBaseManager{tableSpec: ts, keyword: keyword, keywordPlural: keywordPlural}
 	return modelMan
 }
@@ -78,7 +78,7 @@ func (manager *SModelBaseManager) SetAlias(alias string, aliasPlural string) {
 	manager.aliasPlural = aliasPlural
 }
 
-func (manager *SModelBaseManager) TableSpec() *sqlchemy.STableSpec {
+func (manager *SModelBaseManager) TableSpec() ITableSpec {
 	return manager.tableSpec
 }
 

pkg/cloudcommon/db/sharablebase_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package db
 
 import (