From 2156893d72db363e297cc266e89addb892120e83 Mon Sep 17 00:00:00 2001 From: Darren Hague Date: Fri, 9 Jun 2017 14:48:52 +0100 Subject: [PATCH] Refactoring Remove cmd folder - we don't use the command line Clearer interface naming & structure: "keystone" interface is now "identity", with 2 implementations: mock & keystone ElasticSearch EventDetails struct now called ESEventDetails to avoid confusion with hermes.EventDetails --- main.go | 37 ++-- pkg/api/api_test.go | 6 +- pkg/api/core.go | 8 +- pkg/api/server.go | 4 +- pkg/cmd/auth/token.go | 79 ------- pkg/cmd/get.go | 73 ------- pkg/cmd/list.go | 81 -------- pkg/cmd/root.go | 102 --------- pkg/hermes/events.go | 13 +- pkg/hermes/events_test.go | 6 +- pkg/{keystone => identity}/interface.go | 14 +- pkg/{keystone => identity}/keystone.go | 263 +++++++----------------- pkg/identity/keystone_cache.go | 138 +++++++++++++ pkg/identity/mock.go | 61 ++++++ pkg/keystone/mock.go | 66 ------ pkg/policy/policy_test.go | 14 +- pkg/storage/elasticsearch.go | 20 +- pkg/storage/interface.go | 6 +- pkg/storage/mock.go | 12 +- pkg/storage/mock_test.go | 4 +- 20 files changed, 336 insertions(+), 671 deletions(-) delete mode 100644 pkg/cmd/auth/token.go delete mode 100644 pkg/cmd/get.go delete mode 100644 pkg/cmd/list.go delete mode 100644 pkg/cmd/root.go rename pkg/{keystone => identity}/interface.go (83%) rename pkg/{keystone => identity}/keystone.go (64%) create mode 100644 pkg/identity/keystone_cache.go create mode 100644 pkg/identity/mock.go delete mode 100644 pkg/keystone/mock.go diff --git a/main.go b/main.go index 02d47349..1b8b6b2a 100644 --- a/main.go +++ b/main.go @@ -30,8 +30,7 @@ import ( "github.com/databus23/goslo.policy" "github.com/sapcc/hermes/pkg/api" - "github.com/sapcc/hermes/pkg/cmd" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" "github.com/sapcc/hermes/pkg/util" "github.com/spf13/viper" @@ -47,18 +46,7 @@ func main() { keystoneDriver := configuredKeystoneDriver() storageDriver := configuredStorageDriver() readPolicy() - - // If there are args left over after flag processing, we are a Hermes CLI client - if len(flag.Args()) > 0 { - cmd.RootCmd.SetArgs(flag.Args()) - cmd.SetDrivers(keystoneDriver, storageDriver) - if err := cmd.RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } - } else { // otherwise, we are running a Hermes API server - api.Server(keystoneDriver, storageDriver) - } + api.Server(keystoneDriver, storageDriver) } func parseCmdlineFlags() { @@ -104,32 +92,39 @@ func readConfig(configPath *string) { } // Setup environment variable overrides for OpenStack authentication - for _, osVarName := range cmd.OSVars { + var OSVars = []string{"username", "password", "auth_url", "user_domain_name", "project_name", "project_domain_name"} + for _, osVarName := range OSVars { viper.BindEnv("keystone."+osVarName, "OS_"+strings.ToUpper(osVarName)) } } -func configuredKeystoneDriver() keystone.Driver { +var keystoneIdentity = identity.Keystone{} +var mockIdentity = identity.Mock{} + +func configuredKeystoneDriver() identity.Identity { driverName := viper.GetString("hermes.keystone_driver") switch driverName { case "keystone": - return keystone.Keystone() + return keystoneIdentity case "mock": - return keystone.Mock() + return mockIdentity default: log.Printf("Couldn't match a keystone driver for configured value \"%s\"", driverName) return nil } } -func configuredStorageDriver() storage.Driver { +var elasticSearchStorage = storage.ElasticSearch{} +var mockStorage = storage.Mock{} + +func configuredStorageDriver() storage.Storage { driverName := viper.GetString("hermes.storage_driver") switch driverName { case "elasticsearch": - return storage.ElasticSearch() + return elasticSearchStorage case "mock": - return storage.Mock() + return mockStorage default: log.Printf("Couldn't match a storage driver for configured value \"%s\"", driverName) return nil diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 13eb82e8..c3f287fe 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -25,7 +25,7 @@ import ( "encoding/json" "github.com/databus23/goslo.policy" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" "github.com/sapcc/hermes/pkg/test" "github.com/spf13/viper" @@ -52,8 +52,8 @@ func setupTest(t *testing.T) http.Handler { viper.Set("hermes.PolicyEnforcer", policyEnforcer) //create test driver with the domains and projects from start-data.sql - keystone := keystone.Mock() - storage := storage.Mock() + keystone := identity.Mock{} + storage := storage.Mock{} router, _ := NewV1Router(keystone, storage) return router } diff --git a/pkg/api/core.go b/pkg/api/core.go index 772c388f..52be5aa1 100644 --- a/pkg/api/core.go +++ b/pkg/api/core.go @@ -27,7 +27,7 @@ import ( "bytes" "fmt" "github.com/gorilla/mux" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" ) @@ -47,15 +47,15 @@ type versionLinkData struct { } type v1Provider struct { - keystone keystone.Driver - storage storage.Driver + keystone identity.Identity + storage storage.Storage versionData versionData } //NewV1Router creates a http.Handler that serves the Hermes v1 API. //It also returns the versionData for this API version which is needed for the //version advertisement on "GET /". -func NewV1Router(keystone keystone.Driver, storage storage.Driver) (http.Handler, versionData) { +func NewV1Router(keystone identity.Identity, storage storage.Storage) (http.Handler, versionData) { r := mux.NewRouter() p := &v1Provider{ keystone: keystone, diff --git a/pkg/api/server.go b/pkg/api/server.go index c83c116c..0092dfc3 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -6,14 +6,14 @@ import ( "github.com/gorilla/mux" "github.com/rs/cors" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" "github.com/sapcc/hermes/pkg/util" "github.com/spf13/viper" ) // Set up and start the API server, hooking it up to the API router -func Server(keystone keystone.Driver, storage storage.Driver) error { +func Server(keystone identity.Identity, storage storage.Storage) error { fmt.Println("API") mainRouter := mux.NewRouter() diff --git a/pkg/cmd/auth/token.go b/pkg/cmd/auth/token.go deleted file mode 100644 index 3fe738c7..00000000 --- a/pkg/cmd/auth/token.go +++ /dev/null @@ -1,79 +0,0 @@ -/******************************************************************************* -* -* Copyright 2017 Stefan Majewsky -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You should have received a copy of the License along with this -* program. If not, you may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -*******************************************************************************/ - -package auth - -import ( - policy "github.com/databus23/goslo.policy" - "github.com/sapcc/hermes/pkg/keystone" - "github.com/spf13/viper" - "log" - "os" -) - -//Token represents a user's token, as returned from an authentication request -type Token struct { - enforcer *policy.Enforcer - Context policy.Context - err error -} - -// GetToken authenticates using the configured credentials in Keystone, and -// returns a Token instance for checking authorization. Any errors that occur -// during this function are deferred until Require() is called. -func GetToken(keystoneDriver keystone.Driver) *Token { - t := &Token{enforcer: viper.Get("hermes.PolicyEnforcer").(*policy.Enforcer)} - - credentials := keystoneDriver.AuthOptions() - - t.Context, t.err = keystoneDriver.Authenticate(credentials) - return t -} - -//Require checks if the given token has the given permission according to the -//policy.json that is in effect. If not, an error response is written and false -//is returned. -func (t *Token) Require(rule string) bool { - if t.err != nil { - return false - } - - if os.Getenv("DEBUG") == "1" { - t.Context.Logger = log.Printf //or any other function with the same signature - } - - if !t.enforcer.Enforce(rule, t.Context) { - return false - } - return true -} - -//Check is like Require, but does not write error responses. -func (t *Token) Check(rule string) bool { - return t.err == nil && t.enforcer.Enforce(rule, t.Context) -} - -// TenantId is the project_id if used, otherwise the domain_id (which may be empty) -func (t *Token) TenantId() string { - id, project := t.Context.Auth["project_id"] - if project { - return id - } - return t.Context.Auth["domain_id"] -} diff --git a/pkg/cmd/get.go b/pkg/cmd/get.go deleted file mode 100644 index 9fe19e57..00000000 --- a/pkg/cmd/get.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 SAP SE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "errors" - "fmt" - - "encoding/json" - "github.com/sapcc/hermes/pkg/cmd/auth" - "github.com/sapcc/hermes/pkg/hermes" - "github.com/spf13/cobra" -) - -// getCmd represents the get command -var getCmd = &cobra.Command{ - Use: "get ", - Short: "Get the audit event with ID ", - Long: `Get the audit event with ID .`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return errors.New("You must specify exactly one event ID.") - } - - token := auth.GetToken(keystoneDriver) - if !token.Require("event:show") { - return errors.New("You are not authorised to view event details") - } - - eventId := args[0] - event, err := hermes.GetEvent(eventId, token.TenantId(), keystoneDriver, storageDriver) - if err != nil { - return err - } - if event == nil { - return fmt.Errorf("Event %s could not be found in tenant %s", eventId, token.TenantId()) - } - json, err := json.MarshalIndent(event, "", " ") - if err != nil { - return err - } - fmt.Printf("%s", json) - - return nil - }, -} - -func init() { - RootCmd.AddCommand(getCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // getCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // getCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - -} diff --git a/pkg/cmd/list.go b/pkg/cmd/list.go deleted file mode 100644 index 0575a94f..00000000 --- a/pkg/cmd/list.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 SAP SE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "errors" - "fmt" - "github.com/olekukonko/tablewriter" - "github.com/sapcc/hermes/pkg/cmd/auth" - "github.com/sapcc/hermes/pkg/hermes" - "github.com/spf13/cobra" - "os" -) - -// listCmd represents the list command -var listCmd = &cobra.Command{ - Use: "list", - Short: "Lists a project’s or domain's audit events. The project or domain comes from the scope of the authentication parameters.", - Long: `Lists a project’s or domain's audit events. The project or domain comes from the scope of the authentication parameters. - -Date Filters: -The value for the time parameter is a comma-separated list of time stamps in ISO 8601 format. The time stamps can be prefixed with any of these comparison operators: gt: (greater-than), gte: (greater-than-or-equal), lt: (less-than), lte: (less-than-or-equal). -For example, to get a list of events that will expire in January of 2020: -GET /v1/events?time=gte:2020-01-01T00:00:00,lt:2020-02-01T00:00:00 - -Sorting: -The value of the sort parameter is a comma-separated list of sort keys. Supported sort keys include time, source, resource_type, resource_name, and event_type. -Each sort key may also include a direction. Supported directions are :asc for ascending and :desc for descending. The service will use :asc for every key that does not include a direction. -For example, to sort the list from most recently created to oldest: -GET /v1/events?sort=time:desc`, - RunE: func(cmd *cobra.Command, args []string) error { - token := auth.GetToken(keystoneDriver) - if !token.Require("event:list") { - return errors.New("You are not authorised to list events") - } - - eventSlice, total, err := hermes.GetEvents(&hermes.Filter{}, token.TenantId(), keystoneDriver, storageDriver) - if err != nil { - return err - } - - fmt.Printf("Total hits: %d\n", total) - headers := []string{"Source", "Event ID", "Event Type", "Event Time", "Resource Name", "Resource Type", "User Name"} - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader(headers) - table.SetBorder(true) - for _, ev := range eventSlice { - dataRow := []string{ev.Source, ev.ID, ev.Type, ev.Time, ev.ResourceName, ev.ResourceType, ev.Initiator.UserName} - table.Append(dataRow) - } - table.Render() - return nil - }, -} - -func init() { - RootCmd.AddCommand(listCmd) - - listCmd.Flags().StringP("source", "s", "", "Selects all events with this source.") - listCmd.Flags().StringP("resource_type", "r", "", "Selects all events with this resource type.") - listCmd.Flags().StringP("resource_name", "n", "", "Selects all events with this resource name.") - listCmd.Flags().StringP("user_name", "u", "", "Selects all events with this user name.") - listCmd.Flags().StringP("event_type", "e", "", "Selects all events with this event type.") - listCmd.Flags().StringP("time", "t", "", "Date filter to select all events with event_time matching the specified criteria. See above for more detail.") - listCmd.Flags().Int32P("offset", "o", 0, "The starting index within the total list of the events that you would like to retrieve..") - listCmd.Flags().Int32P("limit", "l", 10, "The maximum number of records to return (up to 100). The default limit is 10.") - listCmd.Flags().String("sort", "", "Determines the sorted order of the returned list. See above for more detail.") - -} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go deleted file mode 100644 index 002b9df3..00000000 --- a/pkg/cmd/root.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 SAP SE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "os" - - "github.com/sapcc/hermes/pkg/keystone" - "github.com/sapcc/hermes/pkg/storage" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "strings" -) - -var cfgFile string - -// RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "hermes", - Short: "Command-line client and API server for OpenStack Audit Data service", - Long: `Command-line client and API server for OpenStack Audit Data service.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, -} - -// Execute adds all child commands to the root command sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(-1) - } -} - -var keystoneDriver keystone.Driver -var storageDriver storage.Driver - -// Specify which keystone & storage driver to use -func SetDrivers(keystoneParam keystone.Driver, storageParam storage.Driver) { - keystoneDriver = keystoneParam - storageDriver = storageParam -} - -// When adding a value here, also add a "RootCmd.PersistentFlags().StringVar" line in cmd/root.go's init() -var OSVars = []string{"username", "password", "auth_url", "user_domain_name", "project_name", "project_domain_name"} - -func init() { - //cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports Persistent Flags, which, if defined here, - // will be global for your application. - - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-auth-url", "", "OpenStack Authentication URL") - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-username", "", "OpenStack Username") - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-password", "", "OpenStack Password") - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-user-domain-name", "", "OpenStack User's domain name") - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-project-name", "", "OpenStack Project name to scope to") - RootCmd.PersistentFlags().StringVar(&cfgFile, "os-project-domain-name", "", "OpenStack Project's domain name") - - // Setup command-line flags for OpenStack authentication - for _, val := range OSVars { - flags := RootCmd.PersistentFlags() - lookup := "os-" + strings.Replace(val, "_", "-", -1) - pflag := flags.Lookup(lookup) - viper.BindPFlag("keystone."+val, pflag) - } - // Cobra also supports local flags, which will only run - // when this action is called directly. - //RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} - -// -//// initConfig reads in config file and ENV variables if set. -//func initConfig() { -// if cfgFile != "" { // enable ability to specify config file via flag -// viper.SetConfigFile(cfgFile) -// } -// -// viper.SetConfigName(".hermes") // name of config file (without extension) -// viper.AddConfigPath(os.Getenv("HOME")) // adding home directory as first search path -// viper.AutomaticEnv() // read in environment variables that match -// -// // If a config file is found, read it in. -// if err := viper.ReadInConfig(); err == nil { -// fmt.Println("Using config file:", viper.ConfigFileUsed()) -// } -//} diff --git a/pkg/hermes/events.go b/pkg/hermes/events.go index 2acd22f2..3836c4d4 100644 --- a/pkg/hermes/events.go +++ b/pkg/hermes/events.go @@ -22,7 +22,7 @@ package hermes import ( "fmt" "github.com/jinzhu/copier" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" "github.com/sapcc/hermes/pkg/util" "github.com/spf13/viper" @@ -31,6 +31,7 @@ import ( ) // ListEvent contains high-level data about an event, intended as a list item +// The JSON annotations here are for the JSON to be returned by the API type ListEvent struct { Source string `json:"source"` ID string `json:"event_id"` @@ -75,7 +76,7 @@ type Filter struct { } // GetEvents returns a list of matching events (with filtering) -func GetEvents(filter *Filter, tenantId string, keystoneDriver keystone.Driver, eventStore storage.Driver) ([]*ListEvent, int, error) { +func GetEvents(filter *Filter, tenantId string, keystoneDriver identity.Identity, eventStore storage.Storage) ([]*ListEvent, int, error) { storageFilter, err := storageFilter(filter, keystoneDriver, eventStore) if err != nil { return nil, 0, err @@ -92,7 +93,7 @@ func GetEvents(filter *Filter, tenantId string, keystoneDriver keystone.Driver, return events, total, err } -func storageFilter(filter *Filter, keystoneDriver keystone.Driver, eventStore storage.Driver) (*storage.Filter, error) { +func storageFilter(filter *Filter, keystoneDriver identity.Identity, eventStore storage.Storage) (*storage.Filter, error) { // As per the documentation, the default limit is 10 if filter.Limit == 0 { filter.Limit = 10 @@ -133,7 +134,7 @@ func storageFilter(filter *Filter, keystoneDriver keystone.Driver, eventStore st } // Construct ListEvents and add the names for IDs in the events -func eventsList(eventDetails []*storage.EventDetail, keystoneDriver keystone.Driver) ([]*ListEvent, error) { +func eventsList(eventDetails []*storage.EventDetail, keystoneDriver identity.Identity) ([]*ListEvent, error) { var events []*ListEvent for _, storageEvent := range eventDetails { p := storageEvent.Payload @@ -169,7 +170,7 @@ func eventsList(eventDetails []*storage.EventDetail, keystoneDriver keystone.Dri } // GetEvent returns the CADF detail for event with the specified ID -func GetEvent(eventID string, tenantId string, keystoneDriver keystone.Driver, eventStore storage.Driver) (*storage.EventDetail, error) { +func GetEvent(eventID string, tenantId string, keystoneDriver identity.Identity, eventStore storage.Storage) (*storage.EventDetail, error) { event, err := eventStore.GetEvent(eventID, tenantId) if viper.GetBool("hermes.enrich_keystone_events") { @@ -198,7 +199,7 @@ func GetEvent(eventID string, tenantId string, keystoneDriver keystone.Driver, e return event, err } -func namesForIds(keystoneDriver keystone.Driver, idMap map[string]string, targetType string) map[string]string { +func namesForIds(keystoneDriver identity.Identity, idMap map[string]string, targetType string) map[string]string { nameMap := map[string]string{} var err error diff --git a/pkg/hermes/events_test.go b/pkg/hermes/events_test.go index 767305cb..b8cb66c7 100644 --- a/pkg/hermes/events_test.go +++ b/pkg/hermes/events_test.go @@ -3,7 +3,7 @@ package hermes import ( "testing" - "github.com/sapcc/hermes/pkg/keystone" + "github.com/sapcc/hermes/pkg/identity" "github.com/sapcc/hermes/pkg/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -11,7 +11,7 @@ import ( func Test_GetEvent(t *testing.T) { eventId := "d5eed458-6666-58ec-ad06-8d3cf6bafca1" - event, err := GetEvent(eventId, "", keystone.Mock(), storage.Mock()) + event, err := GetEvent(eventId, "", identity.Mock{}, storage.Mock{}) require.Nil(t, err) require.NotNil(t, event) assert.Equal(t, "d5eed458-6666-58ec-ad06-8d3cf6bafca1", event.Payload.ID) @@ -21,7 +21,7 @@ func Test_GetEvent(t *testing.T) { } func Test_GetEvents(t *testing.T) { - events, total, err := GetEvents(&Filter{}, "", keystone.Mock(), storage.Mock()) + events, total, err := GetEvents(&Filter{}, "", identity.Mock{}, storage.Mock{}) require.Nil(t, err) require.NotNil(t, events) assert.Equal(t, len(events), 3) diff --git a/pkg/keystone/interface.go b/pkg/identity/interface.go similarity index 83% rename from pkg/keystone/interface.go rename to pkg/identity/interface.go index 6414b66b..e3de45d0 100644 --- a/pkg/keystone/interface.go +++ b/pkg/identity/interface.go @@ -17,19 +17,19 @@ * *******************************************************************************/ -package keystone +package identity import ( policy "github.com/databus23/goslo.policy" "github.com/gophercloud/gophercloud" ) -// Driver is an interface that wraps the authentication of the service user and +// Identity is an interface that wraps the authentication of the service user and // token checking of API users. Because it is an interface, the real implementation // can be mocked away in unit tests. -type Driver interface { +type Identity interface { //Return the main gophercloud client from which the respective service - //clients can be derived. For mock drivers, this returns nil, so test code + //clients can be derived. For Mock drivers, this returns nil, so test code //should be prepared to handle a nil Client() where appropriate. Client() *gophercloud.ProviderClient AuthOptions() *gophercloud.AuthOptions @@ -43,9 +43,3 @@ type Driver interface { RoleName(id string) (string, error) GroupName(id string) (string, error) } - -//KeystoneNameId describes just the name and id of a Keystone object. -type KeystoneNameId struct { - UUID string `json:"id"` - Name string `json:"name"` -} diff --git a/pkg/keystone/keystone.go b/pkg/identity/keystone.go similarity index 64% rename from pkg/keystone/keystone.go rename to pkg/identity/keystone.go index a95fbcc5..94bdcb96 100644 --- a/pkg/keystone/keystone.go +++ b/pkg/identity/keystone.go @@ -17,12 +17,11 @@ * *******************************************************************************/ -package keystone +package identity import ( "fmt" - "container/list" policy "github.com/databus23/goslo.policy" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" @@ -31,148 +30,54 @@ import ( "github.com/sapcc/hermes/pkg/util" "github.com/spf13/viper" "sync" - "time" ) -// Real keystone implementation -func Keystone() Driver { - return keystone{} +// Real Keystone implementation +type Keystone struct { + TokenRenewalMutex *sync.Mutex // Used for controlling the token refresh process } -type keystone struct { - TokenRenewalMutex *sync.Mutex -} - -type cache struct { - sync.RWMutex - m map[string]string -} - -func updateCache(cache *cache, key string, value string) { - cache.Lock() - cache.m[key] = value - cache.Unlock() -} - -func getFromCache(cache *cache, key string) (string, bool) { - cache.RLock() - value, exists := cache.m[key] - cache.RUnlock() - return value, exists -} - -type keystoneTokenCache struct { - sync.RWMutex - tMap map[string]*keystoneToken // map tokenID to token struct - eMap map[time.Time][]string // map expiration time to list of tokenIDs - eList *list.List // sorted list of expiration times +// The JSON mappings here are for parsing Keystone responses +type keystoneToken struct { + DomainScope keystoneTokenThing `json:"domain"` + ProjectScope keystoneTokenThingInDomain `json:"project"` + Roles []keystoneTokenThing `json:"roles"` + User keystoneTokenThingInDomain `json:"user"` + ExpiresAt string `json:"expires_at"` } -func addTokenToCache(cache *keystoneTokenCache, id string, token *keystoneToken) { - expiryTime, err := time.Parse("2006-01-02T15:04:05.999999Z", token.ExpiresAt) - if err != nil { - util.LogWarning("Not adding token to cache because time '%s' could not be parsed", token.ExpiresAt) - return - } - cache.Lock() - cache.eList.PushBack(expiryTime) - // If the expiryTime isn't later than the last item in the list, - // move it to the correct place so that we keep the list sorted - lastItem := cache.eList.Back() - if cache.eList.Back() != nil && expiryTime.Before(lastItem.Value.(time.Time)) { - for e := cache.eList.Back(); e != nil; e = e.Prev() { - if expiryTime.After((e.Value).(time.Time)) { - cache.eList.MoveAfter(cache.eList.Back(), e) - } - } - } - if cache.eMap[expiryTime] == nil { - cache.eMap[expiryTime] = []string{id} - } else { - cache.eMap[expiryTime] = append(cache.eMap[expiryTime], id) - } - cache.tMap[id] = token - cacheSize := len(cache.tMap) - cache.Unlock() - util.LogDebug("Added token to cache. Current cache size: %d", cacheSize) +// The JSON mappings here are for parsing Keystone responses +type keystoneTokenThing struct { + ID string `json:"id"` + Name string `json:"name"` } -func getCachedToken(cache *keystoneTokenCache, id string) *keystoneToken { - // First, remove expired tokens from cache - now := time.Now() - elemsToRemove := []*list.Element{} - cache.RLock() - for e := cache.eList.Front(); e != nil; e = e.Next() { - expiryTime := (e.Value).(time.Time) - if now.Before(expiryTime) { - break // list is sorted, so we can stop once we get to an unexpired token - } - // We can't remove from the list as we iterate, so remember which ones to delete - elemsToRemove = append(elemsToRemove, e) - } - cache.RUnlock() - cache.Lock() - for _, elem := range elemsToRemove { - cache.eList.Remove(elem) // Remove the cached expiry time from the sorted list - time := (elem.Value).(time.Time) - tokenIds := cache.eMap[time] - delete(cache.eMap, time) // Remove the cached expiry time from the time:tokenIDs map - for _, tokenId := range tokenIds { - delete(cache.tMap, tokenId) // Remove all the cached tokens - } - } - cacheSize := len(cache.tMap) - cache.Unlock() - if len(elemsToRemove) > 0 { - util.LogDebug("Removed expired token(s) from cache. Current cache size: %d", cacheSize) - } - // Now look for the token in question - cache.RLock() - token := cache.tMap[id] - cache.RUnlock() - if token != nil { - util.LogDebug("Got token from cache. Current cache size: %d", cacheSize) - } - - return token +// The JSON mappings here are for parsing Keystone responses +type keystoneTokenThingInDomain struct { + keystoneTokenThing + Domain keystoneTokenThing `json:"domain"` } -var providerClient *gophercloud.ProviderClient -var domainNameCache *cache -var projectNameCache *cache -var userNameCache *cache -var userIdCache *cache -var roleNameCache *cache -var groupNameCache *cache -var tokenCache *keystoneTokenCache - -func init() { - domainNameCache = &cache{m: make(map[string]string)} - projectNameCache = &cache{m: make(map[string]string)} - userNameCache = &cache{m: make(map[string]string)} - userIdCache = &cache{m: make(map[string]string)} - roleNameCache = &cache{m: make(map[string]string)} - groupNameCache = &cache{m: make(map[string]string)} - tokenCache = &keystoneTokenCache{ - tMap: make(map[string]*keystoneToken), - eMap: make(map[time.Time][]string), - eList: list.New(), - } +//keystoneNameId describes just the name and id of a Identity object. +// The JSON mappings here are for parsing Keystone responses +type keystoneNameId struct { + UUID string `json:"id"` + Name string `json:"name"` } -func (d keystone) keystoneClient() (*gophercloud.ServiceClient, error) { +func (d Keystone) keystoneClient() (*gophercloud.ServiceClient, error) { if d.TokenRenewalMutex == nil { d.TokenRenewalMutex = &sync.Mutex{} } if providerClient == nil { var err error - providerClient, err = openstack.NewClient(viper.GetString("keystone.auth_url")) + providerClient, err = openstack.NewClient(viper.GetString("Keystone.auth_url")) if err != nil { return nil, fmt.Errorf("cannot initialize OpenStack client: %v", err) } err = d.RefreshToken() if err != nil { - return nil, fmt.Errorf("cannot fetch initial Keystone token: %v", err) + return nil, fmt.Errorf("cannot fetch initial Identity token: %v", err) } } @@ -181,10 +86,10 @@ func (d keystone) keystoneClient() (*gophercloud.ServiceClient, error) { ) } -func (d keystone) Client() *gophercloud.ProviderClient { - var kc keystone +func (d Keystone) Client() *gophercloud.ProviderClient { + var kc Keystone - err := viper.UnmarshalKey("keystone", &kc) + err := viper.UnmarshalKey("Keystone", &kc) if err != nil { fmt.Printf("unable to decode into struct, %v", err) } @@ -192,7 +97,7 @@ func (d keystone) Client() *gophercloud.ProviderClient { return nil } -func (d keystone) ValidateToken(token string) (policy.Context, error) { +func (d Keystone) ValidateToken(token string) (policy.Context, error) { cachedToken := getCachedToken(tokenCache, token) if cachedToken != nil { return cachedToken.ToContext(), nil @@ -219,29 +124,7 @@ func (d keystone) ValidateToken(token string) (policy.Context, error) { return tokenData.ToContext(), nil } -func (d keystone) updateCaches(token *keystoneToken, tokenStr string) { - addTokenToCache(tokenCache, tokenStr, token) - if token.DomainScope.ID != "" && token.DomainScope.Name != "" { - updateCache(domainNameCache, token.DomainScope.ID, token.DomainScope.Name) - } - if token.ProjectScope.Domain.ID != "" && token.ProjectScope.Domain.Name != "" { - updateCache(domainNameCache, token.ProjectScope.Domain.ID, token.ProjectScope.Domain.Name) - } - if token.ProjectScope.ID != "" && token.ProjectScope.Name != "" { - updateCache(projectNameCache, token.ProjectScope.ID, token.ProjectScope.Name) - } - if token.User.ID != "" && token.User.Name != "" { - updateCache(userNameCache, token.User.ID, token.User.Name) - updateCache(userIdCache, token.User.Name, token.User.ID) - } - for _, role := range token.Roles { - if role.ID != "" && role.Name != "" { - updateCache(roleNameCache, role.ID, role.Name) - } - } -} - -func (d keystone) Authenticate(credentials *gophercloud.AuthOptions) (policy.Context, error) { +func (d Keystone) Authenticate(credentials *gophercloud.AuthOptions) (policy.Context, error) { client, err := d.keystoneClient() if err != nil { return policy.Context{}, err @@ -260,7 +143,7 @@ func (d keystone) Authenticate(credentials *gophercloud.AuthOptions) (policy.Con return tokenData.ToContext(), nil } -func (d keystone) DomainName(id string) (string, error) { +func (d Keystone) DomainName(id string) (string, error) { cachedName, hit := getFromCache(domainNameCache, id) if hit { return cachedName, nil @@ -279,7 +162,7 @@ func (d keystone) DomainName(id string) (string, error) { } var data struct { - Domain KeystoneNameId `json:"domain"` + Domain keystoneNameId `json:"domain"` } err = result.ExtractInto(&data) if err == nil { @@ -288,7 +171,7 @@ func (d keystone) DomainName(id string) (string, error) { return data.Domain.Name, err } -func (d keystone) ProjectName(id string) (string, error) { +func (d Keystone) ProjectName(id string) (string, error) { cachedName, hit := getFromCache(projectNameCache, id) if hit { return cachedName, nil @@ -307,7 +190,7 @@ func (d keystone) ProjectName(id string) (string, error) { } var data struct { - Project KeystoneNameId `json:"project"` + Project keystoneNameId `json:"project"` } err = result.ExtractInto(&data) if err == nil { @@ -316,7 +199,7 @@ func (d keystone) ProjectName(id string) (string, error) { return data.Project.Name, err } -func (d keystone) UserName(id string) (string, error) { +func (d Keystone) UserName(id string) (string, error) { cachedName, hit := getFromCache(userNameCache, id) if hit { return cachedName, nil @@ -335,7 +218,7 @@ func (d keystone) UserName(id string) (string, error) { } var data struct { - User KeystoneNameId `json:"user"` + User keystoneNameId `json:"user"` } err = result.ExtractInto(&data) if err == nil { @@ -345,7 +228,7 @@ func (d keystone) UserName(id string) (string, error) { return data.User.Name, err } -func (d keystone) UserId(name string) (string, error) { +func (d Keystone) UserId(name string) (string, error) { cachedId, hit := getFromCache(userIdCache, name) if hit { return cachedId, nil @@ -364,7 +247,7 @@ func (d keystone) UserId(name string) (string, error) { } var data struct { - User []KeystoneNameId `json:"user"` + User []keystoneNameId `json:"user"` } err = result.ExtractInto(&data) userId := "" @@ -384,7 +267,7 @@ func (d keystone) UserId(name string) (string, error) { return userId, err } -func (d keystone) RoleName(id string) (string, error) { +func (d Keystone) RoleName(id string) (string, error) { cachedName, hit := getFromCache(roleNameCache, id) if hit { return cachedName, nil @@ -403,7 +286,7 @@ func (d keystone) RoleName(id string) (string, error) { } var data struct { - Role KeystoneNameId `json:"role"` + Role keystoneNameId `json:"role"` } err = result.ExtractInto(&data) if err == nil { @@ -412,7 +295,7 @@ func (d keystone) RoleName(id string) (string, error) { return data.Role.Name, err } -func (d keystone) GroupName(id string) (string, error) { +func (d Keystone) GroupName(id string) (string, error) { cachedName, hit := getFromCache(groupNameCache, id) if hit { return cachedName, nil @@ -431,7 +314,7 @@ func (d keystone) GroupName(id string) (string, error) { } var data struct { - Group KeystoneNameId `json:"group"` + Group keystoneNameId `json:"group"` } err = result.ExtractInto(&data) if err == nil { @@ -440,22 +323,26 @@ func (d keystone) GroupName(id string) (string, error) { return data.Group.Name, err } -type keystoneToken struct { - DomainScope keystoneTokenThing `json:"domain"` - ProjectScope keystoneTokenThingInDomain `json:"project"` - Roles []keystoneTokenThing `json:"roles"` - User keystoneTokenThingInDomain `json:"user"` - ExpiresAt string `json:"expires_at"` -} - -type keystoneTokenThing struct { - ID string `json:"id"` - Name string `json:"name"` -} - -type keystoneTokenThingInDomain struct { - keystoneTokenThing - Domain keystoneTokenThing `json:"domain"` +func (d Keystone) updateCaches(token *keystoneToken, tokenStr string) { + addTokenToCache(tokenCache, tokenStr, token) + if token.DomainScope.ID != "" && token.DomainScope.Name != "" { + updateCache(domainNameCache, token.DomainScope.ID, token.DomainScope.Name) + } + if token.ProjectScope.Domain.ID != "" && token.ProjectScope.Domain.Name != "" { + updateCache(domainNameCache, token.ProjectScope.Domain.ID, token.ProjectScope.Domain.Name) + } + if token.ProjectScope.ID != "" && token.ProjectScope.Name != "" { + updateCache(projectNameCache, token.ProjectScope.ID, token.ProjectScope.Name) + } + if token.User.ID != "" && token.User.Name != "" { + updateCache(userNameCache, token.User.ID, token.User.Name) + updateCache(userIdCache, token.User.Name, token.User.ID) + } + for _, role := range token.Roles { + if role.ID != "" && role.Name != "" { + updateCache(roleNameCache, role.ID, role.Name) + } + } } func (t *keystoneToken) ToContext() policy.Context { @@ -495,16 +382,16 @@ func (t *keystoneToken) ToContext() policy.Context { return c } -//RefreshToken fetches a new Keystone auth token. It is also used +//RefreshToken fetches a new Identity auth token. It is also used //to fetch the initial token on startup. -func (d keystone) RefreshToken() error { +func (d Keystone) RefreshToken() error { //NOTE: This function is very similar to v3auth() in //gophercloud/openstack/client.go, but with a few differences: // //1. thread-safe token renewal //2. proper support for cross-domain scoping - util.LogDebug("Getting service user Keystone token...") + util.LogDebug("Getting service user Identity token...") d.TokenRenewalMutex.Lock() defer d.TokenRenewalMutex.Unlock() @@ -515,10 +402,10 @@ func (d keystone) RefreshToken() error { eo := gophercloud.EndpointOpts{Region: ""} keystone, err := openstack.NewIdentityV3(providerClient, eo) if err != nil { - return fmt.Errorf("cannot initialize Keystone client: %v", err) + return fmt.Errorf("cannot initialize Identity client: %v", err) } - util.LogDebug("Keystone URL: %s", keystone.Endpoint) + util.LogDebug("Identity URL: %s", keystone.Endpoint) result := tokens.Create(keystone, d.AuthOptions()) token, err := result.ExtractToken() @@ -539,13 +426,13 @@ func (d keystone) RefreshToken() error { return nil } -func (d keystone) AuthOptions() *gophercloud.AuthOptions { +func (d Keystone) AuthOptions() *gophercloud.AuthOptions { return &gophercloud.AuthOptions{ - IdentityEndpoint: viper.GetString("keystone.auth_url"), - Username: viper.GetString("keystone.username"), - Password: viper.GetString("keystone.password"), - DomainName: viper.GetString("keystone.user_domain_name"), + IdentityEndpoint: viper.GetString("Keystone.auth_url"), + Username: viper.GetString("Keystone.username"), + Password: viper.GetString("Keystone.password"), + DomainName: viper.GetString("Keystone.user_domain_name"), // Note: gophercloud only allows for user & project in the same domain - TenantName: viper.GetString("keystone.project_name"), + TenantName: viper.GetString("Keystone.project_name"), } } diff --git a/pkg/identity/keystone_cache.go b/pkg/identity/keystone_cache.go new file mode 100644 index 00000000..36771609 --- /dev/null +++ b/pkg/identity/keystone_cache.go @@ -0,0 +1,138 @@ +package identity + +import ( + "container/list" + "time" + "sync" + "github.com/gophercloud/gophercloud" + "github.com/sapcc/hermes/pkg/util" +) + +// Cache type used for the name caches +type cache struct { + // "Inherit from" sync.RWMutex, so the cache can be locked during access/update + sync.RWMutex + // The actual cache - a simple map with no expiry + // (the total number of items will only be in the 10000s, ~100 bytes per item, so ~1Mb per cache) + m map[string]string +} + +var providerClient *gophercloud.ProviderClient +var domainNameCache *cache +var projectNameCache *cache +var userNameCache *cache +var userIdCache *cache +var roleNameCache *cache +var groupNameCache *cache + +// Token cache +type keystoneTokenCache struct { + // "Inherit from" sync.RWMutex, so the cache can be locked during access/update + sync.RWMutex + // tMap: Cached tokens (keystoneToken struct) accessible by the token ID from the request + tMap map[string]*keystoneToken // map tokenID to token struct + // eList: A sorted list of token expiry times, so we don't have to scan the whole list + // every time we check to see what's expired + eList *list.List // sorted list of expiration times + // eMap: If we know a token is expired at time T, we use this map to look up the tokenID + // so we can then remove the token from tMap. + eMap map[time.Time][]string // map expiration time to list of tokenIDs +} + +var tokenCache *keystoneTokenCache + +func init() { + domainNameCache = &cache{m: make(map[string]string)} + projectNameCache = &cache{m: make(map[string]string)} + userNameCache = &cache{m: make(map[string]string)} + userIdCache = &cache{m: make(map[string]string)} + roleNameCache = &cache{m: make(map[string]string)} + groupNameCache = &cache{m: make(map[string]string)} + tokenCache = &keystoneTokenCache{ + tMap: make(map[string]*keystoneToken), + eMap: make(map[time.Time][]string), + eList: list.New(), + } +} + +func updateCache(cache *cache, key string, value string) { + cache.Lock() + cache.m[key] = value + cache.Unlock() +} + +func getFromCache(cache *cache, key string) (string, bool) { + cache.RLock() + value, exists := cache.m[key] + cache.RUnlock() + return value, exists +} + +func addTokenToCache(cache *keystoneTokenCache, id string, token *keystoneToken) { + expiryTime, err := time.Parse("2006-01-02T15:04:05.999999Z", token.ExpiresAt) + if err != nil { + util.LogWarning("Not adding token to cache because time '%s' could not be parsed", token.ExpiresAt) + return + } + cache.Lock() + cache.eList.PushBack(expiryTime) + // If the expiryTime is earlier than the last item in the list, + // move it to the correct place so that we keep the list sorted + lastItem := cache.eList.Back() + if cache.eList.Back() != nil && expiryTime.Before(lastItem.Value.(time.Time)) { + for e := cache.eList.Back(); e != nil; e = e.Prev() { + if expiryTime.After((e.Value).(time.Time)) { + cache.eList.MoveAfter(cache.eList.Back(), e) + } + } + } + if cache.eMap[expiryTime] == nil { + cache.eMap[expiryTime] = []string{id} + } else { + cache.eMap[expiryTime] = append(cache.eMap[expiryTime], id) + } + cache.tMap[id] = token + cacheSize := len(cache.tMap) + cache.Unlock() + util.LogDebug("Added token to cache. Current cache size: %d", cacheSize) +} + +func getCachedToken(cache *keystoneTokenCache, id string) *keystoneToken { + // First, remove expired tokens from cache + now := time.Now() + elemsToRemove := []*list.Element{} + cache.RLock() + for e := cache.eList.Front(); e != nil; e = e.Next() { + expiryTime := (e.Value).(time.Time) + if now.Before(expiryTime) { + break // list is sorted, so we can stop once we get to an unexpired token + } + // We can't remove from the list during the for loop, so remember which ones to delete + elemsToRemove = append(elemsToRemove, e) + } + cache.RUnlock() + cache.Lock() + for _, elem := range elemsToRemove { + cache.eList.Remove(elem) // Remove the cached expiry time from the sorted list + time := (elem.Value).(time.Time) + tokenIds := cache.eMap[time] + delete(cache.eMap, time) // Remove the cached expiry time from the time:tokenIDs map + for _, tokenId := range tokenIds { + delete(cache.tMap, tokenId) // Remove all the cached tokens + } + } + cacheSize := len(cache.tMap) + cache.Unlock() + if len(elemsToRemove) > 0 { + util.LogDebug("Removed expired token(s) from cache. Current cache size: %d", cacheSize) + } + // Now look for the token in question + cache.RLock() + token := cache.tMap[id] + cache.RUnlock() + if token != nil { + util.LogDebug("Got token from cache. Current cache size: %d", cacheSize) + } + + return token +} diff --git a/pkg/identity/mock.go b/pkg/identity/mock.go new file mode 100644 index 00000000..ab656661 --- /dev/null +++ b/pkg/identity/mock.go @@ -0,0 +1,61 @@ +package identity + +import ( + "github.com/databus23/goslo.policy" + "github.com/gophercloud/gophercloud" + "github.com/spf13/viper" +) + +type Mock struct{} + +func (d Mock) keystoneClient() (*gophercloud.ServiceClient, error) { + return nil, nil +} + +func (d Mock) Client() *gophercloud.ProviderClient { + return nil +} + +func (d Mock) ValidateToken(token string) (policy.Context, error) { + + return policy.Context{}, nil +} + +func (d Mock) Authenticate(credentials *gophercloud.AuthOptions) (policy.Context, error) { + return policy.Context{}, nil +} + +func (d Mock) DomainName(id string) (string, error) { + return "monsoon3", nil +} + +func (d Mock) ProjectName(id string) (string, error) { + return "ceilometer-cadf-delete-me", nil +} + +func (d Mock) UserName(id string) (string, error) { + return "I056593", nil +} + +func (d Mock) UserId(name string) (string, error) { + return "eb5cd8f904b06e8b2a6eb86c8b04c08e6efb89b92da77905cc8c475f30b0b812", nil +} + +func (d Mock) RoleName(id string) (string, error) { + return "audit_viewer", nil +} + +func (d Mock) GroupName(id string) (string, error) { + return "admins", nil +} + +func (d Mock) AuthOptions() *gophercloud.AuthOptions { + return &gophercloud.AuthOptions{ + IdentityEndpoint: viper.GetString("Keystone.auth_url"), + Username: viper.GetString("Keystone.username"), + Password: viper.GetString("Keystone.password"), + DomainName: viper.GetString("Keystone.user_domain_name"), + // Note: gophercloud only allows for user & project in the same domain + TenantName: viper.GetString("Keystone.project_name"), + } +} diff --git a/pkg/keystone/mock.go b/pkg/keystone/mock.go deleted file mode 100644 index 2483b917..00000000 --- a/pkg/keystone/mock.go +++ /dev/null @@ -1,66 +0,0 @@ -package keystone - -import ( - "github.com/databus23/goslo.policy" - "github.com/gophercloud/gophercloud" - "github.com/spf13/viper" -) - -type mock struct{} - -// Mock keystone implementation -func Mock() Driver { - return mock{} -} - -func (d mock) keystoneClient() (*gophercloud.ServiceClient, error) { - return nil, nil -} - -func (d mock) Client() *gophercloud.ProviderClient { - return nil -} - -func (d mock) ValidateToken(token string) (policy.Context, error) { - - return policy.Context{}, nil -} - -func (d mock) Authenticate(credentials *gophercloud.AuthOptions) (policy.Context, error) { - return policy.Context{}, nil -} - -func (d mock) DomainName(id string) (string, error) { - return "monsoon3", nil -} - -func (d mock) ProjectName(id string) (string, error) { - return "ceilometer-cadf-delete-me", nil -} - -func (d mock) UserName(id string) (string, error) { - return "I056593", nil -} - -func (d mock) UserId(name string) (string, error) { - return "eb5cd8f904b06e8b2a6eb86c8b04c08e6efb89b92da77905cc8c475f30b0b812", nil -} - -func (d mock) RoleName(id string) (string, error) { - return "audit_viewer", nil -} - -func (d mock) GroupName(id string) (string, error) { - return "admins", nil -} - -func (d mock) AuthOptions() *gophercloud.AuthOptions { - return &gophercloud.AuthOptions{ - IdentityEndpoint: viper.GetString("keystone.auth_url"), - Username: viper.GetString("keystone.username"), - Password: viper.GetString("keystone.password"), - DomainName: viper.GetString("keystone.user_domain_name"), - // Note: gophercloud only allows for user & project in the same domain - TenantName: viper.GetString("keystone.project_name"), - } -} diff --git a/pkg/policy/policy_test.go b/pkg/policy/policy_test.go index 49212767..c600e46e 100644 --- a/pkg/policy/policy_test.go +++ b/pkg/policy/policy_test.go @@ -8,12 +8,10 @@ import ( "github.com/stretchr/testify/assert" ) - // Setup test func GetEnforcer() *policy.Enforcer { const path = "../../etc/policy.json" - policyenforcer, err := util.LoadPolicyFile(path) if err != nil { @@ -45,10 +43,10 @@ func Test_Policy_AuditViewerTrue(t *testing.T) { "tenant_domain_name": "aaaa", }, Request: map[string]string{ - "domain_id": "ca1b267e149d4e44bf53d28d1c8d6bc9", - "project_id": "7a09c05926ec452ca7992af4aa03c31d", + "domain_id": "ca1b267e149d4e44bf53d28d1c8d6bc9", + "project_id": "7a09c05926ec452ca7992af4aa03c31d", }, - Logger: util.LogDebug, + Logger: util.LogDebug, } assert.True(t, policyenforcer.Enforce("event:show", c)) } @@ -76,10 +74,10 @@ func Test_Policy_UnknownRoleFalse(t *testing.T) { "tenant_domain_name": "aaaa", }, Request: map[string]string{ - "domain_id": "ca1b267e149d4e44bf53d28d1c8d6bc9", - "project_id": "7a09c05926ec452ca7992af4aa03c31d", + "domain_id": "ca1b267e149d4e44bf53d28d1c8d6bc9", + "project_id": "7a09c05926ec452ca7992af4aa03c31d", }, - Logger: util.LogDebug, + Logger: util.LogDebug, } assert.False(t, policyenforcer.Enforce("event:show", c)) } diff --git a/pkg/storage/elasticsearch.go b/pkg/storage/elasticsearch.go index c69a384c..331b3764 100644 --- a/pkg/storage/elasticsearch.go +++ b/pkg/storage/elasticsearch.go @@ -9,25 +9,19 @@ import ( elastic "gopkg.in/olivere/elastic.v5" ) -type elasticSearch struct { +type ElasticSearch struct { esClient *elastic.Client } -var es elasticSearch - -// Initialise and return the ES driver -func ElasticSearch() Driver { - return es -} - -func (es *elasticSearch) client() *elastic.Client { +func (es *ElasticSearch) client() *elastic.Client { + // Lazy initialisation - don't connect to ElasticSearch until we need to if es.esClient == nil { es.init() } return es.esClient } -func (es *elasticSearch) init() { +func (es *ElasticSearch) init() { util.LogDebug("Initiliasing ElasticSearch()") // Create a client @@ -40,7 +34,7 @@ func (es *elasticSearch) init() { } } -func (es elasticSearch) GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, error) { +func (es ElasticSearch) GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, error) { index := indexName(tenantId) util.LogDebug("Looking for events in index %s", index) @@ -125,7 +119,7 @@ func (es elasticSearch) GetEvents(filter *Filter, tenantId string) ([]*EventDeta return events, int(total), nil } -func (es elasticSearch) GetEvent(eventId string, tenantId string) (*EventDetail, error) { +func (es ElasticSearch) GetEvent(eventId string, tenantId string) (*EventDetail, error) { index := indexName(tenantId) util.LogDebug("Looking for event %s in index %s", eventId, index) @@ -149,7 +143,7 @@ func (es elasticSearch) GetEvent(eventId string, tenantId string) (*EventDetail, return nil, nil } -func (es elasticSearch) MaxLimit() uint { +func (es ElasticSearch) MaxLimit() uint { return uint(viper.GetInt("elasticsearch.max_result_window")) } diff --git a/pkg/storage/interface.go b/pkg/storage/interface.go index 540b37bd..9addc67e 100644 --- a/pkg/storage/interface.go +++ b/pkg/storage/interface.go @@ -19,9 +19,9 @@ package storage -// Driver is an interface that wraps the underlying event storage mechanism. +// Storage is an interface that wraps the underlying event storage mechanism. // Because it is an interface, the real implementation can be mocked away in unit tests. -type Driver interface { +type Storage interface { /********** requests to ElasticSearch **********/ GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, error) @@ -50,12 +50,14 @@ type Filter struct { // Thanks to the tool at https://mholt.github.io/json-to-go/ +// The JSON annotations are for parsing the result from ElasticSearch type eventListWithTotal struct { Total int `json:"total"` Events []EventDetail `json:"events"` } // EventDetail contains the CADF payload, enhanced with names for IDs +// The JSON annotations are for parsing the result from ElasticSearch AND for generating the Hermes API response type EventDetail struct { PublisherID string `json:"publisher_id"` EventType string `json:"event_type"` diff --git a/pkg/storage/mock.go b/pkg/storage/mock.go index cdb5812b..915142a8 100644 --- a/pkg/storage/mock.go +++ b/pkg/storage/mock.go @@ -4,14 +4,10 @@ import ( "encoding/json" ) -type mock struct{} - // Mock elasticsearch driver with static data -func Mock() Driver { - return mock{} -} +type Mock struct{} -func (m mock) GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, error) { +func (m Mock) GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, error) { var detailedEvents eventListWithTotal json.Unmarshal(mockEvents, &detailedEvents) @@ -24,13 +20,13 @@ func (m mock) GetEvents(filter *Filter, tenantId string) ([]*EventDetail, int, e return events, detailedEvents.Total, nil } -func (m mock) GetEvent(eventId string, tenantId string) (*EventDetail, error) { +func (m Mock) GetEvent(eventId string, tenantId string) (*EventDetail, error) { var parsedEvent EventDetail err := json.Unmarshal(mockEvent, &parsedEvent) return &parsedEvent, err } -func (m mock) MaxLimit() uint { +func (m Mock) MaxLimit() uint { return 100 } diff --git a/pkg/storage/mock_test.go b/pkg/storage/mock_test.go index d9c65ff5..f67a52d1 100644 --- a/pkg/storage/mock_test.go +++ b/pkg/storage/mock_test.go @@ -6,7 +6,7 @@ import ( ) func Test_MockStorage_EventDetail(t *testing.T) { - eventDetail, error := Mock().GetEvent("d5eed458-6666-58ec-ad06-8d3cf6bafca1", "b3b70c8271a845709f9a03030e705da7") + eventDetail, error := Mock{}.GetEvent("d5eed458-6666-58ec-ad06-8d3cf6bafca1", "b3b70c8271a845709f9a03030e705da7") assert.Nil(t, error) assert.Equal(t, "d5eed458-6666-58ec-ad06-8d3cf6bafca1", eventDetail.Payload.ID) @@ -15,7 +15,7 @@ func Test_MockStorage_EventDetail(t *testing.T) { } func Test_MockStorage_Events(t *testing.T) { - eventsList, total, error := Mock().GetEvents(&Filter{}, "b3b70c8271a845709f9a03030e705da7") + eventsList, total, error := Mock{}.GetEvents(&Filter{}, "b3b70c8271a845709f9a03030e705da7") assert.Nil(t, error) assert.Equal(t, total, 24)