From 4c289dee409584c41b735e942b185ec14972dcb4 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 12:09:45 +0200 Subject: [PATCH 01/12] generic local cache access --- agent/cache.go | 127 +++++++++++++++++++++++++++++++++++++++++ agent/remote_config.go | 100 +------------------------------- 2 files changed, 129 insertions(+), 98 deletions(-) create mode 100644 agent/cache.go diff --git a/agent/cache.go b/agent/cache.go new file mode 100644 index 00000000..d88d3c32 --- /dev/null +++ b/agent/cache.go @@ -0,0 +1,127 @@ +package agent + +import ( + "crypto/sha1" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/mitchellh/go-homedir" +) + +const cacheTimeout = 5 * time.Minute + +func (a *Agent) getOrSetLocalCacheData(metadata map[string]interface{}, key string, useTimeout bool, fn func(map[string]interface{}) interface{}) interface{} { + if metadata == nil { + return nil + } + + path, err := getLocalCacheFilePath(metadata, key) + if err != nil { + a.logger.Printf("Local cache: %v", err) + return fn(metadata) + } + + // Loader function + loaderFunc := func(metadata map[string]interface{}, err error, fn func(map[string]interface{}) interface{}) interface{} { + if err != nil { + a.logger.Printf("Local cache: %v", err) + } + if fn == nil { + return nil + } + + // Call the loader + resp := fn(metadata) + + if resp != nil { + // Get local cache file path + path, err := getLocalCacheFilePath(metadata, key) + if err != nil { + return resp + } + + // Save a local cache for the response + if data, err := json.Marshal(&resp); err == nil { + if a.debugMode { + a.logger.Printf("Local cache saving: %s => %s", path, string(data)) + } + if err := ioutil.WriteFile(path, data, 0755); err != nil { + a.logger.Printf("Error writing json file: %v", err) + } + } + } + + return resp + } + + // We try to load the cached version of the remote configuration + file, err := os.Open(path) + if err != nil { + return loaderFunc(metadata, err, fn) + } + defer file.Close() + + // Checks if the cache data is old + if useTimeout { + fInfo, err := file.Stat() + if err != nil || time.Now().Sub(fInfo.ModTime()) > cacheTimeout { + return loaderFunc(metadata, err, fn) + } + } + + // Read the cached value + fileBytes, err := ioutil.ReadAll(file) + if err != nil { + return loaderFunc(metadata, err, fn) + } + + // Unmarshal json data + var res map[string]interface{} + if err := json.Unmarshal(fileBytes, &res); err != nil { + return loaderFunc(metadata, err, fn) + } else { + if a.debugMode { + a.logger.Printf("Local cache loading: %s => %s", path, string(fileBytes)) + } else { + a.logger.Printf("Local cache loading: %s", path) + } + return res + } +} + +// Gets the local cache file path +func getLocalCacheFilePath(metadata map[string]interface{}, key string) (string, error) { + homeDir, err := homedir.Dir() + if err != nil { + return "", err + } + data, err := json.Marshal(metadata) + if err != nil { + return "", err + } + hash := fmt.Sprintf("%x", sha1.Sum(data)) + + var folder string + if runtime.GOOS == "windows" { + folder = fmt.Sprintf("%s/AppData/Roaming/scope/cache", homeDir) + } else { + folder = fmt.Sprintf("%s/.scope/cache", homeDir) + } + + if _, err := os.Stat(folder); err == nil { + return filepath.Join(folder, fmt.Sprintf("%s.%s", hash, key)), nil + } else if os.IsNotExist(err) { + err = os.MkdirAll(folder, 0755) + if err != nil { + return "", err + } + return filepath.Join(folder, fmt.Sprintf("%s.%s", hash, key)), nil + } else { + return "", err + } +} diff --git a/agent/remote_config.go b/agent/remote_config.go index 18250af2..fe9b1641 100644 --- a/agent/remote_config.go +++ b/agent/remote_config.go @@ -2,7 +2,6 @@ package agent import ( "bytes" - "crypto/sha1" "crypto/x509" "encoding/json" "errors" @@ -10,13 +9,8 @@ import ( "io/ioutil" "net/http" "net/url" - "os" - "path/filepath" - "runtime" "time" - "github.com/mitchellh/go-homedir" - "go.undefinedlabs.com/scopeagent/tags" ) @@ -42,11 +36,11 @@ func (a *Agent) loadRemoteConfiguration() map[string]interface{} { jsBytes, _ := json.Marshal(configRequest) a.logger.Printf("Getting remote configuration for: %v", string(jsBytes)) } - return a.getOrSetRemoteConfigurationCache(configRequest, a.getRemoteConfiguration) + return a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration).(map[string]interface{}) } // Gets the remote agent configuration from the endpoint + api/agent/config -func (a *Agent) getRemoteConfiguration(cfgRequest map[string]interface{}) map[string]interface{} { +func (a *Agent) getRemoteConfiguration(cfgRequest map[string]interface{}) interface{} { client := &http.Client{} curl := a.getUrl("api/agent/config") payload, err := msgPackEncodePayload(cfgRequest) @@ -138,93 +132,3 @@ func (a *Agent) getRemoteConfiguration(cfgRequest map[string]interface{}) map[st } return nil } - -// Gets or sets the remote agent configuration local cache -func (a *Agent) getOrSetRemoteConfigurationCache(metadata map[string]interface{}, fn func(map[string]interface{}) map[string]interface{}) map[string]interface{} { - if metadata == nil { - return nil - } - var ( - path string - err error - ) - path, err = getRemoteConfigurationCachePath(metadata) - if err == nil { - // We try to load the cached version of the remote configuration - file, lerr := os.Open(path) - err = lerr - if lerr == nil { - defer file.Close() - fileBytes, lerr := ioutil.ReadAll(file) - err = lerr - if lerr == nil { - var res map[string]interface{} - if lerr = json.Unmarshal(fileBytes, &res); lerr == nil { - if a.debugMode { - a.logger.Printf("Remote configuration cache: %v", string(fileBytes)) - } else { - a.logger.Printf("Remote configuration cache: %v", path) - } - return res - } else { - err = lerr - } - } - } - } - if err != nil { - a.logger.Printf("Remote configuration cache: %v", err) - } - - if fn == nil { - return nil - } - - // Call the loader - resp := fn(metadata) - - if resp != nil && path != "" { - // Save a local cache for the response - if data, err := json.Marshal(&resp); err == nil { - if a.debugMode { - a.logger.Printf("Saving Remote configuration cache: %v", string(data)) - } - if err := ioutil.WriteFile(path, data, 0755); err != nil { - a.logger.Printf("Error writing json file: %v", err) - } - } - } - return resp -} - -// Gets the remote agent configuration local cache path -func getRemoteConfigurationCachePath(metadata map[string]interface{}) (string, error) { - homeDir, err := homedir.Dir() - if err != nil { - return "", err - } - data, err := json.Marshal(metadata) - if err != nil { - return "", err - } - hash := fmt.Sprintf("%x", sha1.Sum(data)) - - var folder string - if runtime.GOOS == "windows" { - folder = fmt.Sprintf("%s/AppData/Roaming/scope/cache", homeDir) - } else { - folder = fmt.Sprintf("%s/.scope/cache", homeDir) - } - - if _, err := os.Stat(folder); err == nil { - return filepath.Join(folder, hash), nil - } else if os.IsNotExist(err) { - err = os.MkdirAll(folder, 0755) - if err != nil { - return "", err - } - return filepath.Join(folder, hash), nil - } else { - return "", err - } -} From 2de5fdc8ff58406c77e59ddae63608faba1ff6ed Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 12:19:51 +0200 Subject: [PATCH 02/12] improvements --- agent/remote_config.go | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/agent/remote_config.go b/agent/remote_config.go index fe9b1641..7d91f8b1 100644 --- a/agent/remote_config.go +++ b/agent/remote_config.go @@ -16,6 +16,27 @@ import ( // Loads the remote agent configuration from local cache, if not exists then retrieve it from the server func (a *Agent) loadRemoteConfiguration() map[string]interface{} { + if a == nil || a.metadata == nil { + return nil + } + configRequest := a.getRemoteConfigRequest() + if a.debugMode { + jsBytes, _ := json.Marshal(configRequest) + a.logger.Printf("Getting remote configuration for: %v", string(jsBytes)) + } + + a.getOrSetLocalCacheData(configRequest, "test", true, func(m map[string]interface{}) interface{} { + return "Hello World" + }) + + config := a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration) + if config != nil { + return config.(map[string]interface{}) + } + return nil +} + +func (a *Agent) getRemoteConfigRequest() map[string]interface{} { if a == nil || a.metadata == nil { return nil } @@ -32,11 +53,7 @@ func (a *Agent) loadRemoteConfiguration() map[string]interface{} { addElementToMapIfEmpty(configRequest, item, a.metadata[item]) } } - if a.debugMode { - jsBytes, _ := json.Marshal(configRequest) - a.logger.Printf("Getting remote configuration for: %v", string(jsBytes)) - } - return a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration).(map[string]interface{}) + return configRequest } // Gets the remote agent configuration from the endpoint + api/agent/config From 59b904da0b343a8ae3bd6c6501d71db6fc83f26e Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 12:33:12 +0200 Subject: [PATCH 03/12] fixes and improvements in the local cache loader --- agent/cache.go | 8 +++++++- agent/remote_config.go | 11 +++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/agent/cache.go b/agent/cache.go index d88d3c32..25197b15 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -3,6 +3,7 @@ package agent import ( "crypto/sha1" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -69,7 +70,12 @@ func (a *Agent) getOrSetLocalCacheData(metadata map[string]interface{}, key stri // Checks if the cache data is old if useTimeout { fInfo, err := file.Stat() - if err != nil || time.Now().Sub(fInfo.ModTime()) > cacheTimeout { + if err != nil { + return loaderFunc(metadata, err, fn) + } + sTime := time.Now().Sub(fInfo.ModTime()) + if sTime > cacheTimeout { + err = errors.New(fmt.Sprintf("The local cache key '%s' has timeout: %v", path, sTime)) return loaderFunc(metadata, err, fn) } } diff --git a/agent/remote_config.go b/agent/remote_config.go index 7d91f8b1..9ce25d69 100644 --- a/agent/remote_config.go +++ b/agent/remote_config.go @@ -24,14 +24,9 @@ func (a *Agent) loadRemoteConfiguration() map[string]interface{} { jsBytes, _ := json.Marshal(configRequest) a.logger.Printf("Getting remote configuration for: %v", string(jsBytes)) } - - a.getOrSetLocalCacheData(configRequest, "test", true, func(m map[string]interface{}) interface{} { - return "Hello World" - }) - - config := a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration) - if config != nil { - return config.(map[string]interface{}) + configResponse := a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration) + if configResponse != nil { + return configResponse.(map[string]interface{}) } return nil } From f6c18bea39d4f085559e63ad744049e34a060dfd Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 14:05:32 +0200 Subject: [PATCH 04/12] new cache struct --- agent/agent.go | 7 +++- agent/cache.go | 92 +++++++++++++++++++++++++----------------- agent/recorder.go | 2 + agent/remote_config.go | 15 ++++--- agent/util.go | 2 +- 5 files changed, 72 insertions(+), 46 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 3b26e199..02bd52e1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -56,13 +56,15 @@ type ( logger *log.Logger printReportOnce sync.Once + + cache *localCache } Option func(*Agent) ) var ( - version = "0.3.1-pre1" + version = "0.3.1-pre2" testingModeFrequency = time.Second nonTestingModeFrequency = time.Minute @@ -376,6 +378,9 @@ func NewAgent(options ...Option) (*Agent, error) { agent.logMetadata() } + // + agent.cache = newLocalCache(agent.getRemoteConfigRequest(), cacheTimeout, agent.debugMode, agent.logger) + agent.recorder = NewSpanRecorder(agent) var recorder tracer.SpanRecorder = agent.recorder if agent.optionalRecorders != nil { diff --git a/agent/cache.go b/agent/cache.go index 25197b15..6adf1f38 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "io/ioutil" + "log" "os" "path/filepath" "runtime" + "sync" "time" "github.com/mitchellh/go-homedir" @@ -16,43 +18,54 @@ import ( const cacheTimeout = 5 * time.Minute -func (a *Agent) getOrSetLocalCacheData(metadata map[string]interface{}, key string, useTimeout bool, fn func(map[string]interface{}) interface{}) interface{} { - if metadata == nil { - return nil +type ( + localCache struct { + m sync.Mutex + tenant interface{} + basePath string + timeout time.Duration + debugMode bool + logger *log.Logger } +) - path, err := getLocalCacheFilePath(metadata, key) - if err != nil { - a.logger.Printf("Local cache: %v", err) - return fn(metadata) +// Create a new local cache +func newLocalCache(tenant map[string]interface{}, timeout time.Duration, debugMode bool, logger *log.Logger) *localCache { + lc := &localCache{ + timeout: timeout, + debugMode: debugMode, + logger: logger, } + lc.SetTenant(tenant) + return lc +} + +// Gets or sets a local cache value +func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, string) interface{}) interface{} { + c.m.Lock() + defer c.m.Unlock() // Loader function - loaderFunc := func(metadata map[string]interface{}, err error, fn func(map[string]interface{}) interface{}) interface{} { + loaderFunc := func(key string, err error, fn func(interface{}, string) interface{}) interface{} { if err != nil { - a.logger.Printf("Local cache: %v", err) + c.logger.Printf("Local cache: %v", err) } if fn == nil { return nil } // Call the loader - resp := fn(metadata) + resp := fn(c.tenant, key) if resp != nil { - // Get local cache file path - path, err := getLocalCacheFilePath(metadata, key) - if err != nil { - return resp - } - + path := fmt.Sprintf("%s.%s", c.basePath, key) // Save a local cache for the response if data, err := json.Marshal(&resp); err == nil { - if a.debugMode { - a.logger.Printf("Local cache saving: %s => %s", path, string(data)) + if c.debugMode { + c.logger.Printf("Local cache saving: %s => %s", path, string(data)) } if err := ioutil.WriteFile(path, data, 0755); err != nil { - a.logger.Printf("Error writing json file: %v", err) + c.logger.Printf("Error writing json file: %v", err) } } } @@ -60,10 +73,12 @@ func (a *Agent) getOrSetLocalCacheData(metadata map[string]interface{}, key stri return resp } + path := fmt.Sprintf("%s.%s", c.basePath, key) + // We try to load the cached version of the remote configuration file, err := os.Open(path) if err != nil { - return loaderFunc(metadata, err, fn) + return loaderFunc(key, err, fn) } defer file.Close() @@ -71,44 +86,46 @@ func (a *Agent) getOrSetLocalCacheData(metadata map[string]interface{}, key stri if useTimeout { fInfo, err := file.Stat() if err != nil { - return loaderFunc(metadata, err, fn) + return loaderFunc(key, err, fn) } sTime := time.Now().Sub(fInfo.ModTime()) if sTime > cacheTimeout { err = errors.New(fmt.Sprintf("The local cache key '%s' has timeout: %v", path, sTime)) - return loaderFunc(metadata, err, fn) + return loaderFunc(key, err, fn) } } // Read the cached value fileBytes, err := ioutil.ReadAll(file) if err != nil { - return loaderFunc(metadata, err, fn) + return loaderFunc(key, err, fn) } // Unmarshal json data var res map[string]interface{} if err := json.Unmarshal(fileBytes, &res); err != nil { - return loaderFunc(metadata, err, fn) + return loaderFunc(key, err, fn) } else { - if a.debugMode { - a.logger.Printf("Local cache loading: %s => %s", path, string(fileBytes)) + if c.debugMode { + c.logger.Printf("Local cache loading: %s => %s", path, string(fileBytes)) } else { - a.logger.Printf("Local cache loading: %s", path) + c.logger.Printf("Local cache loading: %s", path) } return res } } -// Gets the local cache file path -func getLocalCacheFilePath(metadata map[string]interface{}, key string) (string, error) { +// Sets the local cache tenant +func (c *localCache) SetTenant(tenant interface{}) { homeDir, err := homedir.Dir() if err != nil { - return "", err + c.logger.Printf("local cache error: %v", err) + return } - data, err := json.Marshal(metadata) + data, err := json.Marshal(tenant) if err != nil { - return "", err + c.logger.Printf("local cache error: %v", err) + return } hash := fmt.Sprintf("%x", sha1.Sum(data)) @@ -120,14 +137,17 @@ func getLocalCacheFilePath(metadata map[string]interface{}, key string) (string, } if _, err := os.Stat(folder); err == nil { - return filepath.Join(folder, fmt.Sprintf("%s.%s", hash, key)), nil + c.tenant = tenant + c.basePath = filepath.Join(folder, hash) } else if os.IsNotExist(err) { err = os.MkdirAll(folder, 0755) if err != nil { - return "", err + c.logger.Printf("local cache error: %v", err) + return } - return filepath.Join(folder, fmt.Sprintf("%s.%s", hash, key)), nil + c.tenant = tenant + c.basePath = filepath.Join(folder, hash) } else { - return "", err + c.logger.Printf("local cache error: %v", err) } } diff --git a/agent/recorder.go b/agent/recorder.go index 103a37ad..5d994aad 100644 --- a/agent/recorder.go +++ b/agent/recorder.go @@ -46,6 +46,7 @@ type ( logger *log.Logger stats *RecorderStats statsOnce sync.Once + cache *localCache } RecorderStats struct { totalSpans int64 @@ -76,6 +77,7 @@ func NewSpanRecorder(agent *Agent) *SpanRecorder { r.debugMode = agent.debugMode r.metadata = agent.metadata r.logger = agent.logger + r.cache = agent.cache r.flushFrequency = agent.flushFrequency r.url = agent.getUrl("api/agent/ingest") r.client = &http.Client{} diff --git a/agent/remote_config.go b/agent/remote_config.go index 9ce25d69..858ce2e2 100644 --- a/agent/remote_config.go +++ b/agent/remote_config.go @@ -16,15 +16,10 @@ import ( // Loads the remote agent configuration from local cache, if not exists then retrieve it from the server func (a *Agent) loadRemoteConfiguration() map[string]interface{} { - if a == nil || a.metadata == nil { + if a == nil { return nil } - configRequest := a.getRemoteConfigRequest() - if a.debugMode { - jsBytes, _ := json.Marshal(configRequest) - a.logger.Printf("Getting remote configuration for: %v", string(jsBytes)) - } - configResponse := a.getOrSetLocalCacheData(configRequest, "remote", false, a.getRemoteConfiguration) + configResponse := a.cache.GetOrSet("remoteconfig", false, a.getRemoteConfiguration) if configResponse != nil { return configResponse.(map[string]interface{}) } @@ -48,11 +43,15 @@ func (a *Agent) getRemoteConfigRequest() map[string]interface{} { addElementToMapIfEmpty(configRequest, item, a.metadata[item]) } } + if a.debugMode { + jsBytes, _ := json.Marshal(configRequest) + a.logger.Printf("Configuration request: %v", string(jsBytes)) + } return configRequest } // Gets the remote agent configuration from the endpoint + api/agent/config -func (a *Agent) getRemoteConfiguration(cfgRequest map[string]interface{}) interface{} { +func (a *Agent) getRemoteConfiguration(cfgRequest interface{}, key string) interface{} { client := &http.Client{} curl := a.getUrl("api/agent/config") payload, err := msgPackEncodePayload(cfgRequest) diff --git a/agent/util.go b/agent/util.go index 2e34838f..223dc853 100644 --- a/agent/util.go +++ b/agent/util.go @@ -35,7 +35,7 @@ func getSourceRootFromEnv(key string) string { } // Encodes `payload` using msgpack and compress it with gzip -func msgPackEncodePayload(payload map[string]interface{}) (*bytes.Buffer, error) { +func msgPackEncodePayload(payload interface{}) (*bytes.Buffer, error) { binaryPayload, err := msgpack.Marshal(payload) if err != nil { return nil, err From 3807fe8c000791c1135f4d5a5d230b93e31db0d2 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 15:02:08 +0200 Subject: [PATCH 05/12] ntp using cache struct --- agent/cache.go | 12 ++++++++---- agent/ntp.go | 15 ++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/agent/cache.go b/agent/cache.go index 6adf1f38..7cd611ec 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -27,6 +27,9 @@ type ( debugMode bool logger *log.Logger } + cacheItem struct { + Value interface{} + } ) // Create a new local cache @@ -59,8 +62,9 @@ func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, if resp != nil { path := fmt.Sprintf("%s.%s", c.basePath, key) + cItem := &cacheItem{Value: resp} // Save a local cache for the response - if data, err := json.Marshal(&resp); err == nil { + if data, err := json.Marshal(cItem); err == nil { if c.debugMode { c.logger.Printf("Local cache saving: %s => %s", path, string(data)) } @@ -102,8 +106,8 @@ func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, } // Unmarshal json data - var res map[string]interface{} - if err := json.Unmarshal(fileBytes, &res); err != nil { + var cItem cacheItem + if err := json.Unmarshal(fileBytes, &cItem); err != nil { return loaderFunc(key, err, fn) } else { if c.debugMode { @@ -111,7 +115,7 @@ func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, } else { c.logger.Printf("Local cache loading: %s", path) } - return res + return cItem.Value } } diff --git a/agent/ntp.go b/agent/ntp.go index c70d213a..646da0db 100644 --- a/agent/ntp.go +++ b/agent/ntp.go @@ -38,12 +38,17 @@ func (r *SpanRecorder) applyNTPOffset(t time.Time) time.Time { if r.debugMode { r.logger.Println("calculating ntp offset.") } - offset, err := getNTPOffset() - if err == nil { - ntpOffset = offset + offset := r.cache.GetOrSet("ntp", true, func(d interface{}, s string) interface{} { + oSet, err := getNTPOffset() + if err != nil { + r.logger.Printf("error calculating the ntp offset: %v\n", err) + return nil + } + return oSet + }) + if offset != nil { + ntpOffset = offset.(time.Duration) r.logger.Printf("ntp offset: %v\n", ntpOffset) - } else { - r.logger.Printf("error calculating the ntp offset: %v\n", err) } }) return t.Add(ntpOffset) From 77a1fff94fb52f075f217ebb882e43f543c5c3a8 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 15:09:06 +0200 Subject: [PATCH 06/12] change the cache timeout default to 1 minute --- agent/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/cache.go b/agent/cache.go index 7cd611ec..0a5f9275 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -16,7 +16,7 @@ import ( "github.com/mitchellh/go-homedir" ) -const cacheTimeout = 5 * time.Minute +const cacheTimeout = 1 * time.Minute type ( localCache struct { From 60a7e7062bab06fbaa7ec14c803a97f5f22860ee Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 15:15:25 +0200 Subject: [PATCH 07/12] fix offset casting --- agent/ntp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/ntp.go b/agent/ntp.go index 646da0db..3bd050b8 100644 --- a/agent/ntp.go +++ b/agent/ntp.go @@ -47,7 +47,7 @@ func (r *SpanRecorder) applyNTPOffset(t time.Time) time.Time { return oSet }) if offset != nil { - ntpOffset = offset.(time.Duration) + ntpOffset = time.Duration(offset.(float64)) r.logger.Printf("ntp offset: %v\n", ntpOffset) } }) From ad651f0c780cbb9c787c6562e7eae807f2de7cd5 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 15:30:52 +0200 Subject: [PATCH 08/12] fix float64 casting --- agent/ntp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/ntp.go b/agent/ntp.go index 3bd050b8..13c6f310 100644 --- a/agent/ntp.go +++ b/agent/ntp.go @@ -44,7 +44,7 @@ func (r *SpanRecorder) applyNTPOffset(t time.Time) time.Time { r.logger.Printf("error calculating the ntp offset: %v\n", err) return nil } - return oSet + return float64(oSet) }) if offset != nil { ntpOffset = time.Duration(offset.(float64)) From 8c5ac06bed3321e9eff987aa5684557ead5da663 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 15:35:03 +0200 Subject: [PATCH 09/12] NTP request timeout increase --- agent/ntp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/ntp.go b/agent/ntp.go index 13c6f310..7a695cc3 100644 --- a/agent/ntp.go +++ b/agent/ntp.go @@ -9,7 +9,7 @@ import ( const ( server = "pool.ntp.org" retries = 5 - timeout = 1 * time.Second + timeout = 2 * time.Second backoff = 1 * time.Second ) From c5dd4a46c75a1de6749115687756ce153fa40455 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 16:34:24 +0200 Subject: [PATCH 10/12] cached test improvements --- agent/cache.go | 6 +- instrumentation/testing/testing.go | 110 +++++++++++++++++------------ 2 files changed, 67 insertions(+), 49 deletions(-) diff --git a/agent/cache.go b/agent/cache.go index 0a5f9275..174510c3 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -110,11 +110,7 @@ func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, if err := json.Unmarshal(fileBytes, &cItem); err != nil { return loaderFunc(key, err, fn) } else { - if c.debugMode { - c.logger.Printf("Local cache loading: %s => %s", path, string(fileBytes)) - } else { - c.logger.Printf("Local cache loading: %s", path) - } + c.logger.Printf("Local cache loaded: %s (%d bytes)", path, len(fileBytes)) return cItem.Value } } diff --git a/instrumentation/testing/testing.go b/instrumentation/testing/testing.go index 5903a39f..7b6dd78c 100644 --- a/instrumentation/testing/testing.go +++ b/instrumentation/testing/testing.go @@ -60,59 +60,81 @@ func StartTest(t *testing.T, opts ...Option) *Test { // Starts a new test with and uses the caller pc info for Name and Suite func StartTestFromCaller(t *testing.T, pc uintptr, opts ...Option) *Test { - // Get or create a new Test struct - // If we get an old struct we replace the current span and context with a new one. - // Useful if we want to overwrite the Start call with options - test, exist := getOrCreateTest(t) - if exist { - // If there is already one we want to replace it, so we clear the context - test.ctx = context.Background() - } - test.codePC = pc - - for _, opt := range opts { - opt(test) - } - - // Extracting the testing func name (by removing any possible sub-test suffix `{test_func}/{sub_test}`) - // to search the func source code bounds and to calculate the package name. - fullTestName := runner.GetOriginalTestName(t.Name()) - pName, _, testCode := getPackageAndNameAndBoundaries(pc) - - testTags := opentracing.Tags{ - "span.kind": "test", - "test.name": fullTestName, - "test.suite": pName, - "test.framework": "testing", - "test.language": "go", - } - - if testCode != "" { - testTags["test.code"] = testCode - } - if test.ctx == nil { - test.ctx = context.Background() - } + // check if the test is cached + if isTestCached(t, pc) { - span, ctx := opentracing.StartSpanFromContextWithTracer(test.ctx, instrumentation.Tracer(), fullTestName, testTags) - span.SetBaggageItem("trace.kind", "test") + test := &Test{t: t, ctx: context.Background()} + for _, opt := range opts { + opt(test) + } - if isTestCached(t, pc) { + // Extracting the testing func name (by removing any possible sub-test suffix `{test_func}/{sub_test}`) + // to search the func source code bounds and to calculate the package name. + fullTestName := runner.GetOriginalTestName(t.Name()) + pName, _ := getPackageAndName(pc) + + testTags := opentracing.Tags{ + "span.kind": "test", + "test.name": fullTestName, + "test.suite": pName, + "test.framework": "testing", + "test.language": "go", + } + span, _ := opentracing.StartSpanFromContextWithTracer(test.ctx, instrumentation.Tracer(), fullTestName, testTags) + span.SetBaggageItem("trace.kind", "test") span.SetTag("test.status", tags.TestStatus_CACHE) span.Finish() - // Remove the Test struct from the hash map, so a call to Start while we end this instance will create a new struct - removeTest(t) t.SkipNow() - } + return test - test.span = span - test.ctx = ctx + } else { + + // Get or create a new Test struct + // If we get an old struct we replace the current span and context with a new one. + // Useful if we want to overwrite the Start call with options + test, exist := getOrCreateTest(t) + if exist { + // If there is already one we want to replace it, so we clear the context + test.ctx = context.Background() + } + test.codePC = pc + + for _, opt := range opts { + opt(test) + } + + // Extracting the testing func name (by removing any possible sub-test suffix `{test_func}/{sub_test}`) + // to search the func source code bounds and to calculate the package name. + fullTestName := runner.GetOriginalTestName(t.Name()) + pName, _, testCode := getPackageAndNameAndBoundaries(pc) + + testTags := opentracing.Tags{ + "span.kind": "test", + "test.name": fullTestName, + "test.suite": pName, + "test.framework": "testing", + "test.language": "go", + } - logging.Reset() - startCoverage() + if testCode != "" { + testTags["test.code"] = testCode + } - return test + if test.ctx == nil { + test.ctx = context.Background() + } + + span, ctx := opentracing.StartSpanFromContextWithTracer(test.ctx, instrumentation.Tracer(), fullTestName, testTags) + span.SetBaggageItem("trace.kind", "test") + test.span = span + test.ctx = ctx + + logging.Reset() + startCoverage() + + return test + } } // Set test code From e90803482667d753eb309676de239983a5ffea14 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 17:01:02 +0200 Subject: [PATCH 11/12] comments changes --- agent/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/cache.go b/agent/cache.go index 174510c3..ca2fcd07 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -79,7 +79,7 @@ func (c *localCache) GetOrSet(key string, useTimeout bool, fn func(interface{}, path := fmt.Sprintf("%s.%s", c.basePath, key) - // We try to load the cached version of the remote configuration + // We try to load the cached value file, err := os.Open(path) if err != nil { return loaderFunc(key, err, fn) From 76f1cfb9856efbafd4ec819a9ddbe607f007e2a6 Mon Sep 17 00:00:00 2001 From: Daniel Redondo Date: Mon, 11 May 2020 17:14:36 +0200 Subject: [PATCH 12/12] local cache test --- agent/cache.go | 2 +- agent/cache_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 agent/cache_test.go diff --git a/agent/cache.go b/agent/cache.go index ca2fcd07..eede8e39 100644 --- a/agent/cache.go +++ b/agent/cache.go @@ -33,7 +33,7 @@ type ( ) // Create a new local cache -func newLocalCache(tenant map[string]interface{}, timeout time.Duration, debugMode bool, logger *log.Logger) *localCache { +func newLocalCache(tenant interface{}, timeout time.Duration, debugMode bool, logger *log.Logger) *localCache { lc := &localCache{ timeout: timeout, debugMode: debugMode, diff --git a/agent/cache_test.go b/agent/cache_test.go new file mode 100644 index 00000000..a58f081f --- /dev/null +++ b/agent/cache_test.go @@ -0,0 +1,51 @@ +package agent + +import ( + "fmt" + "log" + "os" + "testing" + "time" +) + +func getTenant() interface{} { + return map[string]string{ + "key1": "value1", + "key2": fmt.Sprintf("%v", time.Now()), + } +} + +func TestLocalCache(t *testing.T) { + + tenant := getTenant() + + cache := newLocalCache(tenant, cacheTimeout, true, log.New(os.Stdout, "", 0)) + loader := false + result := cache.GetOrSet("MyKey01", false, func(i interface{}, s string) interface{} { + loader = true + return "hello world" + }) + + if !loader { + t.Fatal("loader has not been executed.") + } + if result.(string) != "hello world" { + t.Fatal("result was different than expected.") + } + + cache2 := newLocalCache(tenant, cacheTimeout, true, log.New(os.Stdout, "", 0)) + loader = false + for i := 0; i < 10; i++ { + result = cache2.GetOrSet("MyKey01", false, func(i interface{}, s string) interface{} { + loader = true + return "hello world" + }) + + if loader { + t.Fatal("loader has been executed.") + } + if result.(string) != "hello world" { + t.Fatal("result was different than expected.") + } + } +}