Skip to content

Commit

Permalink
fix: persist last inventory submission time (#31)
Browse files Browse the repository at this point in the history
* fix: adding support of proper 24 hours inventory resend
Co-authored-by: jhvaras <jhvaras@gmail.com>
Co-authored-by: Cristian Ciutea <cristi.ciutea@gmail.com>
Co-authored-by: noly <noeliaddf@gmail.com>
Co-authored-by: Carlos <croman@newrelic.com>
  • Loading branch information
brushknight committed Jul 23, 2020
1 parent d35cbe7 commit 6503df0
Show file tree
Hide file tree
Showing 35 changed files with 996 additions and 277 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/evanphx/json-patch v4.5.0+incompatible
github.com/fortytw2/leaktest v1.3.1-0.20190606143808-d73c753520d9
github.com/fsnotify/fsnotify v0.9.3
github.com/ghodss/yaml v1.0.0
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/gogo/protobuf v1.1.2-0.20181116123445-07eab6a8298c // indirect
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ github.com/fortytw2/leaktest v1.3.1-0.20190606143808-d73c753520d9 h1:tKHw9zBEj0r
github.com/fortytw2/leaktest v1.3.1-0.20190606143808-d73c753520d9/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v0.9.3 h1:ohbr5HT+EHaq7KpTqPxkowzWw0vTFj25E8HVrVVHNIA=
github.com/fsnotify/fsnotify v0.9.3/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-fsnotify/fsnotify v0.0.0-20180321022601-755488143dae h1:PeVNzgTRtWGm6fVic5i21t+n5ptPGCZuMcSPVMyTWjs=
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
Expand Down
69 changes: 45 additions & 24 deletions internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,23 +65,20 @@ type registerableSender interface {
}

type Agent struct {
plugins []Plugin // Slice of registered plugins
oldPlugins []ids.PluginID // Deprecated plugins whose cached data must be removed, if existing
agentDir string // Base data directory for the agent
extDir string // Location of external data input
userAgent string // User-Agent making requests to warlock
inventories map[string]*inventory // Inventory reaper and sender instances (key: entity ID)
Context *context // Agent context data that is passed around the place
metricsSender registerableSender
store *delta.Store
debugProvide debug.Provide
httpClient backendhttp.Client // http client for both data submission types: events and inventory

connectSrv *identityConnectService

provideIDs ProvideIDs
entityMap entity.KnownIDs

plugins []Plugin // Slice of registered plugins
oldPlugins []ids.PluginID // Deprecated plugins whose cached data must be removed, if existing
agentDir string // Base data directory for the agent
extDir string // Location of external data input
userAgent string // User-Agent making requests to warlock
inventories map[string]*inventory // Inventory reaper and sender instances (key: entity ID)
Context *context // Agent context data that is passed around the place
metricsSender registerableSender
store *delta.Store
debugProvide debug.Provide
httpClient backendhttp.Client // http client for both data submission types: events and inventory
connectSrv *identityConnectService
provideIDs ProvideIDs
entityMap entity.KnownIDs
fpHarvester fingerprint.Harvester
cloudHarvester cloud.Harvester // If it's the case returns information about the cloud where instance is running.
agentID *entity.ID // pointer as it's referred from several points
Expand Down Expand Up @@ -358,15 +355,37 @@ func NewAgent(cfg *config.Config, buildVersion string, ffRetriever feature_flags
// notificationHandler will map ipc messages to functions
notificationHandler := ctl.NewNotificationHandlerWithCancellation(ctx.Ctx)

return New(cfg, ctx, userAgent, idLookupTable, s, connectSrv, provideIDs, httpClient, transport, cloudHarvester,
fpHarvester, notificationHandler)
return New(
cfg,
ctx,
userAgent,
idLookupTable,
s,
connectSrv,
provideIDs,
httpClient,
transport,
cloudHarvester,
fpHarvester,
notificationHandler,
)
}

// New creates a new agent using given context and services.
func New(cfg *config.Config, ctx *context, userAgent string, idLookupTable IDLookup, s *delta.Store,
connectSrv *identityConnectService, provideIDs ProvideIDs, dataClient backendhttp.Client,
transport *http.Transport, cloudHarvester cloud.Harvester, fpHarvester fingerprint.Harvester,
notificationHandler *ctl.NotificationHandlerWithCancellation) (*Agent, error) {
func New(
cfg *config.Config,
ctx *context,
userAgent string,
idLookupTable IDLookup,
s *delta.Store,
connectSrv *identityConnectService,
provideIDs ProvideIDs,
dataClient backendhttp.Client,
transport *http.Transport,
cloudHarvester cloud.Harvester,
fpHarvester fingerprint.Harvester,
notificationHandler *ctl.NotificationHandlerWithCancellation,
) (*Agent, error) {
a := &Agent{
Context: ctx,
debugProvide: debug.ProvideFn,
Expand Down Expand Up @@ -511,7 +530,9 @@ func (a *Agent) registerEntityInventory(entityKey string) error {
if a.Context.cfg.RegisterEnabled {
inv.sender, err = newPatchSenderVortex(entityKey, a.Context.agentKey, a.Context, a.store, a.userAgent, a.Context.AgentIdentity, a.provideIDs, a.entityMap, a.httpClient)
} else {
inv.sender, err = newPatchSender(entityKey, a.Context, a.store, a.userAgent, a.Context.AgentIdentity, a.httpClient)
lastSubmission := delta.NewLastSubmissionStore(a.store.DataDir, entityKey)
lastEntityID := delta.NewEntityIDFilePersist(a.store.DataDir, entityKey)
inv.sender, err = newPatchSender(entityKey, a.Context, a.store, lastSubmission, lastEntityID, a.userAgent, a.Context.AgentIdentity, a.httpClient)
}
if err != nil {
return err
Expand Down
16 changes: 14 additions & 2 deletions internal/agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ func newTesting(cfg *config.Config) *Agent {
cloudDetector := cloud.NewDetector(true, 0, 0, 0, false)
lookups := NewIdLookup(hostname.CreateResolver("", "", true), cloudDetector, cfg.DisplayName)

//ctx := newContextTesting("1.2.3", cfg)
ctx := NewContext(cfg, "1.2.3", testhelpers.NullHostnameResolver, lookups, matcher)

st := delta.NewStore(dataDir, "default", cfg.MaxInventorySize)
Expand All @@ -69,7 +68,20 @@ func newTesting(cfg *config.Config) *Agent {
connectSrv := NewIdentityConnectService(&MockIdentityConnectClient{}, fpHarvester)
provideIDs := NewProvideIDs(&EmptyRegisterClient{}, state.NewRegisterSM())

a, err := New(cfg, ctx, "user-agent", lookups, st, connectSrv, provideIDs, http2.NullHttpClient, &http.Transport{}, cloudDetector, fpHarvester, ctl.NewNotificationHandlerWithCancellation(nil))
a, err := New(
cfg,
ctx,
"user-agent",
lookups,
st,
connectSrv,
provideIDs,
http2.NullHttpClient,
&http.Transport{},
cloudDetector,
fpHarvester,
ctl.NewNotificationHandlerWithCancellation(nil),
)

if err != nil {
panic(err)
Expand Down
83 changes: 83 additions & 0 deletions internal/agent/delta/last_entityId.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package delta

import (
"fmt"
"github.com/newrelic/infrastructure-agent/pkg/entity"
"github.com/newrelic/infrastructure-agent/pkg/helpers"
"io/ioutil"
"os"
"path/filepath"
"strconv"
)

type EntityIDPersist interface {
GetEntityID() (entity.ID, error)
UpdateEntityID(id entity.ID) error
}

// EntityIDFilePersist will store on the given file the EntityID in order to persist it between agent restarts.
type EntityIDFilePersist struct {
readFile func(path string) (entity.ID, error)
writeFile func(content entity.ID, path string) error
filePath string
lastEntityID entity.ID
}

// NewEntityIDFilePersist create a new instance of EntityIDFilePersist.
func NewEntityIDFilePersist(dataDir string, fileName string) *EntityIDFilePersist {
return &EntityIDFilePersist{
readFile: readFileFn,
writeFile: writeFileFn,
filePath: filepath.Join(dataDir, lastEntityIDFolder, helpers.SanitizeFileName(fileName)),
}
}

// GetEntityID will return entityID from memory or disk.
func (e *EntityIDFilePersist) GetEntityID() (entity.ID, error) {
var err error
if e.lastEntityID == entity.EmptyID {
e.lastEntityID, err = e.readFile(e.filePath)
}

return e.lastEntityID, err
}

// UpdateEntityID will store the entityID on memory and disk.
func (e *EntityIDFilePersist) UpdateEntityID(id entity.ID) error {
e.lastEntityID = id

return e.writeFile(id, e.filePath)
}

func readFileFn(filePath string) (entity.ID, error) {
// Check if there is an already stored value on disk.
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return entity.EmptyID, nil
}

buf, err := ioutil.ReadFile(filePath)

if err != nil {
return entity.EmptyID, fmt.Errorf("cannot read file persisted entityID, file: '%s', error: %v", filePath, err)
}

value, err := strconv.ParseInt(string(buf), 10, 64)
if err != nil {
return entity.EmptyID, fmt.Errorf("cannot parse entityID from file content: '%s', error: %v", buf, err)
}

return entity.ID(value), nil
}

func writeFileFn(content entity.ID, filePath string) error {
dir := filepath.Dir(filePath)

if _, err := os.Stat(dir); os.IsNotExist(err) {
if mkDirErr := os.MkdirAll(dir, DATA_DIR_MODE); mkDirErr != nil {
return fmt.Errorf("cannot persist entityID, agent data directory: '%s' does not exist and cannot be created: %v",
dir, mkDirErr)
}
}

return ioutil.WriteFile(filePath, []byte(content.String()), DATA_FILE_MODE)
}
Loading

0 comments on commit 6503df0

Please sign in to comment.