diff --git a/Makefile b/Makefile index ed715be70..201028c05 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ include Makefile.* # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Variable Definitions # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -VERSION = $(shell git describe --match "v[0-9]*" --abbrev=0) +VERSION = $(shell git describe --match "v[0-9]*" --abbrev=0 --tags) COMMIT = $(shell git rev-parse --short HEAD) DATE = $(shell date +%F_%H-%M-%S) @@ -74,7 +74,7 @@ lint: ## Run linter cd sdk && make lint format: ## Format code - cd sdk && go fmt ./... + go fmt ./... && cd sdk && go fmt ./... && cd ../test/performance && go fmt ./... install-tools: ## Install dependencies in tools.go @grep _ ./scripts/tools.go | awk '{print $$2}' | xargs -tI % go install % @@ -84,11 +84,11 @@ install-tools: ## Install dependencies in tools.go # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # local-apk-package: ## Create local apk package GOWORK=off CGO_ENABLED=0 GOARCH=${LOCAL_ARCH} GOOS=linux go build -ldflags=${DEBUG_LDFLAGS} -o ./build/nginx-agent - VERSION=$(shell echo ${VERSION} | tr -d 'v') nfpm pkg --config ./scripts/.local-nfpm.yaml --packager apk --target ./build/${PACKAGE_PREFIX}-$(shell echo ${VERSION} | tr -d 'v')-SNAPSHOT.apk; + VERSION=$(shell echo ${VERSION} | tr -d 'v') nfpm pkg --config ./scripts/.local-nfpm.yaml --packager apk --target ./build/${PACKAGE_PREFIX}-$(shell echo ${VERSION} | tr -d 'v')-SNAPSHOT-${COMMIT}.apk; local-deb-package: ## Create local deb package GOWORK=off CGO_ENABLED=0 GOARCH=${LOCAL_ARCH} GOOS=linux go build -ldflags=${DEBUG_LDFLAGS} -o ./build/nginx-agent - VERSION=$(shell echo ${VERSION} | tr -d 'v') nfpm pkg --config ./scripts/.local-nfpm.yaml --packager deb --target ./build/${PACKAGE_PREFIX}-$(shell echo ${VERSION} | tr -d 'v')-SNAPSHOT.deb; + VERSION=$(shell echo ${VERSION} | tr -d 'v') nfpm pkg --config ./scripts/.local-nfpm.yaml --packager deb --target ./build/${PACKAGE_PREFIX}-$(shell echo ${VERSION} | tr -d 'v')-SNAPSHOT-${COMMIT}.deb; local-rpm-package: ## Create local rpm package GOWORK=off CGO_ENABLED=0 GOARCH=${LOCAL_ARCH} GOOS=linux go build -ldflags=${DEBUG_LDFLAGS} -o ./build/nginx-agent diff --git a/src/core/pipe.go b/src/core/pipe.go index 39cf5f85e..5903bcce9 100644 --- a/src/core/pipe.go +++ b/src/core/pipe.go @@ -44,7 +44,7 @@ func (p *MessagePipe) Register(size int, plugins ...Plugin) error { p.mu.Lock() p.plugins = append(p.plugins, plugins...) - p.bus = messagebus.New(size) + p.bus = messagebus.New(size) for _, plugin := range p.plugins { for _, subscription := range plugin.Subscriptions() { @@ -75,7 +75,7 @@ func (p *MessagePipe) Run() { for { select { case <-p.ctx.Done(): - + for _, r := range p.plugins { r.Close() } diff --git a/test/performance/advanced-metrics/metric_gen/generator.go b/test/performance/advanced-metrics/metric_gen/generator.go index 121405dd2..3246486bf 100644 --- a/test/performance/advanced-metrics/metric_gen/generator.go +++ b/test/performance/advanced-metrics/metric_gen/generator.go @@ -2,10 +2,10 @@ package metric_gen import ( "context" + "crypto/rand" "fmt" "math" "math/big" - "crypto/rand" "strconv" "strings" "sync" @@ -38,7 +38,7 @@ type Generator struct { messageCache []*Message cacheLock sync.Mutex - iterator int + iterator int setMetrics bool setMetricsSets []string @@ -203,7 +203,7 @@ func (g *Generator) makeMetricSet(uniqueDimension bool) string { // choose one string dimension that will be unique uniqueDimPositionBig, _ := rand.Int(rand.Reader, big.NewInt(int64(len(fieldOrder)))) for fieldOrder[uniqueDimPositionBig.Int64()].Type != stringType { - uniqueDimPositionBig, _ = rand.Int(rand.Reader, big.NewInt(int64(len(fieldOrder)) - 8)) + uniqueDimPositionBig, _ = rand.Int(rand.Reader, big.NewInt(int64(len(fieldOrder))-8)) } uniqueDimPosition := int(uniqueDimPositionBig.Int64()) diff --git a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto index 2b84942f1..81959dda8 100644 --- a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto +++ b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto @@ -7,10 +7,6 @@ import "events/event.proto"; import "metrics.proto"; // MetricsService is responsible for ingesting high volume metrics and events -// Note: -// The naming of MetricsService is unfortunate, it is extended to StreamEvents -// to keep backward and forward compatability with NGINX Management Suite products -// This will be refactored when a major revision change for NGINX Agent is introduced service MetricsService { // A client-to-server streaming RPC to deliver high volume metrics reports. rpc Stream(stream MetricsReport) returns (google.protobuf.Empty) {} diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/pipe.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/pipe.go index 39cf5f85e..5903bcce9 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/pipe.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/pipe.go @@ -44,7 +44,7 @@ func (p *MessagePipe) Register(size int, plugins ...Plugin) error { p.mu.Lock() p.plugins = append(p.plugins, plugins...) - p.bus = messagebus.New(size) + p.bus = messagebus.New(size) for _, plugin := range p.plugins { for _, subscription := range plugin.Subscriptions() { @@ -75,7 +75,7 @@ func (p *MessagePipe) Run() { for { select { case <-p.ctx.Done(): - + for _, r := range p.plugins { r.Close() } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/collector/nap.go b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/collector/nap.go index 1888dda02..c11a2e3ab 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/collector/nap.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/collector/nap.go @@ -35,24 +35,21 @@ type syslogServer struct { } // NewNAPCollector gives you a NAP collector for the syslog server. -func NewNAPCollector(cfg *NAPConfig) (*NAPCollector, error) { - var ( - c NAPCollector - err error - ) +func NewNAPCollector(cfg *NAPConfig) (napCollector *NAPCollector, err error) { + napCollector = &NAPCollector{} - c.logger = logrus.StandardLogger().WithFields(componentLogFields) + napCollector.logger = logrus.StandardLogger().WithFields(componentLogFields) if cfg.Logger != nil { - c.logger = cfg.Logger.WithFields(componentLogFields) + napCollector.logger = cfg.Logger.WithFields(componentLogFields) } - c.logger.Infof("Getting %s Collector", monitoring.NAP) + napCollector.logger.Infof("Getting %s Collector", monitoring.NAP) - c.syslog, err = newSyslogServer(c.logger, cfg.SyslogIP, cfg.SyslogPort) + napCollector.syslog, err = newSyslogServer(napCollector.logger, cfg.SyslogIP, cfg.SyslogPort) if err != nil { return nil, err } - return &c, nil + return napCollector, nil } func newSyslogServer(logger *logrus.Entry, ip string, port int) (*syslogServer, error) { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/manager/manager.go b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/manager/manager.go index bda73e1ef..8d5f45b54 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/manager/manager.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/manager/manager.go @@ -2,16 +2,17 @@ package manager import ( "context" - "github.com/nginx/agent/v2/src/core/metrics" "runtime" "sync" + log "github.com/sirupsen/logrus" + models "github.com/nginx/agent/sdk/v2/proto/events" "github.com/nginx/agent/v2/src/core/config" + "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring" "github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/collector" "github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor" - log "github.com/sirupsen/logrus" ) const ( diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/nap.go b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/nap.go index 1663c4376..b64287f40 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/nap.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/nap.go @@ -25,6 +25,7 @@ const ( var ( logFormatKeys = []string{ + // TODO: Remove `date_time` from syslog format as it is unused (NMS-38119) "date_time", "blocking_exception_reason", "dest_port", @@ -65,7 +66,7 @@ var ( ) const ( - // TODO: Identify the usage of the following new keys + // TODO: Identify the usage of the following new keys (NMS-38311) blockingExceptionReason = "blocking_exception_reason" protocol = "protocol" requestStatus = "request_status" @@ -77,22 +78,6 @@ const ( clientApplication = "client_application" clientApplicationVersion = "client_application_version" transportProtocol = "transport_protocol" - - // Using default values instead of overriden keys per older CAS policy - httpRequestMethod = "method" - httpResponseCode = "response_code" - sigCVEs = "sig_cves" - sigIds = "sig_ids" - sigNames = "sig_names" - httpRemotePort = "src_port" - httpURI = "uri" - httpHostname = "vs_name" - requestOutcome = "outcome" - requestOutcomeReason = "outcome_reason" - httpRemoteAddr = "ip_client" - httpServerPort = "dest_port" - isTruncated = "is_truncated" - // Older CAS Naming (this needs to be removed) // httpRequestMethod = "http_request_method" // httpResponseCode = "http_response_code" @@ -110,6 +95,21 @@ const ( // httpServerPort = "http_server_port" // isTruncated = "is_truncated_bool" + // Using default values instead of overriden keys per older CAS policy + httpRequestMethod = "method" + httpResponseCode = "response_code" + sigCVEs = "sig_cves" + sigIds = "sig_ids" + sigNames = "sig_names" + httpRemotePort = "src_port" + httpURI = "uri" + httpHostname = "vs_name" + requestOutcome = "outcome" + requestOutcomeReason = "outcome_reason" + httpRemoteAddr = "ip_client" + httpServerPort = "dest_port" + isTruncated = "is_truncated" + // Existing parsed keys from the log dateTime = "date_time" policyName = "policy_name" @@ -301,8 +301,7 @@ func (f *NAPConfig) getSecurityViolation(logger *logrus.Entry) *models.SecurityV EnforcedBotAnomalies: f.EnforcedBotAnomalies, ViolationContexts: f.getViolationContext(), ViolationsData: f.getViolations(logger), - // The following items needs to be fixed before release - // TODO: https://nginxsoftware.atlassian.net/browse/NMS-38119 + // TODO: The following items needs to be fixed before release (NMS-38119) DateTime: f.DateTime, // remove, metadata has it Outcome: f.RequestOutcome, //rename the proto OutcomeReason: f.RequestOutcomeReason, //rename the proto @@ -316,8 +315,6 @@ func (f *NAPConfig) getSecurityViolation(logger *logrus.Entry) *models.SecurityV } func (f *NAPConfig) getMetadata() (*models.Metadata, error) { - // Set date time as current time with format YYYY-MM-DD HH:MM:SS.SSS - // This is a temporary solution - https://nginxsoftware.atlassian.net/browse/IND-10651 f.DateTime = time.Now().UTC().Format(napDateTimeLayout) t, err := parseNAPDateTime(f.DateTime) @@ -325,9 +322,7 @@ func (f *NAPConfig) getMetadata() (*models.Metadata, error) { return nil, err } - // set the correlation ID correctly - // TODO: https://nginxsoftware.atlassian.net/browse/NMS-37563 - return NewMetadata(t, "123") + return NewMetadata(t, f.SupportID) } func (f *NAPConfig) getViolationContext() string { @@ -474,11 +469,10 @@ func parseNAPDateTime(raw string) (*types.Timestamp, error) { return types.TimestampProto(t) } -// Assumptions while parsing the NAP Syslog data: +// TODO: Assumptions while parsing the NAP Syslog data (NMS-38118) // 1. list values do not contain `commas`, rather have `::` as delimiter // 2. no json values // 3. no other comma exists in the response other than the delimiter comma -// TODO: https://nginxsoftware.atlassian.net/browse/NMS-38118 func parseNAP(logEntry string, logger *logrus.Entry) (*NAPConfig, error) { var waf NAPConfig diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/processor.go b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/processor.go index 0906b8990..6c96a042c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/processor.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/processor/processor.go @@ -4,13 +4,13 @@ import ( "context" "errors" "fmt" - "github.com/nginx/agent/v2/src/core/metrics" "regexp" "sync" "github.com/sirupsen/logrus" pb "github.com/nginx/agent/sdk/v2/proto/events" + "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring" ) @@ -62,6 +62,10 @@ func GetClient(cfg *Config) (*Client, error) { } c.hostPattern = hostPattern + if cfg.CommonDims == nil { + c.logger.Warnf("common dimensions are not passed to NAP Monitoring processor") + cfg.CommonDims = &metrics.CommonDim{} + } c.commonDims = cfg.CommonDims return &c, nil @@ -93,6 +97,18 @@ func (c *Client) processorWorker(ctx context.Context, wg *sync.WaitGroup, id int break } + if event.GetSecurityViolationEvent() == nil { + c.logger.Errorf("event expected as SecurityViolationEvent from nap monitor processing") + break + } + + event.GetSecurityViolationEvent().SystemID = c.commonDims.SystemId + event.GetSecurityViolationEvent().Hostname = c.commonDims.Hostname + event.GetSecurityViolationEvent().InstanceTags = c.commonDims.InstanceTags + event.GetSecurityViolationEvent().InstanceGroup = c.commonDims.InstanceGroup + event.GetSecurityViolationEvent().DisplayName = c.commonDims.DisplayName + event.GetSecurityViolationEvent().NginxID = c.commonDims.NginxId + c.logger.Debugf("%d: Generated Event: %s", id, event) processed <- event diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/comms.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/comms.go index 15699f288..b0e112079 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/comms.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/comms.go @@ -141,7 +141,6 @@ func (r *Comms) reportLoop() { sb.WriteString(", ") } sb.WriteString(report.Events[l-1].GetSecurityViolationEvent().SupportID) - // TODO: In addition to the below, make sure we don't lose data https://nginxsoftware.atlassian.net/browse/NMS-38169 log.Errorf("Failed to send EventReport with error: %v, supportID list: %s", err, sb.String()) } else { log.Tracef("EventReport sent, %v", report) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/extensions.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/extensions.go index cf7b34260..542df28ac 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/extensions.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/extensions.go @@ -1,9 +1,10 @@ package plugins import ( + log "github.com/sirupsen/logrus" + "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" - log "github.com/sirupsen/logrus" ) const ( diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/nap_monitoring.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/nap_monitoring.go index 99e120cb1..e1b14a1c9 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/nap_monitoring.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/nap_monitoring.go @@ -2,7 +2,6 @@ package plugins import ( "context" - "github.com/nginx/agent/v2/src/core/metrics" "time" log "github.com/sirupsen/logrus" @@ -10,6 +9,7 @@ import ( models "github.com/nginx/agent/sdk/v2/proto/events" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" + "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/extensions/nginx-app-protect/monitoring/manager" ) @@ -73,12 +73,8 @@ func (n *NAPMonitoring) Init(pipeline core.MessagePipeInterface) { go n.run() } -// TODO: https://nginxsoftware.atlassian.net/browse/NMS-38140 -// - Identify if we need to process any interactions with NGINX func (n *NAPMonitoring) Process(msg *core.Message) {} -// TODO: https://nginxsoftware.atlassian.net/browse/NMS-38140 -// - Subscribe for Agent config updates func (n *NAPMonitoring) Subscriptions() []string { return []string{} } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go index 10f7d70cc..b558011ff 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/test/utils/environment.go @@ -47,20 +47,24 @@ func NewMockEnvironment() *MockEnvironment { var _ core.Environment = NewMockEnvironment() func (m *MockEnvironment) NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo { - m.Called(agentVersion, tags) - return &proto.HostInfo{ - Agent: agentVersion, - Boot: 0, - Hostname: "test-host", - DisplayName: "", - OsType: "", - Uuid: "", - Uname: "", - Partitons: []*proto.DiskPartition{}, - Network: &proto.Network{}, - Processor: []*proto.CpuInfo{}, - Release: &proto.ReleaseInfo{}, + args := m.Called(agentVersion, tags) + returned, ok := args.Get(0).(*proto.HostInfo) + if !ok { + return &proto.HostInfo{ + Agent: agentVersion, + Boot: 0, + Hostname: "test-host", + DisplayName: "", + OsType: "", + Uuid: "", + Uname: "", + Partitons: []*proto.DiskPartition{}, + Network: &proto.Network{}, + Processor: []*proto.CpuInfo{}, + Release: &proto.ReleaseInfo{}, + } } + return returned } func (m *MockEnvironment) GetHostname() string { diff --git a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto index 2b84942f1..81959dda8 100644 --- a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto +++ b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.svc.proto @@ -7,10 +7,6 @@ import "events/event.proto"; import "metrics.proto"; // MetricsService is responsible for ingesting high volume metrics and events -// Note: -// The naming of MetricsService is unfortunate, it is extended to StreamEvents -// to keep backward and forward compatability with NGINX Management Suite products -// This will be refactored when a major revision change for NGINX Agent is introduced service MetricsService { // A client-to-server streaming RPC to deliver high volume metrics reports. rpc Stream(stream MetricsReport) returns (google.protobuf.Empty) {}