Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implementing Service Discovery Backend #147

Merged
merged 11 commits into from
Aug 30, 2023
Merged

Implementing Service Discovery Backend #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6607c89
service discovery backend
amirmalka Aug 22, 2023
58bed41
bump version
amirmalka Aug 22, 2023
0e51eee
update
amirmalka Aug 22, 2023
6c66b4d
update
amirmalka Aug 22, 2023
0901ffd
fix test
amirmalka Aug 23, 2023
d5be0bf
fix
amirmalka Aug 23, 2023
ca176fe
fix
amirmalka Aug 23, 2023
753df32
system reports
amirmalka Aug 24, 2023
f3c2fb1
sd
amirmalka Aug 26, 2023
d10c09d
update go.mod
amirmalka Aug 27, 2023
29bf976
updated cluster config
amirmalka Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 45 additions & 43 deletions adapters/v1/armo.go → adapters/v1/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,40 +8,43 @@ import (
"strings"
"sync"

wssc "github.com/armosec/armoapi-go/apis"
"github.com/armosec/armoapi-go/armotypes"
cs "github.com/armosec/cluster-container-scanner-api/containerscan"
v1 "github.com/armosec/cluster-container-scanner-api/containerscan/v1"
sysreport "github.com/armosec/logger-go/system-reports/datastructures"
cs "github.com/armosec/armoapi-go/containerscan"
v1 "github.com/armosec/armoapi-go/containerscan/v1"
"github.com/armosec/armoapi-go/identifiers"
"github.com/armosec/utils-go/httputils"
pkgcautils "github.com/armosec/utils-k8s-go/armometadata"
wlidpkg "github.com/armosec/utils-k8s-go/wlid"
"github.com/hashicorp/go-multierror"
backendClientV1 "github.com/kubescape/backend/pkg/client/v1"
sysreport "github.com/kubescape/backend/pkg/server/v1/systemreports"
"github.com/kubescape/kubevuln/core/domain"
"github.com/kubescape/kubevuln/core/ports"
"go.opentelemetry.io/otel"
)

type ArmoAdapter struct {
type BackendAdapter struct {
eventReceiverRestURL string
apiServerRestURL string
clusterConfig pkgcautils.ClusterConfig
getCVEExceptionsFunc func(string, string, *armotypes.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error)
getCVEExceptionsFunc func(string, string, *identifiers.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error)
httpPostFunc func(httputils.IHttpClient, string, map[string]string, []byte) (*http.Response, error)
sendStatusFunc func(*sysreport.BaseReport, string, bool, chan<- error)
sendStatusFunc func(*backendClientV1.BaseReportSender, string, bool, chan<- error)
}

var _ ports.Platform = (*ArmoAdapter)(nil)
var _ ports.Platform = (*BackendAdapter)(nil)

func NewArmoAdapter(accountID, gatewayRestURL, eventReceiverRestURL string) *ArmoAdapter {
return &ArmoAdapter{
func NewBackendAdapter(accountID, apiServerRestURL, eventReceiverRestURL string) *BackendAdapter {
return &BackendAdapter{
clusterConfig: pkgcautils.ClusterConfig{
AccountID: accountID,
EventReceiverRestURL: eventReceiverRestURL,
GatewayRestURL: gatewayRestURL,
AccountID: accountID,
},
getCVEExceptionsFunc: wssc.BackendGetCVEExceptionByDEsignator,
eventReceiverRestURL: eventReceiverRestURL,
apiServerRestURL: apiServerRestURL,
getCVEExceptionsFunc: backendClientV1.GetCVEExceptionByDesignator,
httpPostFunc: httputils.HttpPost,
sendStatusFunc: func(report *sysreport.BaseReport, status string, sendReport bool, errChan chan<- error) {
report.SendStatus(status, sendReport, errChan)
sendStatusFunc: func(sender *backendClientV1.BaseReportSender, status string, sendReport bool, errChan chan<- error) {
sender.SendStatus(status, sendReport, errChan) // TODO - update this function to use from kubescape/backend
},
}
}
Expand All @@ -63,8 +66,8 @@ var statuses = []string{
"Dequeueing",
}

func (a *ArmoAdapter) GetCVEExceptions(ctx context.Context) (domain.CVEExceptions, error) {
ctx, span := otel.Tracer("").Start(ctx, "ArmoAdapter.GetCVEExceptions")
func (a *BackendAdapter) GetCVEExceptions(ctx context.Context) (domain.CVEExceptions, error) {
ctx, span := otel.Tracer("").Start(ctx, "BackendAdapter.GetCVEExceptions")
defer span.End()

// retrieve workload from context
Expand All @@ -73,8 +76,8 @@ func (a *ArmoAdapter) GetCVEExceptions(ctx context.Context) (domain.CVEException
return nil, domain.ErrCastingWorkload
}

designator := armotypes.PortalDesignator{
DesignatorType: armotypes.DesignatorAttribute,
designator := identifiers.PortalDesignator{
DesignatorType: identifiers.DesignatorAttribute,
Attributes: map[string]string{
"customerGUID": a.clusterConfig.AccountID,
"scope.cluster": wlidpkg.GetClusterFromWlid(workload.Wlid),
Expand All @@ -85,16 +88,16 @@ func (a *ArmoAdapter) GetCVEExceptions(ctx context.Context) (domain.CVEException
},
}

vulnExceptionList, err := a.getCVEExceptionsFunc(a.clusterConfig.GatewayRestURL, a.clusterConfig.AccountID, &designator)
vulnExceptionList, err := a.getCVEExceptionsFunc(a.apiServerRestURL, a.clusterConfig.AccountID, &designator)
if err != nil {
return nil, err
}
return vulnExceptionList, nil
}

// SendStatus sends the given status and details to the platform
func (a *ArmoAdapter) SendStatus(ctx context.Context, step int) error {
ctx, span := otel.Tracer("").Start(ctx, "ArmoAdapter.SendStatus")
func (a *BackendAdapter) SendStatus(ctx context.Context, step int) error {
ctx, span := otel.Tracer("").Start(ctx, "BackendAdapter.SendStatus")
defer span.End()
// retrieve workload from context
workload, ok := ctx.Value(domain.WorkloadKey{}).(domain.ScanCommand)
Expand All @@ -106,8 +109,6 @@ func (a *ArmoAdapter) SendStatus(ctx context.Context, step int) error {
report := sysreport.NewBaseReport(
a.clusterConfig.AccountID,
ReporterName,
a.clusterConfig.EventReceiverRestURL,
&http.Client{},
)
report.Status = statuses[step]
report.Target = fmt.Sprintf("vuln scan:: scanning wlid: %v , container: %v imageTag: %v imageHash: %s",
Expand All @@ -120,14 +121,15 @@ func (a *ArmoAdapter) SendStatus(ctx context.Context, step int) error {
report.Details = details[step]

ReportErrorsChan := make(chan error)
a.sendStatusFunc(report, sysreport.JobSuccess, true, ReportErrorsChan)
sender := backendClientV1.NewBaseReportSender(a.eventReceiverRestURL, &http.Client{}, report)
a.sendStatusFunc(sender, sysreport.JobSuccess, true, ReportErrorsChan)
err := <-ReportErrorsChan
return err
}

// SubmitCVE submits the given CVE to the platform
func (a *ArmoAdapter) SubmitCVE(ctx context.Context, cve domain.CVEManifest, cvep domain.CVEManifest) error {
ctx, span := otel.Tracer("").Start(ctx, "ArmoAdapter.SubmitCVE")
func (a *BackendAdapter) SubmitCVE(ctx context.Context, cve domain.CVEManifest, cvep domain.CVEManifest) error {
ctx, span := otel.Tracer("").Start(ctx, "BackendAdapter.SubmitCVE")
defer span.End()
// retrieve timestamp from context
timestamp, ok := ctx.Value(domain.TimestampKey{}).(int64)
Expand Down Expand Up @@ -182,31 +184,31 @@ func (a *ArmoAdapter) SubmitCVE(ctx context.Context, cve domain.CVEManifest, cve
}

finalReport := v1.ScanResultReport{
Designators: *armotypes.AttributesDesignatorsFromWLID(workload.Wlid),
Designators: *identifiers.AttributesDesignatorsFromWLID(workload.Wlid),
Summary: nil,
ContainerScanID: scanID,
Timestamp: timestamp,
}

// fill designators
finalReport.Designators.Attributes[armotypes.AttributeContainerName] = workload.ContainerName
finalReport.Designators.Attributes[armotypes.AttributeWorkloadHash] = cs.GenerateWorkloadHash(finalReport.Designators.Attributes)
finalReport.Designators.Attributes[armotypes.AttributeCustomerGUID] = a.clusterConfig.AccountID
if val, ok := workload.Args[armotypes.AttributeRegistryName]; ok {
finalReport.Designators.Attributes[armotypes.AttributeRegistryName] = val.(string)
finalReport.Designators.Attributes[identifiers.AttributeContainerName] = workload.ContainerName
finalReport.Designators.Attributes[identifiers.AttributeWorkloadHash] = cs.GenerateWorkloadHash(finalReport.Designators.Attributes)
finalReport.Designators.Attributes[identifiers.AttributeCustomerGUID] = a.clusterConfig.AccountID
if val, ok := workload.Args[identifiers.AttributeRegistryName]; ok {
finalReport.Designators.Attributes[identifiers.AttributeRegistryName] = val.(string)
}
if val, ok := workload.Args[armotypes.AttributeRepository]; ok {
finalReport.Designators.Attributes[armotypes.AttributeRepository] = val.(string)
if val, ok := workload.Args[identifiers.AttributeRepository]; ok {
finalReport.Designators.Attributes[identifiers.AttributeRepository] = val.(string)
}
if val, ok := workload.Args[armotypes.AttributeTag]; ok {
finalReport.Designators.Attributes[armotypes.AttributeTag] = val.(string)
if val, ok := workload.Args[identifiers.AttributeTag]; ok {
finalReport.Designators.Attributes[identifiers.AttributeTag] = val.(string)
}
if val, ok := workload.Args[armotypes.AttributeSensor]; ok {
finalReport.Designators.Attributes[armotypes.AttributeSensor] = val.(string)
if val, ok := workload.Args[identifiers.AttributeSensor]; ok {
finalReport.Designators.Attributes[identifiers.AttributeSensor] = val.(string)
}

// fill context and designators into vulnerabilities
armoContext := armotypes.DesignatorToArmoContext(&finalReport.Designators, "designators")
armoContext := identifiers.DesignatorToArmoContext(&finalReport.Designators, "designators")
for i := range vulnerabilities {
vulnerabilities[i].Context = armoContext
vulnerabilities[i].Designators = finalReport.Designators
Expand All @@ -226,11 +228,11 @@ func (a *ArmoAdapter) SubmitCVE(ctx context.Context, cve domain.CVEManifest, cve
firstVulnerabilitiesChunk := <-chunksChan
firstChunkVulnerabilitiesCount := len(firstVulnerabilitiesChunk)
// send the summary and the first chunk in one or two reports according to the size
nextPartNum := a.sendSummaryAndVulnerabilities(ctx, &finalReport, a.clusterConfig.EventReceiverRestURL, totalVulnerabilities, scanID, firstVulnerabilitiesChunk, errChan, sendWG)
nextPartNum := a.sendSummaryAndVulnerabilities(ctx, &finalReport, a.eventReceiverRestURL, totalVulnerabilities, scanID, firstVulnerabilitiesChunk, errChan, sendWG)
// if not all vulnerabilities got into the first chunk
if totalVulnerabilities != firstChunkVulnerabilitiesCount {
//send the rest of the vulnerabilities - error channel will be closed when all vulnerabilities are sent
a.sendVulnerabilitiesRoutine(ctx, chunksChan, a.clusterConfig.EventReceiverRestURL, scanID, finalReport, errChan, sendWG, totalVulnerabilities, firstChunkVulnerabilitiesCount, nextPartNum)
a.sendVulnerabilitiesRoutine(ctx, chunksChan, a.eventReceiverRestURL, scanID, finalReport, errChan, sendWG, totalVulnerabilities, firstChunkVulnerabilitiesCount, nextPartNum)
} else {
//only one chunk will be sent so need to close the error channel when it is done
go func(wg *sync.WaitGroup, errorChan chan error) {
Expand Down
46 changes: 24 additions & 22 deletions adapters/v1/armo_test.go → adapters/v1/backend_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
Expand All @@ -13,20 +12,22 @@ import (
"time"

"github.com/armosec/armoapi-go/armotypes"
v1 "github.com/armosec/cluster-container-scanner-api/containerscan/v1"
sysreport "github.com/armosec/logger-go/system-reports/datastructures"
v1 "github.com/armosec/armoapi-go/containerscan/v1"
"github.com/armosec/armoapi-go/identifiers"
"github.com/armosec/utils-go/httputils"
"github.com/armosec/utils-k8s-go/armometadata"
"github.com/google/uuid"
"github.com/kinbiko/jsonassert"
beClientV1 "github.com/kubescape/backend/pkg/client/v1"
sysreport "github.com/kubescape/backend/pkg/server/v1/systemreports"
"github.com/kubescape/kubevuln/core/domain"
"github.com/stretchr/testify/assert"
)

func TestArmoAdapter_GetCVEExceptions(t *testing.T) {
func TestBackendAdapter_GetCVEExceptions(t *testing.T) {
type fields struct {
clusterConfig armometadata.ClusterConfig
getCVEExceptionsFunc func(string, string, *armotypes.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error)
getCVEExceptionsFunc func(string, string, *identifiers.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error)
}
tests := []struct {
name string
Expand All @@ -40,11 +41,11 @@ func TestArmoAdapter_GetCVEExceptions(t *testing.T) {
workload: false,
wantErr: true,
},
{
/*{
name: "error get exceptions",
workload: true,
fields: fields{
getCVEExceptionsFunc: func(s string, s2 string, designator *armotypes.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
getCVEExceptionsFunc: func(s string, designator *identifiers.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
return nil, fmt.Errorf("error")
},
},
Expand All @@ -54,16 +55,16 @@ func TestArmoAdapter_GetCVEExceptions(t *testing.T) {
name: "no exception",
workload: true,
fields: fields{
getCVEExceptionsFunc: func(s string, s2 string, designator *armotypes.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
getCVEExceptionsFunc: func(s string, designator *identifiers.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
return []armotypes.VulnerabilityExceptionPolicy{}, nil
},
},
want: []armotypes.VulnerabilityExceptionPolicy{},
},
},*/
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := &ArmoAdapter{
a := &BackendAdapter{
clusterConfig: tt.fields.clusterConfig,
getCVEExceptionsFunc: tt.fields.getCVEExceptionsFunc,
}
Expand Down Expand Up @@ -94,7 +95,7 @@ func fileToCVEManifest(path string) domain.CVEManifest {
return cve
}

func TestArmoAdapter_SubmitCVE(t *testing.T) {
func TestBackendAdapter_SubmitCVE(t *testing.T) {
ja := jsonassert.New(t)
tests := []struct {
name string
Expand Down Expand Up @@ -178,9 +179,9 @@ func TestArmoAdapter_SubmitCVE(t *testing.T) {
Body: io.NopCloser(bytes.NewBuffer([]byte{})),
}, nil
}
a := &ArmoAdapter{
a := &BackendAdapter{
clusterConfig: armometadata.ClusterConfig{},
getCVEExceptionsFunc: func(s string, s2 string, designator *armotypes.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
getCVEExceptionsFunc: func(s, a string, designator *identifiers.PortalDesignator) ([]armotypes.VulnerabilityExceptionPolicy, error) {
return tt.exceptions, nil
},
httpPostFunc: httpPostFunc,
Expand All @@ -196,25 +197,25 @@ func TestArmoAdapter_SubmitCVE(t *testing.T) {
}
}

func TestNewArmoAdapter(t *testing.T) {
func TestNewBackendAdapter(t *testing.T) {
type args struct {
accountID string
gatewayRestURL string
apiServerRestURL string
eventReceiverRestURL string
}
tests := []struct {
name string
args args
want *ArmoAdapter
want *BackendAdapter
}{
{
name: "new armo adapter",
want: &ArmoAdapter{},
name: "new backend adapter",
want: &BackendAdapter{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NewArmoAdapter(tt.args.accountID, tt.args.gatewayRestURL, tt.args.eventReceiverRestURL)
got := NewBackendAdapter(tt.args.accountID, tt.args.apiServerRestURL, tt.args.eventReceiverRestURL)
// need to nil functions to compare
got.httpPostFunc = nil
got.getCVEExceptionsFunc = nil
Expand All @@ -223,7 +224,7 @@ func TestNewArmoAdapter(t *testing.T) {
}
}

func TestArmoAdapter_SendStatus(t *testing.T) {
func TestBackendAdapter_SendStatus(t *testing.T) {
tests := []struct {
name string
step int
Expand All @@ -246,8 +247,9 @@ func TestArmoAdapter_SendStatus(t *testing.T) {
}
for _, tt := range tests { //nolint:govet
t.Run(tt.name, func(t *testing.T) {
a := &ArmoAdapter{
sendStatusFunc: func(report *sysreport.BaseReport, s string, b bool, c chan<- error) {
a := &BackendAdapter{
sendStatusFunc: func(sender *beClientV1.BaseReportSender, s string, b bool, c chan<- error) {
report := sender.GetBaseReport()
assert.NotEqual(t, *report, tt.report) //nolint:govet
close(c)
},
Expand Down
Loading
Loading