Skip to content

Refactor resources - save SBOM on disk #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dwertent merged 5 commits into from
Apr 16, 2023
Merged

Refactor resources - save SBOM on disk #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3001c74
save the SBOMs on the disk
Mar 30, 2023
4a3e3b3
Merge branch 'main' of github.com:kubescape/sniffer into refactor-res…
Apr 2, 2023
aa4cf3c
log to debug
Apr 2, 2023
c362f6c
PR corrections
Apr 3, 2023
8be6ca6
add to former commit
Apr 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions pkg/conthandler/container_main_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (ch *ContainerHandler) afterTimerActions() error {
fileList := containerData.containerAggregator.GetContainerRealtimeFileList()

if err = <-containerData.syncChannel[StepGetSBOM]; err != nil {
logger.L().Ctx(context.GetBackgroundContext()).Warning("failed to get SBOM", []helpers.IDetails{helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("container name", containerData.event.GetContainerName()), helpers.String("k8s resource ", containerData.event.GetK8SWorkloadID()), helpers.Error(err)}...)
logger.L().Ctx(context.GetBackgroundContext()).Debug("failed to get SBOM", []helpers.IDetails{helpers.String("container ID", afterTimerActionsData.containerID), helpers.String("container name", containerData.event.GetContainerName()), helpers.String("k8s resource ", containerData.event.GetK8SWorkloadID()), helpers.Error(err)}...)
continue
}
if err = containerData.sbomClient.FilterSBOM(fileList); err != nil {
Expand Down Expand Up @@ -133,6 +133,11 @@ func createTicker() *time.Ticker {
return time.NewTicker(config.GetConfigurationConfigContext().GetUpdateDataPeriod())
}

func (ch *ContainerHandler) deleteResources(watchedContainer watchedContainerData, contEvent v1.ContainerEventData) {
watchedContainer.sbomClient.CleanResources()
ch.watchedContainers.Delete(contEvent.GetContainerID())
}

func (ch *ContainerHandler) startRelevancyProcess(contEvent v1.ContainerEventData) {
containerDataInterface, exist := ch.watchedContainers.Load(contEvent.GetContainerID())
if !exist {
Expand All @@ -158,10 +163,10 @@ func (ch *ContainerHandler) startRelevancyProcess(contEvent v1.ContainerEventDat
if err != nil {
logger.L().Ctx(context.GetBackgroundContext()).Warning("we have failed to stop to aggregate data", helpers.String("container ID", contEvent.GetContainerID()), helpers.String("container name", contEvent.GetContainerName()), helpers.String("k8s resources", contEvent.GetK8SWorkloadID()))
}
ch.watchedContainers.Delete(contEvent.GetContainerID())
break
}
}
ch.deleteResources(watchedContainer, contEvent)
}

func getShortContainerID(containerID string) string {
Expand Down
8 changes: 6 additions & 2 deletions pkg/sbom/sbom.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const (

type SBOMStructure struct {
storageClient SBOMStorageClient
SBOMData SBOMFormat
SBOMData v1.SBOMFormat
firstReport bool
imageID string
wlid string
Expand All @@ -37,7 +37,7 @@ func CreateSBOMStorageClient(sc storageclient.StorageClient, wlid, imageID strin
storageClient: SBOMStorageClient{
client: sc,
},
SBOMData: v1.CreateSBOMDataSPDXVersionV040(),
SBOMData: v1.CreateSBOMDataSPDXVersionV040(instanceID),
firstReport: true,
instanceID: instanceID,
wlid: wlid,
Expand Down Expand Up @@ -89,6 +89,10 @@ func (sc *SBOMStructure) StoreFilterSBOM(instanceID string) error {
return errorsOfSBOM[DataAlreadyExist]
}

func (sc *SBOMStructure) CleanResources() {
sc.SBOMData.CleanResources()
}

func IsAlreadyExist() error {
return errorsOfSBOM[DataAlreadyExist]
}
1 change: 1 addition & 0 deletions pkg/sbom/sbom_interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ type SBOMClient interface {
GetSBOM(imageID string) error
FilterSBOM(sbomFileRelevantMap map[string]bool) error
StoreFilterSBOM(instanceID string) error
CleanResources()
}
6 changes: 3 additions & 3 deletions pkg/sbom/sbom_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestGetSBOM(t *testing.T) {
SBOMClient := CreateSBOMStorageClient(storageclient.CreateSBOMStorageHttpClientMock(), "", "", &instanceidhandler.InstanceID{})
err := SBOMClient.GetSBOM(storageclient.NGINX)
if err != nil {
t.Fatalf("fail to get sbom")
t.Fatalf("fail to get sbom, %v", err)
}

}
Expand All @@ -24,13 +24,13 @@ func TestFilterSBOM(t *testing.T) {
SBOMClient := CreateSBOMStorageClient(storageclient.CreateSBOMStorageHttpClientMock(), "", "", &instanceidhandler.InstanceID{})
err := SBOMClient.GetSBOM(storageclient.NGINX)
if err != nil {
t.Fatalf("fail to get sbom")
t.Fatalf("fail to get sbom, %v", err)
}
err = SBOMClient.FilterSBOM(map[string]bool{
"/usr/share/adduser/adduser.conf": true,
})
if err != nil {
t.Fatalf("fail to filter sbom")
t.Fatalf("fail to filter sbom, %v", err)
}

}
Expand Down
5 changes: 5 additions & 0 deletions pkg/sbom/testdata/nginx-spdx-format-mock.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@
"spdxElementId": "SPDXRef-Package-deb-adduser-3e9282034226b93f",
"relatedSpdxElement": "SPDXRef-e97fef92a7d904e0",
"relationshipType": "CONTAINS"
},
{
"relatedSpdxElement": "SPDXRef-DOCUMENT",
"relationshipType": "DESCRIBES",
"spdxElementId": "SPDXRef-DOCUMENT"
}
]
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/sbom/testdata/not-spdx-format.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"aaa": 12345
}
1 change: 1 addition & 0 deletions pkg/sbom/sbom_format_interface.go → pkg/sbom/v1/sbom_format_interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ type SBOMFormat interface {
AddResourceVersionIfNeeded(string)
StoreFilteredSBOMName(string)
StoreMetadata(wlidData, imageID string, instanceID instanceidhandler.IInstanceID)
CleanResources()
}
125 changes: 104 additions & 21 deletions pkg/sbom/v1/sbom_spdx_storage_format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package sbom

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"sniffer/pkg/context"
"sync"

"github.com/armosec/utils-k8s-go/wlid"
Expand All @@ -22,39 +26,84 @@ const (
KubescapeOrganizationName = "Kubescape"
KubescapeNodeAgentName = "KubescapeNodeAgent"
RelationshipContainType = "CONTAINS"
directorySBOM = "SBOM"
)

var spdxDataDirPath string

type SBOMData struct {
spdxData spdxv1beta1.SBOMSPDXv2p3
spdxDataPath string
filteredSpdxData spdxv1beta1.SBOMSPDXv2p3Filtered
relevantRealtimeFilesBySPDXIdentifier sync.Map
newRelevantData bool
alreadyExistSBOM bool
instanceID instanceidhandler.IInstanceID
}

func createSBOMDir() {
wd, err := os.Getwd()
if err != nil {
logger.L().Ctx(context.GetBackgroundContext()).Fatal("failed to get working directory", helpers.Error(err))
}
spdxDataDirPath = fmt.Sprintf("%s/%s", wd, directorySBOM)
err = os.MkdirAll(spdxDataDirPath, os.ModeDir|os.ModePerm)
if err != nil {
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What will happen in such a case? Do we want the nodeAgent to panic?
Here you just print an error without doing anything.
If the program will continue running, then it is not an error.

logger.L().Ctx(context.GetBackgroundContext()).Fatal("failed to create directory for SBOM resources", helpers.String("directory path", spdxDataDirPath), helpers.Error(err))
}
}

func init() {
createSBOMDir()
}

func CreateSBOMDataSPDXVersionV040() *SBOMData {
func CreateSBOMDataSPDXVersionV040(instanceID instanceidhandler.IInstanceID) SBOMFormat {
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add unitest


return &SBOMData{
spdxDataPath: fmt.Sprintf("%s/%s", spdxDataDirPath, instanceID.GetHashed()),
filteredSpdxData: spdxv1beta1.SBOMSPDXv2p3Filtered{},
relevantRealtimeFilesBySPDXIdentifier: sync.Map{},
newRelevantData: false,
alreadyExistSBOM: false,
instanceID: instanceID,
}
}

func (sbom *SBOMData) saveSBOM(spdxData *spdxv1beta1.SBOMSPDXv2p3) error {
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add unitest

f, err := os.Create(sbom.spdxDataPath)
if err != nil {
return err
}
defer f.Close()

data, err := json.Marshal(spdxData)
if err != nil {
return err
}
_, err = f.Write(data)
if err != nil {
return err
}
return nil
}

func (sbom *SBOMData) StoreSBOM(sbomData any) error {
spdxData, ok := sbomData.(*spdxv1beta1.SBOMSPDXv2p3)
if !ok {
return fmt.Errorf("storage format: StoreSBOM: SBOM data format is not supported")
}

sbom.spdxData = *spdxData
for i := range sbom.spdxData.Spec.SPDX.Files {
sbom.relevantRealtimeFilesBySPDXIdentifier.Store(spdxv1beta1.ElementID(sbom.spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier), false)
err := sbom.saveSBOM(spdxData)
if err != nil {
return err
}

sbom.filteredSpdxData.Spec = sbom.spdxData.Spec
sbom.filteredSpdxData.Status = sbom.spdxData.Status
sbom.spdxData.Spec.SPDX.CreationInfo.Creators = append(sbom.spdxData.Spec.SPDX.CreationInfo.Creators, []spdxv1beta1.Creator{
for i := range spdxData.Spec.SPDX.Files {
sbom.relevantRealtimeFilesBySPDXIdentifier.Store(spdxv1beta1.ElementID(spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier), false)
}

sbom.filteredSpdxData.Spec = spdxData.Spec
sbom.filteredSpdxData.Status = spdxData.Status
sbom.filteredSpdxData.Spec.SPDX.CreationInfo.Creators = append(sbom.filteredSpdxData.Spec.SPDX.CreationInfo.Creators, []spdxv1beta1.Creator{
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please explain what was changed here?
Are we accessing a different object?
And why are there no interfaces here? what if we change the SPDX format?

Copy link
Copy Markdown
Contributor Author

@rcohencyberarmor rcohencyberarmor Apr 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this all file is an implementation for our SPDX format - the interface is in the same dir

{
CreatorType: Organization,
Creator: KubescapeOrganizationName,
Expand All @@ -74,40 +123,67 @@ func (sbom *SBOMData) StoreSBOM(sbomData any) error {
return nil
}

func (sbom *SBOMData) getSBOMDataSPDXFormat() (*spdxv1beta1.SBOMSPDXv2p3, error) {
file, err := os.Open(sbom.spdxDataPath)
if err != nil {
return nil, err
}
defer file.Close()

bytes, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}

spdxData := spdxv1beta1.SBOMSPDXv2p3{}
err = json.Unmarshal(bytes, &spdxData)
if err != nil {
return nil, err
}

return &spdxData, nil
}

func (sbom *SBOMData) FilterSBOM(sbomFileRelevantMap map[string]bool) error {
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a really important function.
Add unitests for it.

sbom.newRelevantData = false

spdxData, err := sbom.getSBOMDataSPDXFormat()
if err != nil {
return err
}

//filter relevant file list
for i := range sbom.spdxData.Spec.SPDX.Files {
if exist := sbomFileRelevantMap[sbom.spdxData.Spec.SPDX.Files[i].FileName]; exist {
if data, _ := sbom.relevantRealtimeFilesBySPDXIdentifier.Load(spdxv1beta1.ElementID(sbom.spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier)); data != nil && !data.(bool) {
sbom.filteredSpdxData.Spec.SPDX.Files = append(sbom.filteredSpdxData.Spec.SPDX.Files, sbom.spdxData.Spec.SPDX.Files[i])
sbom.relevantRealtimeFilesBySPDXIdentifier.Store(spdxv1beta1.ElementID(sbom.spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier), true)
for i := range spdxData.Spec.SPDX.Files {
if exist := sbomFileRelevantMap[spdxData.Spec.SPDX.Files[i].FileName]; exist {
if data, _ := sbom.relevantRealtimeFilesBySPDXIdentifier.Load(spdxv1beta1.ElementID(spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier)); data != nil && !data.(bool) {
sbom.filteredSpdxData.Spec.SPDX.Files = append(sbom.filteredSpdxData.Spec.SPDX.Files, spdxData.Spec.SPDX.Files[i])
sbom.relevantRealtimeFilesBySPDXIdentifier.Store(spdxv1beta1.ElementID(spdxData.Spec.SPDX.Files[i].FileSPDXIdentifier), true)
sbom.newRelevantData = true
}
}
}

//filter relationship list
for i := range sbom.spdxData.Spec.SPDX.Relationships {
switch sbom.spdxData.Spec.SPDX.Relationships[i].Relationship {
for i := range spdxData.Spec.SPDX.Relationships {
switch spdxData.Spec.SPDX.Relationships[i].Relationship {
case RelationshipContainType:
if data, _ := sbom.relevantRealtimeFilesBySPDXIdentifier.Load(spdxv1beta1.ElementID(sbom.spdxData.Spec.SPDX.Relationships[i].RefB.ElementRefID)); data != nil && data.(bool) {
sbom.filteredSpdxData.Spec.SPDX.Relationships = append(sbom.filteredSpdxData.Spec.SPDX.Relationships, sbom.spdxData.Spec.SPDX.Relationships[i])
if data, _ := sbom.relevantRealtimeFilesBySPDXIdentifier.Load(spdxv1beta1.ElementID(spdxData.Spec.SPDX.Relationships[i].RefB.ElementRefID)); data != nil && data.(bool) {
sbom.filteredSpdxData.Spec.SPDX.Relationships = append(sbom.filteredSpdxData.Spec.SPDX.Relationships, spdxData.Spec.SPDX.Relationships[i])
}
default:
sbom.filteredSpdxData.Spec.SPDX.Relationships = append(sbom.filteredSpdxData.Spec.SPDX.Relationships, sbom.spdxData.Spec.SPDX.Relationships[i])
sbom.filteredSpdxData.Spec.SPDX.Relationships = append(sbom.filteredSpdxData.Spec.SPDX.Relationships, spdxData.Spec.SPDX.Relationships[i])
}
}

//filter relevant package list
for i := range sbom.spdxData.Spec.SPDX.Packages {
for i := range spdxData.Spec.SPDX.Packages {
relevantPackageMap := make(map[spdxv1beta1.DocElementID]bool)
for j := range sbom.filteredSpdxData.Spec.SPDX.Relationships {
switch sbom.filteredSpdxData.Spec.SPDX.Relationships[j].Relationship {
case RelationshipContainType:
if alreadyExist := relevantPackageMap[sbom.filteredSpdxData.Spec.SPDX.Relationships[j].RefA]; !alreadyExist {
if spdxv1beta1.ElementID(sbom.filteredSpdxData.Spec.SPDX.Relationships[j].RefA.ElementRefID) == sbom.spdxData.Spec.SPDX.Packages[i].PackageSPDXIdentifier {
sbom.filteredSpdxData.Spec.SPDX.Packages = append(sbom.filteredSpdxData.Spec.SPDX.Packages, sbom.spdxData.Spec.SPDX.Packages[i])
if spdxv1beta1.ElementID(sbom.filteredSpdxData.Spec.SPDX.Relationships[j].RefA.ElementRefID) == spdxData.Spec.SPDX.Packages[i].PackageSPDXIdentifier {
sbom.filteredSpdxData.Spec.SPDX.Packages = append(sbom.filteredSpdxData.Spec.SPDX.Packages, spdxData.Spec.SPDX.Packages[i])
}
}
}
Expand Down Expand Up @@ -176,3 +252,10 @@ func (sc *SBOMData) AddResourceVersionIfNeeded(resourceVersion string) {
sc.filteredSpdxData.SetResourceVersion(resourceVersion)
}
}

func (sc *SBOMData) CleanResources() {
Copy link
Copy Markdown

@dwertent dwertent Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add unitest.

err := os.Remove(sc.spdxDataPath)
if err != nil {
logger.L().Debug("fail to remove file", helpers.String("file name", sc.spdxDataPath), helpers.Error(err))
}
}
Loading