feat: Persist file artifacts (#1084)

## Description:
<!-- Describe this change, how it works, and the motivation behind it.
-->

## Is this change user facing?
YES/NO
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful
information. -->

---------

Co-authored-by: leoporoli <leandroporoli@gmail.com>

2d

@@ -0,0 +1 @@
+Error: Docker context switch for buildx failed

container-engine-lib/lib/database_accessors/enclave_db/enclave_db.go

@@ -3,6 +3,7 @@ package enclave_db
 import (
 	"github.com/kurtosis-tech/stacktrace"
 	bolt "go.etcd.io/bbolt"
+	"os"
 	"sync"
 	"time"
 )
@@ -45,3 +46,16 @@ func GetOrCreateEnclaveDatabase() (*EnclaveDB, error) {
 
 	return &EnclaveDB{databaseInstance}, nil
 }
+
+func EraseDatabase() error {
+	path := databaseInstance.Path()
+	err := databaseInstance.Close()
+	if err != nil {
+		return stacktrace.Propagate(err, "Failed to close database during erase process")
+	}
+	err = os.Remove(path)
+	if err != nil {
+		return stacktrace.Propagate(err, "Failed to erase database file during erase process '%v'", path)
+	}
+	return nil
+}

container-engine-lib/lib/database_accessors/enclave_db/file_artifacts_db/file_artifacts_db.go

@@ -0,0 +1,163 @@
+package file_artifacts_db
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/consts"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db"
+	"github.com/kurtosis-tech/stacktrace"
+	bolt "go.etcd.io/bbolt"
+)
+
+type FileArtifactPersisted struct {
+	enclaveDb *enclave_db.EnclaveDB
+	data      *fileArtifactData
+}
+
+type fileArtifactData struct {
+	ArtifactNameToArtifactUuid map[string]string
+	ShortenedUuidToFullUuid    map[string][]string
+	ArtifactContentMd5         map[string][]byte
+}
+
+var (
+	fileArtifactBucketName    = []byte("file-artifact")
+	fileArtifactDataStructKey = []byte("file-artifact-data-struct")
+)
+
+func (fileArtifactDb *FileArtifactPersisted) ListFiles() map[string]bool {
+	artifactNameSet := make(map[string]bool)
+	for artifactName := range fileArtifactDb.data.ArtifactNameToArtifactUuid {
+		artifactNameSet[artifactName] = true
+	}
+	return artifactNameSet
+}
+
+func (fileArtifactDb *FileArtifactPersisted) Persist() error {
+	err := fileArtifactDb.enclaveDb.Update(func(tx *bolt.Tx) error {
+		jsonData, err := json.Marshal(fileArtifactDb.data)
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred marshalling data '%v'", fileArtifactDb.data)
+		}
+		return tx.Bucket(fileArtifactBucketName).Put(fileArtifactDataStructKey, jsonData)
+	})
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred during file artifact db update")
+	}
+	return nil
+}
+
+func (fileArtifactDb *FileArtifactPersisted) SetArtifactUuid(artifactName string, artifactUuid string) {
+	fileArtifactDb.data.ArtifactNameToArtifactUuid[artifactName] = artifactUuid
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetArtifactUuid(artifactName string) (string, bool) {
+	value, found := fileArtifactDb.data.ArtifactNameToArtifactUuid[artifactName]
+	return value, found
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetArtifactUuidMap() map[string]string {
+	return fileArtifactDb.data.ArtifactNameToArtifactUuid
+}
+
+func (fileArtifactDb *FileArtifactPersisted) DeleteArtifactUuid(artifactName string) {
+	delete(fileArtifactDb.data.ArtifactNameToArtifactUuid, artifactName)
+}
+
+func (fileArtifactDb *FileArtifactPersisted) SetFullUuid(shortenedUuid string, fullUuid []string) {
+	fileArtifactDb.data.ShortenedUuidToFullUuid[shortenedUuid] = fullUuid
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetFullUuid(shortenedUuid string) ([]string, bool) {
+	value, found := fileArtifactDb.data.ShortenedUuidToFullUuid[shortenedUuid]
+	return value, found
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetFullUuidMap() map[string][]string {
+	return fileArtifactDb.data.ShortenedUuidToFullUuid
+}
+
+func (fileArtifactDb *FileArtifactPersisted) DeleteFullUuid(shortenedUuid string) {
+	delete(fileArtifactDb.data.ShortenedUuidToFullUuid, shortenedUuid)
+}
+
+func (fileArtifactDb *FileArtifactPersisted) DeleteFullUuidIndex(shortenedUuid string, targetArtifactIdx int) {
+	artifactUuids := fileArtifactDb.data.ShortenedUuidToFullUuid[shortenedUuid]
+	fileArtifactDb.data.ShortenedUuidToFullUuid[shortenedUuid] = append(artifactUuids[0:targetArtifactIdx], artifactUuids[targetArtifactIdx+1:]...)
+}
+
+func (fileArtifactDb *FileArtifactPersisted) SetContentMd5(artifactName string, md5 []byte) {
+	fileArtifactDb.data.ArtifactContentMd5[artifactName] = md5
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetContentMd5(artifactName string) ([]byte, bool) {
+	value, found := fileArtifactDb.data.ArtifactContentMd5[artifactName]
+	return value, found
+}
+
+func (fileArtifactDb *FileArtifactPersisted) GetContentMd5Map() map[string][]byte {
+	return fileArtifactDb.data.ArtifactContentMd5
+}
+
+func (fileArtifactDb *FileArtifactPersisted) DeleteContentMd5(artifactName string) {
+	delete(fileArtifactDb.data.ArtifactContentMd5, artifactName)
+}
+
+func GetOrCreateNewFileArtifactsDb() (*FileArtifactPersisted, error) {
+	data := fileArtifactData{
+		map[string]string{},
+		map[string][]string{},
+		map[string][]byte{},
+	}
+	db, err := enclave_db.GetOrCreateEnclaveDatabase()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "Failed to get enclave database")
+	}
+	fileArtifactPersisted, err := getFileArtifactsDbFromEnclaveDb(db, &data)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "Failed to hydrate pre-existing file artifacts")
+	}
+	return fileArtifactPersisted, nil
+}
+
+func getFileArtifactsDbFromEnclaveDb(db *enclave_db.EnclaveDB, data *fileArtifactData) (*FileArtifactPersisted, error) {
+	err := db.Update(func(tx *bolt.Tx) error {
+		bucket, bucketErr := tx.CreateBucket(fileArtifactBucketName)
+		// New bucket was created
+		if errors.Is(bucketErr, bolt.ErrBucketExists) {
+			content := tx.Bucket(fileArtifactBucketName).Get(fileArtifactDataStructKey)
+			if err := json.Unmarshal(content, data); err != nil {
+				return stacktrace.Propagate(err, "An error occurred restoring previous file artifact db state from '%v'", content)
+			}
+			return nil
+		}
+		// No need to create new bucket
+		if bucketErr == nil {
+			if err := bucket.Put(fileArtifactDataStructKey, consts.EmptyValueForJsonSet); err != nil {
+				return stacktrace.Propagate(err, "An error occurred while creating updating artifact bucket o '%v'", consts.EmptyValueForJsonSet)
+			}
+			return nil
+		}
+		// Unexpected error
+		return stacktrace.Propagate(bucketErr, "An error occurred while creating file artifact bucket")
+	})
+	if err == nil {
+		return &FileArtifactPersisted{
+			db,
+			data,
+		}, nil
+	}
+	return nil, stacktrace.Propagate(err, "An error occurred while getting file artifact db")
+}
+
+func GetFileArtifactsDbForTesting(db *enclave_db.EnclaveDB, nameToUuid map[string]string) (*FileArtifactPersisted, error) {
+	fileArtifactPersisted, err := getFileArtifactsDbFromEnclaveDb(db, &fileArtifactData{
+		nameToUuid,
+		map[string][]string{},
+		map[string][]byte{},
+	})
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "Failed to hydrate pre-existing file artifacts")
+	}
+	return fileArtifactPersisted, nil
+}

container-engine-lib/lib/database_accessors/enclave_db/file_artifacts_db/file_artifacts_db_test.go

@@ -0,0 +1,31 @@
+package file_artifacts_db
+
+import (
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/test_helpers"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestFileArtifactPersistance(t *testing.T) {
+	enclaveDb, cleaningFunction, err := test_helpers.CreateEnclaveDbForTesting()
+	require.Nil(t, err)
+	defer cleaningFunction()
+	fileArtifactDb, err := GetFileArtifactsDbForTesting(enclaveDb, map[string]string{})
+	require.Nil(t, err)
+	require.Empty(t, fileArtifactDb.data.ArtifactNameToArtifactUuid)
+	require.Empty(t, fileArtifactDb.data.ArtifactContentMd5)
+	require.Empty(t, fileArtifactDb.data.ShortenedUuidToFullUuid)
+	fileArtifactDb.SetArtifactUuid("1", "1")
+	fileArtifactDb.SetContentMd5("1", []byte("1"))
+	fileArtifactDb.SetFullUuid("1", []string{"1"})
+	require.Nil(t, fileArtifactDb.Persist())
+	fileArtifactDb, err = getFileArtifactsDbFromEnclaveDb(enclaveDb, &fileArtifactData{
+		map[string]string{},
+		map[string][]string{},
+		map[string][]byte{},
+	})
+	require.Nil(t, err)
+	require.Len(t, fileArtifactDb.GetArtifactUuidMap(), 1)
+	require.Len(t, fileArtifactDb.GetFullUuidMap(), 1)
+	require.Len(t, fileArtifactDb.GetContentMd5Map(), 1)
+}

core/server/commons/enclave_data_directory/enclave_data_directory.go

@@ -6,6 +6,7 @@
 package enclave_data_directory
 
 import (
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/file_artifacts_db"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider"
 	"github.com/kurtosis-tech/stacktrace"
 	"path"
@@ -49,13 +50,19 @@ func (dir EnclaveDataDirectory) GetFilesArtifactStore() (*FilesArtifactStore, er
 		return nil, stacktrace.Propagate(err, "An error occurred ensuring the files artifact store dirpath '%v' exists.", absoluteDirpath)
 	}
 
+	var dbError error
 	// NOTE: We use a 'once' to initialize the filesArtifactStore because it contains a mutex,
 	// and we don't ever want multiple filesArtifactStore instances in existence
 	once.Do(func() {
-		currentFilesArtifactStore = newFilesArtifactStore(absoluteDirpath, relativeDirpath)
+		db, err := file_artifacts_db.GetOrCreateNewFileArtifactsDb()
+		if err != nil {
+			dbError = stacktrace.Propagate(err, "Failed to get file artifacts db")
+			return
+		}
+		currentFilesArtifactStore = newFilesArtifactStoreFromDb(absoluteDirpath, relativeDirpath, db)
 	})
 
-	return currentFilesArtifactStore, nil
+	return currentFilesArtifactStore, dbError
 }
 
 func (dir EnclaveDataDirectory) GetGitPackageContentProvider() (*git_package_content_provider.GitPackageContentProvider, error) {

core/server/commons/enclave_data_directory/enclave_data_directory_test.go

@@ -6,7 +6,9 @@
 package enclave_data_directory
 
 import (
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"os"
 	"path"
 	"testing"
@@ -20,6 +22,9 @@ func TestGetFilesArtifactStore(t *testing.T) {
 
 	artifactStore, err := enclaveDir.GetFilesArtifactStore()
 	assert.Nil(t, err)
+	defer func() {
+		require.Nil(t, enclave_db.EraseDatabase())
+	}()
 
 	expectedAbsDirpath := path.Join(enclaveDirpath, artifactStoreDirname)
 	_, err = os.Stat(expectedAbsDirpath)