Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/cloudstic/cmd_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -395,8 +395,11 @@ func cloneGlobalFlags(src *globalFlags) *globalFlags {
clone := *src

store := *src.store
profile := *src.profile
profilesFile := *src.profilesFile
s3Endpoint := *src.s3Endpoint
s3Region := *src.s3Region
s3Profile := *src.s3Profile
s3AccessKey := *src.s3AccessKey
s3SecretKey := *src.s3SecretKey
sourceSFTPPassword := *src.sourceSFTPPassword
Expand All @@ -416,8 +419,11 @@ func cloneGlobalFlags(src *globalFlags) *globalFlags {
debug := *src.debug

clone.store = &store
clone.profile = &profile
clone.profilesFile = &profilesFile
clone.s3Endpoint = &s3Endpoint
clone.s3Region = &s3Region
clone.s3Profile = &s3Profile
clone.s3AccessKey = &s3AccessKey
clone.s3SecretKey = &s3SecretKey
clone.sourceSFTPPassword = &sourceSFTPPassword
Expand Down
15 changes: 15 additions & 0 deletions cmd/cloudstic/cmd_backup_profile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,16 +272,31 @@ func TestApplyProfileAuthToBackupArgs_CLIFlagsPreserved(t *testing.T) {
func TestCloneGlobalFlags_Independence(t *testing.T) {
orig := newTestGlobalFlags()
*orig.store = "original-store"
*orig.profile = "orig-profile"
*orig.profilesFile = "/tmp/orig-profiles.yaml"
*orig.s3Profile = "orig-s3-profile"

clone := cloneGlobalFlags(orig)
*clone.store = "modified-store"
*clone.profile = "clone-profile"
*clone.profilesFile = "/tmp/clone-profiles.yaml"
*clone.s3Profile = "clone-s3-profile"

if *orig.store != "original-store" {
t.Fatalf("original store=%q want original-store", *orig.store)
}
if *clone.store != "modified-store" {
t.Fatalf("clone store=%q want modified-store", *clone.store)
}
if *orig.profile != "orig-profile" {
t.Fatalf("original profile=%q want orig-profile", *orig.profile)
}
if *orig.profilesFile != "/tmp/orig-profiles.yaml" {
t.Fatalf("original profilesFile=%q want /tmp/orig-profiles.yaml", *orig.profilesFile)
}
if *orig.s3Profile != "orig-s3-profile" {
t.Fatalf("original s3Profile=%q want orig-s3-profile", *orig.s3Profile)
}
}

func TestApplyProfileStoreToGlobalFlags_AllFields(t *testing.T) {
Expand Down
8 changes: 7 additions & 1 deletion cmd/cloudstic/cmd_profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,9 @@ func (r *runner) runProfileNew() int {
createdStore := false

if a.store != "" {
if _, err := parseStoreURI(a.store); err != nil {
return r.fail("Invalid store URI: %v", err)
}
cfg.Stores[a.storeRef] = cloudstic.ProfileStore{URI: a.store}
createdStore = true
} else if a.storeRef != "" {
Expand Down Expand Up @@ -479,13 +482,16 @@ func (r *runner) promptStoreSelection(cfg *cloudstic.ProfilesConfig) (string, bo
if refName == "" {
return "", false, r.fail("Store reference name is required")
}
uri, err := r.promptLine("Store URI (e.g. s3://bucket/path, local:/path, sftp://host/path)", "")
uri, err := r.promptLine("Store URI (e.g. s3:bucket/path, local:/path, sftp://host/path)", "")
if err != nil {
return "", false, r.fail("Failed to read store URI: %v", err)
}
if uri == "" {
return "", false, r.fail("Store URI is required")
}
if _, err := parseStoreURI(uri); err != nil {
return "", false, r.fail("Invalid store URI: %v", err)
}
cfg.Stores[refName] = cloudstic.ProfileStore{URI: uri}
return refName, true, 0
}
Expand Down
26 changes: 24 additions & 2 deletions cmd/cloudstic/cmd_profile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ func TestRunProfileNew_CloudSourceRequiresAuthRef(t *testing.T) {
"-profiles-file", profilesPath,
"-name", "drive-backup",
"-source", "gdrive:/",
"-store-ref", "s", "-store", "s3://bucket",
"-store-ref", "s", "-store", "s3:bucket",
}
var out strings.Builder
var errOut strings.Builder
Expand All @@ -342,7 +342,7 @@ func TestRunProfileNew_RejectsUnknownAuthRef(t *testing.T) {
"-profiles-file", profilesPath,
"-name", "work-drive",
"-source", "gdrive-changes:/Team",
"-store-ref", "s", "-store", "s3://bucket",
"-store-ref", "s", "-store", "s3:bucket",
"-auth-ref", "google-work",
}
var out strings.Builder
Expand Down Expand Up @@ -605,6 +605,28 @@ func TestRunProfileNew_InvalidSource(t *testing.T) {
}
}

func TestRunProfileNew_InvalidStoreURI(t *testing.T) {
tmpDir := t.TempDir()
profilesPath := filepath.Join(tmpDir, "profiles.yaml")
os.Args = []string{
"cloudstic", "profile", "new",
"-profiles-file", profilesPath,
"-name", "bad-store",
"-source", "local:/data",
"-store-ref", "s", "-store", "s3://bucket",
}
var out strings.Builder
var errOut strings.Builder
r := &runner{out: &out, errOut: &errOut}

if code := r.runProfile(); code == 0 {
t.Fatal("expected non-zero exit code")
}
if !strings.Contains(errOut.String(), "Invalid store URI") {
t.Fatalf("unexpected error output: %s", errOut.String())
}
}

func TestRunProfileList_WithOneDriveAuth(t *testing.T) {
tmpDir := t.TempDir()
profilesPath := filepath.Join(tmpDir, "profiles.yaml")
Expand Down
12 changes: 6 additions & 6 deletions cmd/cloudstic/completion.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ _cloudstic() {
-*)
# skip flags and their values
case "${words[i]}" in
-store|-profile|-profiles-file|-s3-endpoint|-s3-region|-s3-profile|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-all-profiles|-auth-ref|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
((i++)) ;;
esac
-store|-profile|-profiles-file|-s3-endpoint|-s3-region|-s3-profile|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-auth-ref|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
((i++)) ;;
esac
;;
*)
cmd="${words[i]}"
Expand Down Expand Up @@ -281,9 +281,9 @@ _cloudstic() {
-*)
# Skip flags with values
case "${words[i]}" in
-store|-profile|-profiles-file|-s3-endpoint|-s3-region|-s3-profile|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-all-profiles|-auth-ref|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
(( i++ )) ;;
esac
-store|-profile|-profiles-file|-s3-endpoint|-s3-region|-s3-profile|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-auth-ref|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
(( i++ )) ;;
esac
;;
*)
cmd="${words[i]}"
Expand Down
1 change: 0 additions & 1 deletion cmd/cloudstic/completion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ func TestCompletionBash(t *testing.T) {
"-profiles-file",
"-profile", "-all-profiles",
"-auth-ref",
"-auth-ref",
// Value completions
"local: s3: b2: sftp://",
"gdrive", "onedrive",
Expand Down
29 changes: 28 additions & 1 deletion internal/engine/backup_scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,14 @@ func (bm *BackupManager) scanIncremental(ctx context.Context, oldRoot string, in
switch fc.Type {
case source.ChangeDelete:
bm.recordRemoved(fc.Meta.Type)
s.root, err = bm.tree.Delete(s.root, primaryParentID(&fc.Meta), fc.Meta.FileID)
deleteParentID := primaryParentID(&fc.Meta)
if deleteParentID == "" {
deleteParentID, err = bm.lookupDeleteParentID(s.root, fc.Meta.FileID)
if err != nil {
return err
}
}
s.root, err = bm.tree.Delete(s.root, deleteParentID, fc.Meta.FileID)
if err != nil {
return fmt.Errorf("hamt delete %s: %w", fc.Meta.FileID, err)
}
Expand All @@ -138,6 +145,26 @@ func (bm *BackupManager) scanIncremental(ctx context.Context, oldRoot string, in
return s.root, s.pending, s.totalBytes, newToken, nil
}

func (bm *BackupManager) lookupDeleteParentID(root, fileID string) (string, error) {
if root == "" {
return "", nil
}

ref, err := bm.tree.LookupByFileID(root, fileID)
if err != nil {
return "", fmt.Errorf("lookup old file for delete %s: %w", fileID, err)
}
if ref == "" {
return "", nil
}

oldMeta, err := bm.loadMeta(ref)
if err != nil {
return "", fmt.Errorf("load old file metadata for delete %s: %w", fileID, err)
}
return primaryParentID(oldMeta), nil
}

// detectChange compares meta against the previous snapshot. It returns whether
// the entry changed, and the old value ref (empty when the entry is new).
//
Expand Down
72 changes: 72 additions & 0 deletions internal/engine/backup_scan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/cloudstic/cli/internal/core"
"github.com/cloudstic/cli/internal/hamt"
"github.com/cloudstic/cli/internal/ui"
"github.com/cloudstic/cli/pkg/source"
"github.com/cloudstic/cli/pkg/store"
)

Expand Down Expand Up @@ -228,3 +229,74 @@ func TestDetectChange_NativeFileCarriesForwardMetadata(t *testing.T) {
t.Errorf("Expected same ref (metadata carried forward), got %q vs %q", ref2, ref)
}
}

type mockIncrementalSource struct {
*MockSource
startToken string
changes []source.FileChange
newToken string
}

func (s *mockIncrementalSource) GetStartPageToken() (string, error) {
return s.startToken, nil
}

func (s *mockIncrementalSource) WalkChanges(_ context.Context, _ string, callback func(source.FileChange) error) (string, error) {
for _, ch := range s.changes {
if err := callback(ch); err != nil {
return "", err
}
}
return s.newToken, nil
}

func TestScanIncremental_DeleteWithoutParentUsesExistingMetadataParent(t *testing.T) {
ctx := context.Background()
base := NewMockSource()
base.Files["FOLDER_1"] = MockFile{Meta: core.FileMeta{FileID: "FOLDER_1", Name: "folder", Type: core.FileTypeFolder}}
base.Files["FILE_1"] = MockFile{
Meta: core.FileMeta{
FileID: "FILE_1",
Name: "a.txt",
Type: core.FileTypeFile,
Parents: []string{"FOLDER_1"},
Size: 3,
},
Content: []byte("abc"),
}

inc := &mockIncrementalSource{
MockSource: base,
startToken: "tok-1",
newToken: "tok-2",
}

dest := NewMockStore()
mgr := NewBackupManager(inc, dest, ui.NewNoOpReporter(), nil)
_, err := mgr.Run(ctx)
if err != nil {
t.Fatalf("first backup failed: %v", err)
}

deleteOnly := []source.FileChange{{
Type: source.ChangeDelete,
Meta: core.FileMeta{FileID: "FILE_1", Type: core.FileTypeFile},
}}
inc.changes = deleteOnly
delete(base.Files, "FILE_1")

mgr2 := NewBackupManager(inc, dest, ui.NewNoOpReporter(), nil)
second, err := mgr2.Run(ctx)
if err != nil {
t.Fatalf("second backup failed: %v", err)
}

tree := hamt.NewTree(store.NewCompressedStore(dest))
ref, err := tree.Lookup(second.Root, "FOLDER_1", "FILE_1")
if err != nil {
t.Fatalf("lookup failed: %v", err)
}
if ref != "" {
t.Fatalf("expected FILE_1 to be deleted, got ref %q", ref)
}
}
6 changes: 5 additions & 1 deletion pkg/source/gdrive.go
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,11 @@ func (s *GDriveSource) visitEntryWithPath(f *drive.File, pathMap map[string]stri
p := meta.Name
if len(f.Parents) > 0 {
if parentPath, ok := pathMap[f.Parents[0]]; ok {
p = parentPath + "/" + meta.Name
if parentPath == "" {
p = meta.Name
} else {
p = parentPath + "/" + meta.Name
}
}
}
meta.Paths = []string{p}
Expand Down
2 changes: 1 addition & 1 deletion pkg/source/gdrive_changes.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ func (s *GDriveChangeSource) shouldExcludeChange(fc FileChange, excludedIDs map[
}

func (s *GDriveChangeSource) changeToFileChange(ch *drive.Change) FileChange {
if ch.Removed || (ch.File != nil && ch.File.Trashed) {
if ch.Removed || ch.File == nil || ch.File.Trashed {
return FileChange{
Type: ChangeDelete,
Meta: core.FileMeta{FileID: ch.FileId},
Expand Down
54 changes: 54 additions & 0 deletions pkg/source/gdrive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,38 @@ func TestVisitEntryWithPath_PathComputation(t *testing.T) {
}
}

func TestVisitEntryWithPath_RootRelativePath_NoLeadingSlash(t *testing.T) {
s := &GDriveSource{
exclude: NewExcludeMatcher(nil),
mimeTypes: make(map[string]string),
}

var got string
callback := func(meta core.FileMeta) error {
if len(meta.Paths) > 0 {
got = meta.Paths[0]
}
return nil
}

pathMap := map[string]string{"rootFolderID": ""}
excludedPaths := make(map[string]bool)

err := s.visitEntryWithPath(&drive.File{
Id: "file1",
Name: "child.txt",
MimeType: "text/plain",
Parents: []string{"rootFolderID"},
}, pathMap, excludedPaths, callback)
if err != nil {
t.Fatalf("visitEntryWithPath: %v", err)
}

if got != "child.txt" {
t.Fatalf("path = %q, want %q", got, "child.txt")
}
}

func TestChangeToFileChange_RecordsMimeType(t *testing.T) {
s := &GDriveChangeSource{
GDriveSource: GDriveSource{
Expand Down Expand Up @@ -426,6 +458,28 @@ func TestChangeToFileChange_DeletedFile(t *testing.T) {
}
}

func TestChangeToFileChange_NilFilePayload(t *testing.T) {
s := &GDriveChangeSource{
GDriveSource: GDriveSource{
exclude: NewExcludeMatcher(nil),
mimeTypes: make(map[string]string),
},
}

fc := s.changeToFileChange(&drive.Change{
FileId: "file1",
Removed: false,
File: nil,
})

if fc.Type != ChangeDelete {
t.Errorf("Type = %v, want ChangeDelete", fc.Type)
}
if fc.Meta.FileID != "file1" {
t.Errorf("FileID = %q, want %q", fc.Meta.FileID, "file1")
}
}

func TestChangeToFileChange_TrashedFile(t *testing.T) {
s := &GDriveChangeSource{
GDriveSource: GDriveSource{
Expand Down
3 changes: 2 additions & 1 deletion pkg/source/local_source_darwin.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,8 @@ func getVolumeLabel(path string) string {
}

// Parse attrreference: offset from start of attrreference field, then length
if buf[0] < 12 {
attrLen := *(*uint32)(unsafe.Pointer(&buf[0]))
if attrLen < 12 {
return ""
}
off := *(*int32)(unsafe.Pointer(&buf[4]))
Expand Down
Loading
Loading