Fix non-directory source path handling

Makefile

@@ -31,8 +31,12 @@ fixture:
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/bar/baz/
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-upload/dest_only_file
+	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload-file
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-delete
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete/dest_only_file
+	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-delete-file
+	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete-file
+	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete-file/dest_only_file
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-dryrun
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-dryrun/dest_only_file
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-directory

go.sum

@@ -19,4 +19,5 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

s3sync.go

@@ -55,6 +55,7 @@ type fileInfo struct {
 	path           string
 	size           int64
 	lastModified   time.Time
+	singleFile     bool
 	existsInSource bool
 }
 
@@ -216,7 +217,13 @@ func (m *Manager) syncS3ToLocal(chJob chan func(), sourcePath *s3Path, destPath
 }
 
 func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string) error {
-	targetFilename := filepath.Join(destPath, file.name)
+	var targetFilename string
+	if !strings.HasSuffix(destPath, "/") && file.singleFile {
+		// Destination path is not a directory and source is a single file.
+		targetFilename = destPath
+	} else {
+		targetFilename = filepath.Join(destPath, file.name)
+	}
 	targetDir := filepath.Dir(targetFilename)
 
 	println("Downloading", file.name, "to", targetFilename)
@@ -236,9 +243,16 @@ func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string)
 
 	defer writer.Close()
 
+	var sourceFile string
+	if file.singleFile {
+		sourceFile = file.name
+	} else {
+		sourceFile = filepath.Join(sourcePath.bucketPrefix, file.name)
+	}
+
 	_, err = s3manager.NewDownloaderWithClient(m.s3).Download(writer, &s3.GetObjectInput{
 		Bucket: aws.String(sourcePath.bucket),
-		Key:    aws.String(filepath.Join(sourcePath.bucketPrefix, file.name)),
+		Key:    aws.String(sourceFile),
 	})
 
 	if err != nil {
@@ -249,7 +263,13 @@ func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string)
 }
 
 func (m *Manager) deleteLocal(file *fileInfo, destPath string) error {
-	targetFilename := filepath.Join(destPath, file.name)
+	var targetFilename string
+	if !strings.HasSuffix(destPath, "/") && file.singleFile {
+		// Destination path is not a directory and source is a single file.
+		targetFilename = destPath
+	} else {
+		targetFilename = filepath.Join(destPath, file.name)
+	}
 
 	println("Deleting", targetFilename)
 	if m.dryrun {
@@ -260,10 +280,18 @@ func (m *Manager) deleteLocal(file *fileInfo, destPath string) error {
 }
 
 func (m *Manager) upload(file *fileInfo, sourcePath string, destPath *s3Path) error {
-	sourceFilename := filepath.Join(sourcePath, file.name)
+	var sourceFilename string
+	if file.singleFile {
+		sourceFilename = file.name
+	} else {
+		sourceFilename = filepath.Join(sourcePath, file.name)
+	}
 
 	destFile := *destPath
-	destFile.bucketPrefix = filepath.Join(destPath.bucketPrefix, file.name)
+	if strings.HasSuffix(destPath.bucketPrefix, "/") || destPath.bucketPrefix == "" || !file.singleFile {
+		// If source is a single file and destination is not a directory, use destination URL as is.
+		destFile.bucketPrefix = filepath.Join(destPath.bucketPrefix, file.name)
+	}
 
 	println("Uploading", file.name, "to", destFile.String())
 	if m.dryrun {
@@ -294,7 +322,10 @@ func (m *Manager) upload(file *fileInfo, sourcePath string, destPath *s3Path) er
 
 func (m *Manager) deleteRemote(file *fileInfo, destPath *s3Path) error {
 	destFile := *destPath
-	destFile.bucketPrefix = filepath.Join(destPath.bucketPrefix, file.name)
+	if strings.HasSuffix(destPath.bucketPrefix, "/") || destPath.bucketPrefix == "" || !file.singleFile {
+		// If source is a single file and destination is not a directory, use destination URL as is.
+		destFile.bucketPrefix = filepath.Join(destPath.bucketPrefix, file.name)
+	}
 
 	println("Deleting", destFile.String())
 	if m.dryrun {
@@ -347,11 +378,22 @@ func (m *Manager) listS3FileWithToken(c chan *fileInfo, path *s3Path, token *str
 			sendErrorInfoToChannel(c, err)
 			continue
 		}
-		c <- &fileInfo{
-			name:         name,
-			path:         *object.Key,
-			size:         *object.Size,
-			lastModified: *object.LastModified,
+		if name == "." {
+			// Single file was specified
+			c <- &fileInfo{
+				name:         filepath.Base(*object.Key),
+				path:         filepath.Dir(*object.Key),
+				size:         *object.Size,
+				lastModified: *object.LastModified,
+				singleFile:   true,
+			}
+		} else {
+			c <- &fileInfo{
+				name:         name,
+				path:         *object.Key,
+				size:         *object.Size,
+				lastModified: *object.LastModified,
+			}
 		}
 	}
 
@@ -377,17 +419,19 @@ func listLocalFiles(basePath string) chan *fileInfo {
 			sendErrorInfoToChannel(c, err)
 			return
 		}
-		sendFileInfoToChannel(c, basePath, basePath, stat)
 
 		if !stat.IsDir() {
+			sendFileInfoToChannel(c, filepath.Dir(basePath), basePath, stat, true)
 			return
 		}
 
+		sendFileInfoToChannel(c, basePath, basePath, stat, false)
+
 		err = filepath.Walk(basePath, func(path string, stat os.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
-			sendFileInfoToChannel(c, basePath, path, stat)
+			sendFileInfoToChannel(c, basePath, path, stat, false)
 			return nil
 		})
 
@@ -399,7 +443,7 @@ func listLocalFiles(basePath string) chan *fileInfo {
 	return c
 }
 
-func sendFileInfoToChannel(c chan *fileInfo, basePath, path string, stat os.FileInfo) {
+func sendFileInfoToChannel(c chan *fileInfo, basePath, path string, stat os.FileInfo, singleFile bool) {
 	if stat == nil || stat.IsDir() {
 		return
 	}
@@ -409,6 +453,7 @@ func sendFileInfoToChannel(c chan *fileInfo, basePath, path string, stat os.File
 		path:         path,
 		size:         stat.Size(),
 		lastModified: stat.ModTime(),
+		singleFile:   singleFile,
 	}
 }
 

s3sync_test.go

@@ -88,6 +88,39 @@ func TestS3sync(t *testing.T) {
 			t.Fatal("Sync should be successful", err)
 		}
 	})
+	t.Run("DownloadSingleFile", func(t *testing.T) {
+		temp, err := ioutil.TempDir("", "s3synctest")
+		defer os.RemoveAll(temp)
+
+		if err != nil {
+			t.Fatal("Failed to create temp dir")
+		}
+
+		destOnlyFilename := filepath.Join(temp, "dest_only_file")
+		const destOnlyFileSize = 10
+		if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil {
+			t.Fatal("Failed to write", err)
+		}
+
+		// Download to ./README.md
+		if err := New(getSession()).Sync("s3://example-bucket/README.md", temp+"/"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+		// Download to ./foo/README.md
+		if err := New(getSession()).Sync("s3://example-bucket/README.md", filepath.Join(temp, "foo")+"/"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+		// Download to ./test.md
+		if err := New(getSession()).Sync("s3://example-bucket/README.md", filepath.Join(temp, "test.md")); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		fileHasSize(t, destOnlyFilename, destOnlyFileSize)
+		fileHasSize(t, filepath.Join(temp, dummyFilename), dummyFileSize)
+		fileHasSize(t, filepath.Join(temp, "foo", dummyFilename), dummyFileSize)
+		fileHasSize(t, filepath.Join(temp, "test.md"), dummyFileSize)
+	})
+
 	t.Run("Upload", func(t *testing.T) {
 		temp, err := ioutil.TempDir("", "s3synctest")
 		defer os.RemoveAll(temp)
@@ -134,6 +167,49 @@ func TestS3sync(t *testing.T) {
 			t.Error("Unexpected keys", objs)
 		}
 	})
+	t.Run("UploadSingleFile", func(t *testing.T) {
+		temp, err := ioutil.TempDir("", "s3synctest")
+		defer os.RemoveAll(temp)
+
+		if err != nil {
+			t.Fatal("Failed to create temp dir")
+		}
+
+		filePath := filepath.Join(temp, dummyFilename)
+		if err := ioutil.WriteFile(filePath, make([]byte, dummyFileSize), 0644); err != nil {
+			t.Fatal("Failed to write", err)
+		}
+
+		// Copy README.md to s3://example-bucket-upload-file/README.md
+		if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		// Copy README.md to s3://example-bucket-upload-file/foo/README.md
+		if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file/foo/"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		// Copy README.md to s3://example-bucket-upload-file/foo/test.md
+		if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file/foo/test.md"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		objs := listObjectsSorted(t, "example-bucket-upload-file")
+		if n := len(objs); n != 3 {
+			t.Fatalf("Number of the files should be 3 (result: %v)", objs)
+		}
+		for _, obj := range objs {
+			if obj.size != dummyFileSize {
+				t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size)
+			}
+		}
+		if objs[0].path != "README.md" ||
+			objs[1].path != "foo/README.md" ||
+			objs[2].path != "foo/test.md" {
+			t.Error("Unexpected keys", objs)
+		}
+	})
 }
 
 func TestDelete(t *testing.T) {
@@ -172,6 +248,30 @@ func TestDelete(t *testing.T) {
 		fileHasSize(t, filepath.Join(temp, "foo", dummyFilename), dummyFileSize)
 		fileHasSize(t, filepath.Join(temp, "bar/baz", dummyFilename), dummyFileSize)
 	})
+	t.Run("DeleteLocalSingleFile", func(t *testing.T) {
+		temp, err := ioutil.TempDir("", "s3synctest")
+		defer os.RemoveAll(temp)
+
+		if err != nil {
+			t.Fatal("Failed to create temp dir")
+		}
+
+		destOnlyFilename := filepath.Join(temp, "dest_only_file")
+		const destOnlyFileSize = 10
+		if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil {
+			t.Fatal("Failed to write", err)
+		}
+
+		if err := New(getSession(), WithDelete()).Sync(
+			"s3://example-bucket/dest_only_file", destOnlyFilename,
+		); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		if _, err := os.Stat(destOnlyFilename); !os.IsNotExist(err) {
+			t.Error("Destination-only-file should be removed by sync")
+		}
+	})
 	t.Run("DeleteRemote", func(t *testing.T) {
 		temp, err := ioutil.TempDir("", "s3synctest")
 		defer os.RemoveAll(temp)
@@ -219,6 +319,31 @@ func TestDelete(t *testing.T) {
 			t.Error("Unexpected keys", objs)
 		}
 	})
+	t.Run("DeleteRemoteSingleFile", func(t *testing.T) {
+		temp, err := ioutil.TempDir("", "s3synctest")
+		defer os.RemoveAll(temp)
+
+		if err != nil {
+			t.Fatal("Failed to create temp dir")
+		}
+
+		if err := New(
+			getSession(), WithDelete(),
+		).Sync(filepath.Join(temp, "dest_only_file"), "s3://example-bucket-delete-file/dest_only_file"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		objs := listObjectsSorted(t, "example-bucket-delete-file")
+		if n := len(objs); n != 1 {
+			t.Fatalf("Number of the files should be 1 (result: %v)", objs)
+		}
+		if objs[0].size != dummyFileSize {
+			t.Errorf("Object size should be %d, actual %d", dummyFileSize, objs[0].size)
+		}
+		if objs[0].path != "README.md" {
+			t.Error("Unexpected keys", objs)
+		}
+	})
 }
 
 func TestDryRun(t *testing.T) {
@@ -342,8 +467,8 @@ func TestPartialS3sync(t *testing.T) {
 		t.Fatal("Sync should be successful")
 	}
 
-	if atomic.LoadUint32(&syncCount) != 1 {
-		t.Fatal("Only 1 file should be synced")
+	if n := atomic.LoadUint32(&syncCount); n != 1 {
+		t.Fatalf("Only 1 file should be synced, %d files synced", n)
 	}
 
 	fileHasSize(t, filepath.Join(temp, dummyFilename), expectedFileSize)
@@ -457,9 +582,10 @@ func createLoggerWithLogFunc(log func(v ...interface{})) LoggerIF {
 func fileHasSize(t *testing.T, filename string, expectedSize int) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
-		t.Fatal(filename, "is not synced")
+		t.Error(filename, "is not synced")
+		return
 	}
 	if len(data) != expectedSize {
-		t.Fatal(filename, "is not synced")
+		t.Error(filename, "is not synced")
 	}
 }