test: ensure utf-8 sources/archives filenames work properly (#3925)

This now works for files added to the source archive **after** the `git
archive` command is run. During the `git archive`, it seems that the
`tar` is still using some format that doesn't support utf-8 by default.

Tomorrow I'll look more into it.

see https://pkg.go.dev/archive/tar#Format

closes #3812

---------

Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>
Co-Authored-By: Mohammed Al Sahaf <msaa1990@gmail.com>

internal/pipe/sourcearchive/source_test.go

@@ -23,6 +23,8 @@ func TestArchive(t *testing.T) {
 			require.NoError(t, os.WriteFile("code.rb", []byte("not really code"), 0o655))
 			require.NoError(t, os.WriteFile("code.py", []byte("print 1"), 0o655))
 			require.NoError(t, os.WriteFile("README.md", []byte("# my dope fake project"), 0o655))
+			require.NoError(t, os.WriteFile("ملف.go", []byte("محتوى عربي"), 0o655))
+			require.NoError(t, os.WriteFile("🤔.patch", []byte("thinking"), 0o655))
 			require.NoError(t, os.WriteFile(".gitignore", []byte(`
 added-later.txt
 ignored.txt
@@ -40,6 +42,8 @@ subfolder/
 			require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655))
 			require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655))
 			require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
+			require.NoError(t, os.WriteFile("ملف.txt", []byte("محتوى عربي"), 0o655))
+			require.NoError(t, os.WriteFile("🤝", []byte("it works"), 0o655))
 			require.NoError(t, os.MkdirAll("subfolder", 0o755))
 			require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655))
 
@@ -68,15 +72,18 @@ subfolder/
 					format,
 					[]string{
 						"foo-1.0.0/",
-						"foo-1.0.0/.gitignore",
-						"foo-1.0.0/.gitattributes",
 						"foo-1.0.0/.VERSION",
+						"foo-1.0.0/.gitattributes",
+						"foo-1.0.0/.gitignore",
 						"foo-1.0.0/README.md",
+						"foo-1.0.0/added-later.txt",
 						"foo-1.0.0/code.py",
 						"foo-1.0.0/code.rb",
 						"foo-1.0.0/code.txt",
-						"foo-1.0.0/added-later.txt",
 						"foo-1.0.0/subfolder/file.md",
+						"foo-1.0.0/ملف.go",
+						"foo-1.0.0/ملف.txt",
+						"foo-1.0.0/🤔.patch",
 					},
 				)
 			})
@@ -102,12 +109,14 @@ subfolder/
 					format,
 					[]string{
 						"foo-1.0.0/",
-						"foo-1.0.0/.gitignore",
-						"foo-1.0.0/.gitattributes",
 						"foo-1.0.0/.VERSION",
+						"foo-1.0.0/.gitattributes",
+						"foo-1.0.0/.gitignore",
 						"foo-1.0.0/README.md",
 						"foo-1.0.0/code.py",
 						"foo-1.0.0/code.rb",
+						"foo-1.0.0/ملف.go",
+						"foo-1.0.0/🤔.patch",
 					},
 				)
 			})

internal/testlib/archive.go

@@ -136,7 +136,7 @@ func doLsTar(f io.Reader) []string {
 		if h == nil || err == io.EOF {
 			break
 		}
-		if h.Format == tar.FormatPAX {
+		if h.Name == "pax_global_header" {
 			continue
 		}
 		paths = append(paths, h.Name)

pkg/archive/archive_test.go

@@ -50,18 +50,28 @@ func TestArchive(t *testing.T) {
 
 			a, err := Copying(f1, f2, format)
 			require.NoError(t, err)
+			require.NoError(t, f1.Close())
+
 			require.NoError(t, a.Add(config.File{
 				Source:      empty.Name(),
 				Destination: "added_later.txt",
 			}))
+			require.NoError(t, a.Add(config.File{
+				Source:      empty.Name(),
+				Destination: "ملف.txt",
+			}))
 			require.NoError(t, a.Close())
-			require.NoError(t, f1.Close())
 			require.NoError(t, f2.Close())
 
-			require.Equal(t, []string{"empty.txt", "added_later.txt"}, testlib.LsArchive(t, f2.Name(), format))
+			require.ElementsMatch(
+				t,
+				[]string{"empty.txt", "added_later.txt", "ملف.txt"},
+				testlib.LsArchive(t, f2.Name(), format),
+			)
 		})
 	}
 
+	// unsupported format...
 	t.Run("7z", func(t *testing.T) {
 		_, err := New(io.Discard, "7z")
 		require.EqualError(t, err, "invalid archive format: 7z")

pkg/archive/tar/tar.go

@@ -30,13 +30,15 @@ func Copying(source io.Reader, target io.Writer) (Archive, error) {
 	w := New(target)
 	r := tar.NewReader(source)
 	for {
-		h, err := r.Next()
-		if err == io.EOF || h == nil {
+		header, err := r.Next()
+		if err == io.EOF || header == nil {
 			break
 		}
-
-		w.files[h.Name] = true
-		if err := w.tw.WriteHeader(h); err != nil {
+		if err != nil {
+			return Archive{}, err
+		}
+		w.files[header.Name] = true
+		if err := w.tw.WriteHeader(header); err != nil {
 			return w, err
 		}
 		if _, err := io.Copy(w.tw, r); err != nil {

pkg/archive/tar/tar_test.go

@@ -176,6 +176,10 @@ func TestCopying(t *testing.T) {
 		Source:      "../testdata/foo.txt",
 		Destination: "foo.txt",
 	}))
+	require.NoError(t, t1.Add(config.File{
+		Source:      "../testdata/foo.txt",
+		Destination: "ملف.txt",
+	}))
 	require.NoError(t, t1.Close())
 	require.NoError(t, f1.Close())
 
@@ -188,10 +192,14 @@ func TestCopying(t *testing.T) {
 		Source:      "../testdata/sub1/executable",
 		Destination: "executable",
 	}))
+	require.NoError(t, t2.Add(config.File{
+		Source:      "../testdata/sub1/executable",
+		Destination: "ملف.exe",
+	}))
 	require.NoError(t, t2.Close())
 	require.NoError(t, f2.Close())
 	require.NoError(t, f1.Close())
 
-	require.Equal(t, []string{"foo.txt"}, testlib.LsArchive(t, f1.Name(), "tar"))
-	require.Equal(t, []string{"foo.txt", "executable"}, testlib.LsArchive(t, f2.Name(), "tar"))
+	require.Equal(t, []string{"foo.txt", "ملف.txt"}, testlib.LsArchive(t, f1.Name(), "tar"))
+	require.Equal(t, []string{"foo.txt", "ملف.txt", "executable", "ملف.exe"}, testlib.LsArchive(t, f2.Name(), "tar"))
 }