From d92fb1ea55a1650a65cea816ee6cf3a592750a91 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Fri, 19 Apr 2024 15:27:26 +0200 Subject: [PATCH 1/2] Revert "fix: Add ModTime to fileListEntry and consider it when extracting" This reverts commit 164252ded52dd74731abb6352152604490fa4816. --- embed_util/embedded_files.go | 9 +-------- embed_util/file_list.go | 16 ++++++---------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/embed_util/embedded_files.go b/embed_util/embedded_files.go index 7c1b836f..bf4e15ad 100644 --- a/embed_util/embedded_files.go +++ b/embed_util/embedded_files.go @@ -9,7 +9,6 @@ import ( "io/fs" "os" "path/filepath" - "time" ) type EmbeddedFiles struct { @@ -129,7 +128,7 @@ func (e *EmbeddedFiles) copyEmbeddedFilesToTmp(embedFs fs.FS, fl *fileList) erro if resolvedFle.Mode.Type() == existingSt.Mode().Type() { if resolvedFle.Mode.IsDir() { continue - } else if existingSt.Size() == resolvedFle.Size && existingSt.ModTime().Unix() == resolvedFle.ModTime { + } else if existingSt.Size() == resolvedFle.Size { // unchanged continue } @@ -177,12 +176,6 @@ func (e *EmbeddedFiles) copyEmbeddedFilesToTmp(embedFs fs.FS, fl *fileList) erro if err != nil { return err } - if !resolvedFle.Mode.IsDir() { - err = os.Chtimes(path, time.Time{}, time.Unix(resolvedFle.ModTime, 0)) - if err != nil { - return err - } - } } return nil diff --git a/embed_util/file_list.go b/embed_util/file_list.go index e4c9689d..f26aaa4c 100644 --- a/embed_util/file_list.go +++ b/embed_util/file_list.go @@ -20,7 +20,6 @@ type fileList struct { type fileListEntry struct { Name string `json:"name"` Size int64 `json:"size"` - ModTime int64 `json:"modTime"` Mode fs.FileMode `json:"perm"` Symlink string `json:"symlink,omitempty"` Compressed bool `json:"compressed,omitempty"` @@ -49,10 +48,9 @@ func buildFileListFromDir(dir string) (*fileList, error) { } fle := fileListEntry{ - Name: relPath, - Size: info.Size(), - ModTime: info.ModTime().Unix(), - Mode: info.Mode(), + Name: relPath, + Size: info.Size(), + Mode: info.Mode(), } if info.Mode().Type() == fs.ModeSymlink { @@ -94,17 +92,15 @@ func buildFileListFromFs(embedFs fs.FS) (*fileList, error) { } fle := fileListEntry{ - Name: path, - Size: info.Size(), - ModTime: info.ModTime().Unix(), - Mode: info.Mode() | 0o600, + Name: path, + Size: info.Size(), + Mode: info.Mode() | 0o600, } if info.Mode().Type() == fs.ModeSymlink { return fmt.Errorf("symlink not supported in buildFileListFromFs") } else if info.Mode().IsDir() { fle.Size = 0 - fle.ModTime = 0 } fl.Files = append(fl.Files, fle) From bfa52d525ff53551cb1fa858ba7bfd06861beded Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Fri, 19 Apr 2024 15:50:20 +0200 Subject: [PATCH 2/2] feat: Allow to call BuildAndWriteFilesList on non-copied embedded dirs --- embed_util/file_list.go | 4 ++-- embed_util/packer.go | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/embed_util/file_list.go b/embed_util/file_list.go index f26aaa4c..eb2f8070 100644 --- a/embed_util/file_list.go +++ b/embed_util/file_list.go @@ -43,7 +43,7 @@ func buildFileListFromDir(dir string) (*fileList, error) { return err } - if relPath == "." { + if relPath == "." || relPath == "files.json" { return nil } @@ -82,7 +82,7 @@ func buildFileListFromFs(embedFs fs.FS) (*fileList, error) { var fl fileList err := fs.WalkDir(embedFs, ".", func(path string, d fs.DirEntry, err error) error { - if path == "." { + if path == "." || path == "files.json" { return nil } diff --git a/embed_util/packer.go b/embed_util/packer.go index f75aa9d0..ac45f870 100644 --- a/embed_util/packer.go +++ b/embed_util/packer.go @@ -27,13 +27,33 @@ func CopyForEmbed(out string, dir string) error { return err } + return doWriteFilesList(out, fl) +} + +func BuildAndWriteFilesList(dir string) error { + fl, err := buildFileListFromDir(dir) + if err != nil { + return err + } + return doWriteFilesList(dir, fl) +} + +func doWriteFilesList(dir string, fl *fileList) error { + var err error + fl.ContentHash, err = calcContentHash(dir, fl) + if err != nil { + return err + } b, err := json.MarshalIndent(fl, "", " ") if err != nil { return err } - err = os.WriteFile(filepath.Join(out, "files.json"), b, 0o644) - return err + err = os.WriteFile(filepath.Join(dir, "files.json"), b, 0o644) + if err != nil { + return err + } + return nil } func WriteEmbedGoFile(targetDir string, goOs string, goArch string) error { @@ -128,36 +148,40 @@ func copyFiles(out string, dir string, fl *fileList) error { return err } + return nil +} + +func calcContentHash(dir string, fl *fileList) (string, error) { hash := sha256.New() for _, fle := range fl.Files { path := filepath.Join(dir, fle.Name) st, err := os.Lstat(path) if err != nil { - return err + return "", err } if st.Mode().Type() == os.ModeSymlink { sl, err := os.Readlink(path) if err != nil { - return err + return "", err } _ = binary.Write(hash, binary.LittleEndian, "symlink") _ = binary.Write(hash, binary.LittleEndian, sl) } else if st.Mode().IsDir() { err = os.MkdirAll(path, fle.Mode.Perm()) if err != nil { - return err + return "", err } _ = binary.Write(hash, binary.LittleEndian, "dir") _ = binary.Write(hash, binary.LittleEndian, fle.Name) } else if st.Mode().IsRegular() { - outPath := filepath.Join(out, fle.Name) + outPath := filepath.Join(dir, fle.Name) if fle.Compressed { outPath += ".gz" } data, err := os.ReadFile(outPath) if err != nil { - return err + return "", err } _ = binary.Write(hash, binary.LittleEndian, "regular") @@ -165,6 +189,5 @@ func copyFiles(out string, dir string, fl *fileList) error { _ = binary.Write(hash, binary.LittleEndian, data) } } - fl.ContentHash = hex.EncodeToString(hash.Sum(nil)) - return nil + return hex.EncodeToString(hash.Sum(nil)), nil }