-
Notifications
You must be signed in to change notification settings - Fork 498
/
manifest.go
107 lines (92 loc) · 2.25 KB
/
manifest.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
package weights
import (
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"hash/crc32"
"io"
"os"
"path"
)
// Manifest contains metadata about weights files in a model
type Manifest struct {
Files map[string]Metadata `json:"files"`
}
// Metadata contains information about a file
type Metadata struct {
// CRC32 is the CRC32 checksum of the file encoded as a hexadecimal string
CRC32 string `json:"crc32"`
}
// NewManifest creates a new manifest
func NewManifest() *Manifest {
return &Manifest{}
}
// LoadManifest loads a manifest from a file
func LoadManifest(filename string) (*Manifest, error) {
if _, err := os.Stat(filename); err != nil {
return nil, err
}
file, err := os.Open(filename)
if err != nil {
return nil, err
}
defer file.Close()
m := &Manifest{}
decoder := json.NewDecoder(file)
if err := decoder.Decode(m); err != nil {
return nil, err
}
return m, nil
}
// Save saves a manifest to a file
func (m *Manifest) Save(filename string) error {
if err := os.MkdirAll(path.Dir(filename), 0o755); err != nil {
return err
}
file, err := os.Create(filename)
if err != nil {
return err
}
defer file.Close()
encoder := json.NewEncoder(file)
return encoder.Encode(m)
}
// Equal compares the files in two manifests for strict equality
func (m *Manifest) Equal(other *Manifest) bool {
if len(m.Files) != len(other.Files) {
return false
}
for path, crc32 := range m.Files {
if otherCrc32, ok := other.Files[path]; !ok || otherCrc32 != crc32 {
return false
}
}
return true
}
// AddFile adds a file to the manifest, calculating its CRC32 checksum
func (m *Manifest) AddFile(path string) error {
crc32Algo := crc32.NewIEEE()
// generate checksum of file
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", path, err)
}
defer file.Close()
_, err = io.Copy(crc32Algo, file)
if err != nil {
return fmt.Errorf("failed to generate checksum of file %s: %w", path, err)
}
checksum := crc32Algo.Sum32()
// encode checksum as hexadecimal string
bytes := make([]byte, 4)
binary.LittleEndian.PutUint32(bytes, checksum)
encoded := hex.EncodeToString(bytes)
if m.Files == nil {
m.Files = make(map[string]Metadata)
}
m.Files[path] = Metadata{
CRC32: encoded,
}
return nil
}