From b4407c0594a638037b4d7c876279559fc8f0c324 Mon Sep 17 00:00:00 2001 From: Manuel Carmona Date: Tue, 13 Aug 2019 15:44:42 +0100 Subject: [PATCH] siva: reload metadata Signed-off-by: Manuel Carmona --- siva/library.go | 76 ++++----- siva/location.go | 71 +++++---- siva/metadata.go | 354 ++++++++++++++++++++++++++++-------------- siva/metadata_test.go | 158 ++++++++++--------- siva/repository.go | 22 ++- 5 files changed, 411 insertions(+), 270 deletions(-) diff --git a/siva/library.go b/siva/library.go index 23740f4..d6cc395 100644 --- a/siva/library.go +++ b/siva/library.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "os" "path/filepath" "strings" "sync" @@ -31,7 +32,7 @@ type Library struct { locReg *locationRegistry locMu sync.Mutex options *LibraryOptions - metadata *LibraryMetadata + metadata *libMetadata } // LibraryOptions hold configuration options for the library. @@ -61,23 +62,20 @@ type LibraryOptions struct { // Performance enables performance options in read only git repositories // (ExclusiveAccess and KeepDescriptors). Performance bool - // DontGenerateID stops the library from generating a library ID if it's - // not provided or already in metadata. - DontGenerateID bool + // MetadataReadOnly doesn't create or modify metadata for the library. + MetadataReadOnly bool } var _ borges.Library = (*Library)(nil) const ( - timeout = 20 * time.Second - // txTimeout is the default transaction TransactionTimeout. + timeout = 20 * time.Second txTimeout = 60 * time.Second registryCacheSize = 10000 ) -// NewLibrary creates a new siva.Library. If id is not provided the library ID -// is read from the existing metadata or generated if not available and the -// option DontGenerateID is not set. +// NewLibrary creates a new siva.Library. When is not in MetadataReadOnly it +// will generate an id if not provided the first time the metadata is created. func NewLibrary( id string, fs billy.Filesystem, @@ -90,24 +88,23 @@ func NewLibrary( ops = &(*options) } - metadata, err := loadLibraryMetadata(fs) - if err != nil { - // TODO: skip metadata if corrupted? - return nil, err - } + var ( + metadata *libMetadata + err error + ) - if metadata == nil { - metadata = NewLibraryMetadata("", -1) + if ops.MetadataReadOnly { + metadata, err = loadLibraryMetadata(fs) + } else { + metadata, err = loadOrCreateLibraryMetadata(id, fs) } - if id == "" { - if metadata.ID == "" && !ops.DontGenerateID { - metadata.GenerateID() - } + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if metadata != nil && id == "" { id = metadata.ID - } else { - metadata.SetID(id) } if ops.RegistryCache <= 0 { @@ -137,21 +134,14 @@ func NewLibrary( tmp = osfs.New(dir) } - lib := &Library{ + return &Library{ id: borges.LibraryID(id), fs: fs, tmp: tmp, locReg: lr, options: ops, metadata: metadata, - } - - err = lib.SaveMetadata() - if err != nil { - return nil, err - } - - return lib, nil + }, nil } // ID implements borges.Library interface. @@ -349,24 +339,20 @@ func (l *Library) locations(ctx context.Context) ([]borges.Location, error) { } // Version returns version stored in metadata or -1 if not defined. -func (l *Library) Version() int { - return l.metadata.Version() -} - -// SetVersion sets the current version to the given number. -func (l *Library) SetVersion(n int) { - if l.metadata == nil { - l.metadata = NewLibraryMetadata(string(l.id), -1) +func (l *Library) Version() (int, error) { + if l.metadata != nil { + return l.metadata.version() } - l.metadata.SetVersion(n) + return -1, nil } -// SaveMetadata writes the metadata to the library yaml file. -func (l *Library) SaveMetadata() error { - if l.metadata != nil && l.metadata.dirty { - return l.metadata.Save(l.fs) +// SetVersion sets the current version to the given number. +func (l *Library) SetVersion(n int) error { + if l.metadata == nil { + return nil } - return nil + l.metadata.setVersion(n) + return l.metadata.save() } diff --git a/siva/location.go b/siva/location.go index 64019a9..1619faf 100644 --- a/siva/location.go +++ b/siva/location.go @@ -36,7 +36,7 @@ type Location struct { lib *Library checkpoint *checkpoint txer *transactioner - metadata *LocationMetadata + metadata *locationMetadata // references and config cache refs memory.ReferenceStorage @@ -58,10 +58,14 @@ func newLocation( path string, create bool, ) (*Location, error) { - metadata, err := loadLocationMetadata(lib.fs, locationMetadataPath(path)) - if err != nil { - // TODO: skip metadata if corrupted? log a warning? - return nil, err + var metadata *locationMetadata + if lib.metadata != nil { + var err error + metadata, err = loadOrCreateLocationMetadata(lib.fs, string(id)) + if err != nil { + // TODO: skip metadata if corrupted? log a warning? + return nil, err + } } cp, err := newCheckpoint(lib.fs, path, create) @@ -104,7 +108,11 @@ func (l *Location) checkAndUpdate() error { return err } - version := l.lib.Version() + version, err := l.lib.Version() + if err != nil { + return err + } + if l.fSize == stat.Size() && l.fTime == stat.ModTime() && l.version == version && l.checkpoint.Offset() == cp.Offset() { return nil @@ -175,9 +183,18 @@ func (l *Location) fs(mode borges.Mode, cp *checkpoint) (sivafs.SivaFS, error) { return nil, borges.ErrLocationNotExists.New(string(l.id)) } - if l.metadata != nil { - version := l.lib.Version() - if o := l.metadata.Offset(version); o > 0 { + if l.lib.metadata != nil { + version, err := l.lib.Version() + if err != nil { + return nil, err + } + + o, err := l.metadata.offset(version) + if err != nil { + return nil, err + } + + if o > 0 { offset = o } } @@ -561,40 +578,40 @@ func (l *Location) repository( return newRepository(id, sto, fs, mode, l.lib.options.Transactional, l) } -func (l *Location) createMetadata() { - if l.metadata == nil { - l.metadata = NewLocationMetadata(make(map[int]Version)) - } -} - // LastVersion returns the last defined version number in metadata or -1 if // there are not versions. func (l *Location) LastVersion() int { - return l.metadata.Last() + return l.metadata.last() } -// Version returns an specific version. Second return value is false if the -// version does not exist. -func (l *Location) Version(v int) (Version, bool) { - return l.metadata.Version(v) +// Version returns an specific version. If the given version does not exist +// an error is returned. +func (l *Location) Version(v int) (*Version, error) { + if l.lib.metadata != nil { + return l.metadata.version(v) + } + + return nil, errLocVersionNotExists.New() } // SetVersion adds or changes a version to the location. -func (l *Location) SetVersion(n int, v Version) { - l.createMetadata() - l.metadata.SetVersion(n, v) +func (l *Location) SetVersion(n int, v *Version) { + if l.lib.metadata != nil { + l.metadata.setVersion(n, v) + } } // DeleteVersion removes the given version number. func (l *Location) DeleteVersion(n int) { - l.createMetadata() - l.metadata.DeleteVersion(n) + if l.lib.metadata != nil { + l.metadata.deleteVersion(n) + } } // SaveMetadata writes the location metadata to disk. func (l *Location) SaveMetadata() error { - if l.metadata != nil && l.metadata.Dirty() { - return l.metadata.Save(l.lib.fs, l.path) + if l.metadata != nil { + return l.metadata.save() } return nil diff --git a/siva/metadata.go b/siva/metadata.go index d69ec40..c3412ed 100644 --- a/siva/metadata.go +++ b/siva/metadata.go @@ -3,154 +3,233 @@ package siva import ( "io/ioutil" "os" + "time" + + billy "gopkg.in/src-d/go-billy.v4" + errors "gopkg.in/src-d/go-errors.v1" "github.com/ghodss/yaml" "github.com/google/uuid" - billy "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-billy.v4/util" ) const ( - // LibraryMetadataFile is the name of the file that holds library metadata. - LibraryMetadataFile = "library.yaml" + libraryMetadataFile = "library.yaml" ) -// LibraryMetadata holds information about the library. -type LibraryMetadata struct { +type libMetadata struct { // ID is the library indentifyer. It is a generated UUID id no ID is // provided. - ID string `json:"id,omiempty"` + ID string `json:"id,omitempty"` // CurrentVersion holds the version used for reading. CurrentVersion int `json:"version"` - dirty bool + dirty bool + fs billy.Filesystem + size int64 + modTime time.Time +} + +// loadOrCreateLibraryMetadata loads the library metadata from disk. If the data +// doesn't exist on disk it creates new data generating a new ID in case the +// passed id is an empty string.. +func loadOrCreateLibraryMetadata( + id string, + fs billy.Filesystem, +) (*libMetadata, error) { + m, err := loadLibraryMetadata(fs) + if err == nil || !os.IsNotExist(err) { + return m, err + } + + if id == "" { + var err error + id, err = generateLibID() + if err != nil { + return nil, err + } + } + + return newLibraryMetadata(id, fs) } -// NewLibraryMetadata creates a new LibraryMetadata. -func NewLibraryMetadata(id string, Version int) *LibraryMetadata { - return &LibraryMetadata{ +// newLibraryMetadata builds a new libMetadata. It persists the data on disk. +func newLibraryMetadata( + id string, + fs billy.Filesystem, +) (*libMetadata, error) { + m := &libMetadata{ ID: id, - CurrentVersion: Version, + CurrentVersion: 0, + fs: fs, + dirty: true, } + + if err := m.save(); err != nil { + return nil, err + } + + fi, err := fs.Stat(libraryMetadataFile) + if err != nil { + return nil, err + } + + m.fs = fs + m.modTime = fi.ModTime() + m.size = fi.Size() + + return m, nil } -// Version returns the version stored in the library metadata file or -1 +// version returns the version stored in the library metadata file or -1 // if it's not set. -func (m *LibraryMetadata) Version() int { - if m == nil { - return -1 +func (m *libMetadata) version() (int, error) { + if m.dirty { + return m.CurrentVersion, nil + } + + fi, err := m.fs.Stat(libraryMetadataFile) + if err != nil { + return -1, err + } + + if fi.ModTime() != m.modTime || fi.Size() != m.size { + metadata, err := loadLibraryMetadata(m.fs) + if err != nil { + return -1, err + } + + m.ID = metadata.ID + m.CurrentVersion = metadata.CurrentVersion + m.modTime = fi.ModTime() + m.size = fi.Size() + m.dirty = false } - return m.CurrentVersion + return m.CurrentVersion, nil } -// SetVersion changes the current version. -func (m *LibraryMetadata) SetVersion(v int) { +func (m *libMetadata) setVersion(v int) { if v != m.CurrentVersion { m.dirty = true m.CurrentVersion = v } } -// SetVersion changes the current version. -func (m *LibraryMetadata) SetID(id string) { - if m != nil && id != m.ID { +// setID changes the current version. +func (m *libMetadata) setID(id string) { + if id != m.ID { m.dirty = true m.ID = id } } -// GenerateID creates a new UUID and uses it as ID. -func (m *LibraryMetadata) GenerateID() error { +func generateLibID() (string, error) { uuid, err := uuid.NewUUID() if err != nil { - return err + return "", err } - m.SetID(uuid.String()) - return nil + return uuid.String(), nil } -// Save writes metadata to the library yaml file. -func (m *LibraryMetadata) Save(fs billy.Filesystem) error { +func (m *libMetadata) save() error { + if !m.dirty { + return nil + } + data, err := yaml.Marshal(m) if err != nil { return err } - tmp := LibraryMetadataFile + ".tmp" - defer fs.Remove(tmp) + tmp := libraryMetadataFile + ".tmp" + defer m.fs.Remove(tmp) - err = util.WriteFile(fs, tmp, data, 0666) - if err != nil { + if err = util.WriteFile(m.fs, tmp, data, 0666); err != nil { return err } - return fs.Rename(tmp, LibraryMetadataFile) -} + if err = m.fs.Rename(tmp, libraryMetadataFile); err != nil { + return err + } -// Dirty returns true if the metadata was changed and it needs to be written. -func (m *LibraryMetadata) Dirty() bool { - return m.dirty + m.dirty = false + return nil } -// parseLibraryMetadata parses the yaml representation of library metadata. -func parseLibraryMetadata(d []byte) (*LibraryMetadata, error) { - var m LibraryMetadata - - err := yaml.Unmarshal(d, &m) +func loadLibraryMetadata(fs billy.Filesystem) (*libMetadata, error) { + mf, err := fs.Open(libraryMetadataFile) if err != nil { return nil, err } + defer mf.Close() - return &m, nil -} + data, err := ioutil.ReadAll(mf) + if err != nil { + return nil, err + } -// loadLibraryMetadata reads and parses a library metadata file. -func loadLibraryMetadata(fs billy.Filesystem) (*LibraryMetadata, error) { - mf, err := fs.Open(LibraryMetadataFile) - if os.IsNotExist(err) { - return nil, nil + m, err := parseLibraryMetadata(data) + if err != nil { + return nil, err } + + m.fs = fs + + fi, err := fs.Stat(libraryMetadataFile) if err != nil { return nil, err } - defer mf.Close() - data, err := ioutil.ReadAll(mf) + m.modTime = fi.ModTime() + m.size = fi.Size() + + return m, nil +} + +func parseLibraryMetadata(d []byte) (*libMetadata, error) { + var metadata libMetadata + + err := yaml.Unmarshal(d, &metadata) if err != nil { return nil, err } - return parseLibraryMetadata(data) + return &metadata, nil } -// Version describes a bookmark in the siva file. +// Version represents a valid siva file point to read from. type Version struct { - // Offset is a position in the siva file. Offset uint64 `json:"offset"` - // Size is block size of the Version. - Size uint64 `json:"size,omiempty"` + Size uint64 `json:"size,omiempty"` } -// LocationMetadata holds extra data associated with a siva file. -type LocationMetadata struct { - // Versions holds a numbered list of bookmarks in the siva file. - Versions map[int]Version `json:"versions"` +type locationMetadata struct { + Versions map[int]*Version `json:"versions"` - dirty bool + dirty bool + fs billy.Filesystem + path string + size int64 + modTime time.Time } -// NewLocationMetadata creates a new LocationMetadata. -func NewLocationMetadata(versions map[int]Version) *LocationMetadata { - return &LocationMetadata{ - Versions: versions, +const locMetadataFileExt = ".yaml" + +func newLocationMetadata( + id string, + fs billy.Filesystem, +) *locationMetadata { + return &locationMetadata{ + Versions: make(map[int]*Version), + fs: fs, + path: id + locMetadataFileExt, } } -// parseLocationMetadata parses the yaml representation of location metadata. -func parseLocationMetadata(d []byte) (*LocationMetadata, error) { - var m LocationMetadata +func parseLocationMetadata(d []byte) (*locationMetadata, error) { + var m locationMetadata err := yaml.Unmarshal(d, &m) if err != nil { @@ -160,15 +239,24 @@ func parseLocationMetadata(d []byte) (*LocationMetadata, error) { return &m, nil } -// loadLocationMetadata reads and parses a location metadata file. +func loadOrCreateLocationMetadata( + fs billy.Filesystem, + id string, +) (*locationMetadata, error) { + path := id + locMetadataFileExt + m, err := loadLocationMetadata(fs, path) + if os.IsNotExist(err) { + return newLocationMetadata(id, fs), nil + } + + return m, err +} + func loadLocationMetadata( fs billy.Filesystem, path string, -) (*LocationMetadata, error) { +) (*locationMetadata, error) { mf, err := fs.Open(path) - if os.IsNotExist(err) { - return nil, nil - } if err != nil { return nil, err } @@ -179,20 +267,19 @@ func loadLocationMetadata( return nil, err } - return parseLocationMetadata(data) -} + m, err := parseLocationMetadata(data) + if err != nil { + return nil, err + } -// locationMetadataPath returns the path for a location metadata file. -func locationMetadataPath(path string) string { - return path + ".yaml" -} + m.fs = fs + m.path = path -// Last returns the last Version or -1 if there are no Versions. -func (m *LocationMetadata) Last() int { - if m == nil { - return -1 - } + return m, nil +} +// last returns the last Version or -1 if there are no Versions. +func (m *locationMetadata) last() int { last := -1 for i := range m.Versions { if i > last { @@ -204,7 +291,7 @@ func (m *LocationMetadata) Last() int { } // closest searches for the last Version lesser or equal to the one provided. -func (m *LocationMetadata) closest(v int) int { +func (m *locationMetadata) closest(v int) int { closest := -1 for i := range m.Versions { if i <= v && i > closest { @@ -215,51 +302,83 @@ func (m *LocationMetadata) closest(v int) int { return closest } -// Offset picks the closest Version from metadata and returns its offsets. +// offset picks the closest Version from metadata and returns its offsets. // If there are not Versions defined returns offset 0 that means to use // the latest siva index when used with siva filesystem. -func (m *LocationMetadata) Offset(c int) uint64 { - Version := m.Last() +func (m *locationMetadata) offset(c int) (uint64, error) { + version := m.last() + + if version < 0 { + return 0, nil + } - if Version < 0 { - return 0 + b, err := m.version(c) + if err == nil { + return b.Offset, nil } - if v, ok := m.Versions[c]; ok { - return v.Offset + if !errLocVersionNotExists.Is(err) { + return 0, err } if closest := m.closest(c); closest >= 0 { - Version = closest + version = closest } - return m.Versions[Version].Offset + return m.Versions[version].Offset, nil } -// Version returns information for a given version. Second return argument -// is false if the version does not exist. -func (m *LocationMetadata) Version(v int) (Version, bool) { - if m == nil { - return Version{}, false +var errLocVersionNotExists = errors.NewKind("location version not exists") + +// version returns information for a given version. If the version does not +// exist an error is returned +func (m *locationMetadata) version(v int) (*Version, error) { + if m.dirty { + Version, ok := m.Versions[v] + if !ok { + return nil, errLocVersionNotExists.New() + } + + return Version, nil } - d, ok := m.Versions[v] - return d, ok + fi, err := m.fs.Stat(m.path) + if err != nil { + return nil, err + } + + if fi.ModTime() != m.modTime || fi.Size() != m.size { + metadata, err := loadLocationMetadata(m.fs, m.path) + if err != nil { + return nil, err + } + + m.Versions = metadata.Versions + m.modTime = fi.ModTime() + m.size = fi.Size() + m.dirty = false + } + + Version, ok := m.Versions[v] + if !ok { + return nil, errLocVersionNotExists.New() + } + + return Version, nil } -// SetVersion changes or adds the information for a version. -func (m *LocationMetadata) SetVersion(n int, v Version) { - Version, ok := m.Versions[n] - if ok && Version == v { +func (m *locationMetadata) setVersion(n int, b *Version) { + current, ok := m.Versions[n] + if ok && (current.Offset == b.Offset && current.Size == b.Size) { return } - m.Versions[n] = v + m.Versions[n] = b m.dirty = true } -// DeleteVersion deletes a given version. -func (m *LocationMetadata) DeleteVersion(n int) { +// deleteVersion deletes a given version. +func (m *locationMetadata) deleteVersion(n int) { _, ok := m.Versions[n] if !ok { return @@ -269,26 +388,23 @@ func (m *LocationMetadata) DeleteVersion(n int) { m.dirty = true } -// Dirty returns true if metadata was changed and needs saving. -func (m *LocationMetadata) Dirty() bool { - return m.dirty -} +func (m *locationMetadata) save() error { + if !m.dirty { + return nil + } -// Save writes metadata to the yaml file for the give siva path. -func (m *LocationMetadata) Save(fs billy.Filesystem, path string) error { data, err := yaml.Marshal(m) if err != nil { return err } - path = locationMetadataPath(path) - tmp := path + ".tmp" - defer fs.Remove(tmp) + tmp := m.path + ".tmp" + defer m.fs.Remove(tmp) - err = util.WriteFile(fs, tmp, data, 0666) + err = util.WriteFile(m.fs, tmp, data, 0666) if err != nil { return err } - return fs.Rename(tmp, path) + return m.fs.Rename(tmp, m.path) } diff --git a/siva/metadata_test.go b/siva/metadata_test.go index 59256ab..6d94af1 100644 --- a/siva/metadata_test.go +++ b/siva/metadata_test.go @@ -17,25 +17,25 @@ import ( func TestMetadataSiva(t *testing.T) { require := require.New(t) - meta := LocationMetadata{ - Versions: map[int]Version{ - 0: { + meta := &locationMetadata{ + Versions: map[int]*Version{ + 0: &Version{ Offset: 16, Size: 17, }, - 1: { + 1: &Version{ Offset: 32, Size: 33, }, - 10: { + 10: &Version{ Offset: 42, Size: 43, }, - 20: { + 20: &Version{ Offset: 52, Size: 53, }, - 2: { + 2: &Version{ Offset: 62, Size: 63, }, @@ -47,7 +47,7 @@ func TestMetadataSiva(t *testing.T) { m, err := parseLocationMetadata(data) require.NoError(err) - require.EqualValues(meta, *m) + require.EqualValues(*meta, *m) } func TestMetadataLibrary(t *testing.T) { @@ -65,7 +65,7 @@ versions: offset: 17421 ` - libMetadata = `--- + metadata = `--- version: ` ) @@ -131,27 +131,43 @@ version: ` "gitserver.com/b", "gitserver.com/c", "gitserver.com/d", + "gitserver.com/e", }, }, } + fs, _ := setupFS(t, "../_testdata/rooted", true, 0) + path := "cf2e799463e1a00dbd1addd2003b0c7db31dbfe2" + locMetadataFileExt + err := util.WriteFile(fs, path, []byte(rootedVersions), 0666) + require.NoError(t, err) + for _, test := range tests { t.Run(fmt.Sprintf("version-%v", test.version), func(t *testing.T) { require := require.New(t) - fs, _ := setupFS(t, "../_testdata/rooted", true, 0) - if test.version != -1 { - libMd := libMetadata + strconv.Itoa(test.version) - err := util.WriteFile(fs, LibraryMetadataFile, []byte(libMd), 0666) + var lib *Library + if test.version == -1 { + // do not create metadata + var err error + lib, err = NewLibrary("test", fs, &LibraryOptions{ + MetadataReadOnly: true, + }) require.NoError(err) - } + } else { + libMd := metadata + strconv.Itoa(test.version) + err := util.WriteFile(fs, libraryMetadataFile, []byte(libMd), 0666) + require.NoError(err) + defer func() { + require.NoError(fs.Remove(libraryMetadataFile)) + }() - path := locationMetadataPath("cf2e799463e1a00dbd1addd2003b0c7db31dbfe2.siva") - err := util.WriteFile(fs, path, []byte(rootedVersions), 0666) - require.NoError(err) + lib, err = NewLibrary("test", fs, &LibraryOptions{}) + require.NoError(err) + } - lib, err := NewLibrary("test", fs, &LibraryOptions{}) + v, err := lib.Version() require.NoError(err) + require.Equal(test.version, v) it, err := lib.Repositories(borges.ReadOnlyMode) require.NoError(err) @@ -168,61 +184,61 @@ version: ` } } -func TestMetadataWriteLibrary(t *testing.T) { +func TestMetadataLibraryWrite(t *testing.T) { require := require.New(t) fs, _ := setupFS(t, "../_testdata/rooted", true, 0) - // library does not have metadata - lib, err := NewLibrary("test", fs, &LibraryOptions{}) + // library does not have metadata in MetadataReadOnly mode + lib, err := NewLibrary("test", fs, &LibraryOptions{ + MetadataReadOnly: true, + }) require.NoError(err) - version := lib.Version() + version, err := lib.Version() + require.NoError(err) require.Equal(-1, version) - err = lib.SaveMetadata() + // library creating metadata, since theres is no previous metadata + // a new metadata file will be created with id "test" + lib, err = NewLibrary("test", fs, &LibraryOptions{}) require.NoError(err) - _, err = fs.Stat(LibraryMetadataFile) + _, err = fs.Stat(libraryMetadataFile) require.NoError(err, "library metadata file should exist") - // set version in library metadata - lib, err = NewLibrary("", fs, &LibraryOptions{}) + version, err = lib.Version() require.NoError(err) - - version = lib.Version() - require.Equal(-1, version) + require.Equal(0, version) require.Equal(borges.LibraryID("test"), lib.ID()) - lib.SetVersion(1) - - err = lib.SaveMetadata() - require.NoError(err) + require.NoError(lib.SetVersion(1)) - _, err = fs.Stat(LibraryMetadataFile) + _, err = fs.Stat(libraryMetadataFile) require.NoError(err, "library metadata file should exist") - // modify version and id in library metadata - lib, err = NewLibrary("overwrite", fs, &LibraryOptions{}) + // modify version and id in library metadata, since there is previous + // metadata that is loaded the id will be ignored + lib, err = NewLibrary("foo", fs, &LibraryOptions{}) require.NoError(err) - version = lib.Version() + version, err = lib.Version() + require.NoError(err) require.Equal(1, version) - require.Equal(borges.LibraryID("overwrite"), lib.ID()) + require.Equal(borges.LibraryID("foo"), lib.ID()) - lib.SetVersion(10) - err = lib.SaveMetadata() - require.NoError(err) + require.NoError(lib.SetVersion(10)) // check modified version lib, err = NewLibrary("", fs, &LibraryOptions{}) require.NoError(err) - version = lib.Version() + version, err = lib.Version() + require.NoError(err) require.Equal(10, version) - require.Equal(borges.LibraryID("overwrite"), lib.ID()) + require.Equal(borges.LibraryID("test"), lib.ID()) } -func TestMetadataWriteLocation(t *testing.T) { +func TestMetadataLocationWrite(t *testing.T) { require := require.New(t) fs, _ := setupFS(t, "../_testdata/rooted", true, 0) @@ -256,7 +272,7 @@ func TestMetadataWriteLocation(t *testing.T) { last := l.LastVersion() require.Equal(-1, last) - l.SetVersion(0, Version{ + l.SetVersion(0, &Version{ Offset: 3180, }) @@ -278,7 +294,7 @@ func TestMetadataWriteLocation(t *testing.T) { require.ElementsMatch([]string{"gitserver.com/a"}, repos) - l.SetVersion(1, Version{ + l.SetVersion(1, &Version{ Offset: 6557, }) l.DeleteVersion(0) @@ -296,11 +312,12 @@ func TestMetadataWriteLocation(t *testing.T) { l, ok = loc.(*Location) require.True(ok, "location must be siva.Location") - _, ok = l.Version(0) - require.False(ok, "version 0 should not exist, it was deleted") + _, err = l.Version(0) + require.True(errLocVersionNotExists.Is(err), + "version 0 should not exist, it was deleted") - v, ok := l.Version(1) - require.True(ok, "version 1 should exist") + v, err := l.Version(1) + require.NoError(err, "version 1 should exist") require.Equal(uint64(6557), v.Offset) require.Equal(1, l.LastVersion()) @@ -326,6 +343,7 @@ func TestMetadataVersionOnCommit(t *testing.T) { fs, _ := setupFS(t, "../_testdata/rooted", true, 0) lib, err := NewLibrary("test", fs, &LibraryOptions{ + Transactional: true, }) require.NoError(err) @@ -374,13 +392,13 @@ func TestMetadataVersionOnCommit(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("version-%v", test.version), func(t *testing.T) { - lib.SetVersion(test.version) + require.NoError(lib.SetVersion(test.version)) loc, err := lib.Location("cf2e799463e1a00dbd1addd2003b0c7db31dbfe2") require.NoError(err) sivaLoc := loc.(*Location) - version, ok := sivaLoc.Version(test.version) - require.True(ok, "version must exist") + version, err := sivaLoc.Version(test.version) + require.NoError(err, "version must exist") require.Equal(test.offset, version.Offset) require.Equal(test.size, version.Size) @@ -402,36 +420,34 @@ func TestMetadataLibraryID(t *testing.T) { require := require.New(t) fs := memfs.New() + // lib with no stored metadata in MetadataReadOnly mode + // doesn't generate ids lib, err := NewLibrary("", fs, &LibraryOptions{ - DontGenerateID: true, + MetadataReadOnly: true, }) require.NoError(err) + require.Equal(string(lib.ID()), "") files, err := fs.ReadDir("") require.NoError(err) require.Len(files, 0) - err = lib.SaveMetadata() - require.NoError(err) - - files, err = fs.ReadDir("") + // lib creating metadata with the given id + lib, err = NewLibrary("test", fs, &LibraryOptions{}) require.NoError(err) - require.Len(files, 0) - libM, err := NewLibrary("", fs, nil) - require.NoError(err) + require.Equal(string(lib.ID()), "test") - id := string(libM.ID()) - require.NotEmpty(id) + // lib with empty id using stored metadata will get the id from the + // metadata + lib, err = NewLibrary("", fs, &LibraryOptions{}) - err = lib.SaveMetadata() - require.NoError(err) + require.Equal(string(lib.ID()), "test") - files, err = fs.ReadDir("") - require.NoError(err) - require.Len(files, 1) + // lib will generate an id in case of an empty + // id is given if there's no previous metadata + fs = memfs.New() + lib, err = NewLibrary("", fs, &LibraryOptions{}) - libM, err = NewLibrary("", fs, nil) - require.NoError(err) - require.Equal(id, string(libM.ID())) + require.NotEmpty(string(lib.ID())) } diff --git a/siva/repository.go b/siva/repository.go index a5a5a58..e751448 100644 --- a/siva/repository.go +++ b/siva/repository.go @@ -159,6 +159,11 @@ func (r *Repository) saveVersion() error { return nil } + metadata := r.location.metadata + if metadata == nil { + return nil + } + offset, err := r.location.size() if err != nil { return err @@ -167,16 +172,17 @@ func (r *Repository) saveVersion() error { size := offset + 1 // work with metadata directly to get the offset of previous version - metadata := r.location.metadata - if metadata != nil { - metadata.DeleteVersion(r.createVersion) - previousOffset := metadata.Offset(r.createVersion) - if previousOffset > 0 { - size = offset - previousOffset - } + metadata.deleteVersion(r.createVersion) + previousOffset, err := metadata.offset(r.createVersion) + if err != nil { + return err + } + + if previousOffset > 0 { + size = offset - previousOffset } - r.location.SetVersion(r.createVersion, Version{ + r.location.SetVersion(r.createVersion, &Version{ Offset: offset, Size: size, })