Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

siva: reload metadata #84

Merged
merged 1 commit into from
Aug 20, 2019
Merged

siva: reload metadata #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 31 additions & 45 deletions siva/library.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
Expand All @@ -31,7 +32,7 @@ type Library struct {
locReg *locationRegistry
locMu sync.Mutex
options *LibraryOptions
metadata *LibraryMetadata
metadata *libMetadata
}

// LibraryOptions hold configuration options for the library.
Expand Down Expand Up @@ -61,23 +62,20 @@ type LibraryOptions struct {
// Performance enables performance options in read only git repositories
// (ExclusiveAccess and KeepDescriptors).
Performance bool
// DontGenerateID stops the library from generating a library ID if it's
// not provided or already in metadata.
DontGenerateID bool
// MetadataReadOnly doesn't create or modify metadata for the library.
MetadataReadOnly bool
}

var _ borges.Library = (*Library)(nil)

const (
timeout = 20 * time.Second
// txTimeout is the default transaction TransactionTimeout.
timeout = 20 * time.Second
txTimeout = 60 * time.Second
registryCacheSize = 10000
)

// NewLibrary creates a new siva.Library. If id is not provided the library ID
// is read from the existing metadata or generated if not available and the
// option DontGenerateID is not set.
// NewLibrary creates a new siva.Library. When is not in MetadataReadOnly it
// will generate an id if not provided the first time the metadata is created.
func NewLibrary(
id string,
fs billy.Filesystem,
Expand All @@ -90,24 +88,23 @@ func NewLibrary(
ops = &(*options)
}

metadata, err := loadLibraryMetadata(fs)
if err != nil {
// TODO: skip metadata if corrupted?
return nil, err
}
var (
metadata *libMetadata
err error
)

if metadata == nil {
metadata = NewLibraryMetadata("", -1)
if ops.MetadataReadOnly {
metadata, err = loadLibraryMetadata(fs)
} else {
metadata, err = loadOrCreateLibraryMetadata(id, fs)
}

if id == "" {
if metadata.ID == "" && !ops.DontGenerateID {
metadata.GenerateID()
}
if err != nil && !os.IsNotExist(err) {
return nil, err
}

if metadata != nil && id == "" {
id = metadata.ID
} else {
metadata.SetID(id)
}

if ops.RegistryCache <= 0 {
Expand Down Expand Up @@ -137,21 +134,14 @@ func NewLibrary(
tmp = osfs.New(dir)
}

lib := &Library{
return &Library{
id: borges.LibraryID(id),
fs: fs,
tmp: tmp,
locReg: lr,
options: ops,
metadata: metadata,
}

err = lib.SaveMetadata()
if err != nil {
return nil, err
}

return lib, nil
}, nil
}

// ID implements borges.Library interface.
Expand Down Expand Up @@ -349,24 +339,20 @@ func (l *Library) locations(ctx context.Context) ([]borges.Location, error) {
}

// Version returns version stored in metadata or -1 if not defined.
func (l *Library) Version() int {
return l.metadata.Version()
}

// SetVersion sets the current version to the given number.
func (l *Library) SetVersion(n int) {
if l.metadata == nil {
l.metadata = NewLibraryMetadata(string(l.id), -1)
func (l *Library) Version() (int, error) {
if l.metadata != nil {
return l.metadata.version()
}

l.metadata.SetVersion(n)
return -1, nil
}

// SaveMetadata writes the metadata to the library yaml file.
func (l *Library) SaveMetadata() error {
if l.metadata != nil && l.metadata.dirty {
return l.metadata.Save(l.fs)
// SetVersion sets the current version to the given number.
func (l *Library) SetVersion(n int) error {
if l.metadata == nil {
return nil
}

return nil
l.metadata.setVersion(n)
return l.metadata.save()
}
71 changes: 44 additions & 27 deletions siva/location.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ type Location struct {
lib *Library
checkpoint *checkpoint
txer *transactioner
metadata *LocationMetadata
metadata *locationMetadata

// references and config cache
refs memory.ReferenceStorage
Expand All @@ -58,10 +58,14 @@ func newLocation(
path string,
create bool,
) (*Location, error) {
metadata, err := loadLocationMetadata(lib.fs, locationMetadataPath(path))
if err != nil {
// TODO: skip metadata if corrupted? log a warning?
return nil, err
var metadata *locationMetadata
if lib.metadata != nil {
var err error
metadata, err = loadOrCreateLocationMetadata(lib.fs, string(id))
if err != nil {
// TODO: skip metadata if corrupted? log a warning?
return nil, err
}
}

cp, err := newCheckpoint(lib.fs, path, create)
Expand Down Expand Up @@ -104,7 +108,11 @@ func (l *Location) checkAndUpdate() error {
return err
}

version := l.lib.Version()
version, err := l.lib.Version()
if err != nil {
return err
}

if l.fSize == stat.Size() && l.fTime == stat.ModTime() &&
l.version == version && l.checkpoint.Offset() == cp.Offset() {
return nil
Expand Down Expand Up @@ -175,9 +183,18 @@ func (l *Location) fs(mode borges.Mode, cp *checkpoint) (sivafs.SivaFS, error) {
return nil, borges.ErrLocationNotExists.New(string(l.id))
}

if l.metadata != nil {
version := l.lib.Version()
if o := l.metadata.Offset(version); o > 0 {
if l.lib.metadata != nil {
version, err := l.lib.Version()
if err != nil {
return nil, err
}

o, err := l.metadata.offset(version)
if err != nil {
return nil, err
}

if o > 0 {
offset = o
}
}
Expand Down Expand Up @@ -561,40 +578,40 @@ func (l *Location) repository(
return newRepository(id, sto, fs, mode, l.lib.options.Transactional, l)
}

func (l *Location) createMetadata() {
if l.metadata == nil {
l.metadata = NewLocationMetadata(make(map[int]Version))
}
}

// LastVersion returns the last defined version number in metadata or -1 if
// there are not versions.
func (l *Location) LastVersion() int {
return l.metadata.Last()
return l.metadata.last()
}

// Version returns an specific version. Second return value is false if the
// version does not exist.
func (l *Location) Version(v int) (Version, bool) {
return l.metadata.Version(v)
// Version returns an specific version. If the given version does not exist
// an error is returned.
func (l *Location) Version(v int) (*Version, error) {
if l.lib.metadata != nil {
return l.metadata.version(v)
}

return nil, errLocVersionNotExists.New()
}

// SetVersion adds or changes a version to the location.
func (l *Location) SetVersion(n int, v Version) {
l.createMetadata()
l.metadata.SetVersion(n, v)
func (l *Location) SetVersion(n int, v *Version) {
if l.lib.metadata != nil {
l.metadata.setVersion(n, v)
}
}

// DeleteVersion removes the given version number.
func (l *Location) DeleteVersion(n int) {
l.createMetadata()
l.metadata.DeleteVersion(n)
if l.lib.metadata != nil {
l.metadata.deleteVersion(n)
}
}

// SaveMetadata writes the location metadata to disk.
func (l *Location) SaveMetadata() error {
if l.metadata != nil && l.metadata.Dirty() {
return l.metadata.Save(l.lib.fs, l.path)
if l.metadata != nil {
return l.metadata.save()
}

return nil
Expand Down
Loading