-
Notifications
You must be signed in to change notification settings - Fork 342
/
graveler.go
2791 lines (2400 loc) · 92.9 KB
/
graveler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package graveler
import (
"bytes"
"context"
"errors"
"fmt"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/google/uuid"
"github.com/hashicorp/go-multierror"
"github.com/rs/xid"
"github.com/treeverse/lakefs/pkg/ident"
"github.com/treeverse/lakefs/pkg/kv"
"github.com/treeverse/lakefs/pkg/logging"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
//go:generate mockgen -source=graveler.go -destination=mock/graveler.go -package=mock
const (
ListingDefaultBatchSize = 1000
ListingMaxBatchSize = 100000
MergeStrategySrcWins = "source-wins"
MergeStrategyDestWins = "dest-wins"
SettingsRelativeKey = "%s/settings/%s.json" // where the settings are saved relative to the storage namespace
)
// Basic Types
// DiffType represents the type of the change
type DiffType uint8
const (
DiffTypeAdded DiffType = iota
DiffTypeRemoved
DiffTypeChanged
DiffTypeConflict
)
type RefModType rune
const (
RefModTypeTilde RefModType = '~'
RefModTypeCaret RefModType = '^'
RefModTypeAt RefModType = '@'
RefModTypeDollar RefModType = '$'
)
type RefModifier struct {
Type RefModType
Value int
}
// RawRef is a parsed Ref that includes 'BaseRef' that holds the branch/tag/hash and a list of
// ordered modifiers that applied to the reference.
// Example: master~2 will be parsed into {BaseRef:"master", Modifiers:[{Type:RefModTypeTilde, Value:2}]}
type RawRef struct {
BaseRef string
Modifiers []RefModifier
}
type DiffSummary struct {
Count map[DiffType]int
Incomplete bool // true when Diff summary has missing Information (could happen when skipping ranges with same bounds)
}
// ReferenceType represents the type of the reference
type ReferenceType uint8
const (
ReferenceTypeCommit ReferenceType = iota
ReferenceTypeTag
ReferenceTypeBranch
)
// ResolvedBranchModifier indicates if the ref specified one of the committed/staging modifiers, and which.
type ResolvedBranchModifier int
const (
ResolvedBranchModifierNone ResolvedBranchModifier = iota
ResolvedBranchModifierCommitted
ResolvedBranchModifierStaging
)
// ResolvedRef include resolved information of Ref/RawRef:
// Type: Branch / Tag / Commit
// BranchID: for type ReferenceTypeBranch will hold the branch ID
// ResolvedBranchModifier: branch indicator if resolved to a branch the latest commit, staging or none was specified.
// CommitID: the commit ID of the branch head, tag or specific hash.
// StagingToken: empty if ResolvedBranchModifier is ResolvedBranchModifierCommitted.
//
type ResolvedRef struct {
Type ReferenceType
ResolvedBranchModifier ResolvedBranchModifier
BranchRecord
}
// MergeStrategy changes from dest or source are automatically overridden in case of a conflict
type MergeStrategy int
const (
MergeStrategyNone MergeStrategy = iota
MergeStrategyDest
MergeStrategySource
)
// MetaRangeAddress is the URI of a metarange file.
type MetaRangeAddress string
// RangeAddress is the URI of a range file.
type RangeAddress string
// RangeInfo contains information on a Range
type RangeInfo struct {
// ID is the identifier for the written Range.
// Calculated by a hash function to all keys and values' identity.
ID RangeID
// MinKey is the first key in the Range.
MinKey Key
// MaxKey is the last key in the Range.
MaxKey Key
// Count is the number of records in the Range.
Count int
// EstimatedRangeSizeBytes is Approximate size of each Range
EstimatedRangeSizeBytes uint64
}
// MetaRangeInfo contains information on a MetaRange
type MetaRangeInfo struct {
// ID is the identifier for the written MetaRange.
// Calculated by a hash function to all keys and values' identity.
ID MetaRangeID
}
type WriteCondition struct {
IfAbsent bool
}
type WriteConditionOption func(condition *WriteCondition)
func IfAbsent(v bool) WriteConditionOption {
return func(condition *WriteCondition) {
condition.IfAbsent = v
}
}
// function/methods receiving the following basic types could assume they passed validation
// StorageNamespace is the URI to the storage location
type StorageNamespace string
// RepositoryID is an identifier for a repo
type RepositoryID string
// Key represents a logical path for a value
type Key []byte
// Ref could be a commit ID, a branch name, a Tag
type Ref string
// TagID represents a named tag pointing at a commit
type TagID string
type CommitParents []CommitID
// BranchID is an identifier for a branch
type BranchID string
// CommitID is a content addressable hash representing a Commit object
type CommitID string
// MetaRangeID represents a snapshot of the MetaRange, referenced by a commit
type MetaRangeID string
// RangeID represents a part of a MetaRange, useful only for plumbing.
type RangeID string
// StagingToken represents a namespace for writes to apply as uncommitted
type StagingToken string
// Metadata key/value strings to hold metadata information on value and commit
type Metadata map[string]string
// Repository represents repository metadata
type Repository struct {
StorageNamespace StorageNamespace `db:"storage_namespace"`
CreationDate time.Time `db:"creation_date"`
DefaultBranchID BranchID `db:"default_branch"`
// RepositoryState represents the state of the repository, only ACTIVE repository is considered a valid one.
// other states represent in invalid temporary or terminal state
State RepositoryState
// InstanceUID identifies repository in a unique way. Since repositories with same name can be deleted and recreated
// this field identifies the specific instance, and used in the KV store key path to store all the entities belonging
// to this specific instantiation of repo with the given ID
InstanceUID string
}
func NewRepository(storageNamespace StorageNamespace, defaultBranchID BranchID) Repository {
return Repository{
StorageNamespace: storageNamespace,
CreationDate: time.Now().UTC(),
DefaultBranchID: defaultBranchID,
State: RepositoryState_ACTIVE,
InstanceUID: NewRepoInstanceID(),
}
}
type RepositoryRecord struct {
RepositoryID RepositoryID `db:"id"`
*Repository
}
// Value represents metadata or a given object (modified date, physical address, etc)
type Value struct {
Identity []byte `db:"identity"`
Data []byte `db:"data"`
}
// ValueRecord holds Key with the associated Value information
type ValueRecord struct {
Key Key `db:"key"`
*Value
}
func (v *ValueRecord) IsTombstone() bool {
return v.Value == nil
}
func (cp CommitParents) Identity() []byte {
commits := make([]string, len(cp))
for i, v := range cp {
commits[i] = string(v)
}
buf := ident.NewAddressWriter()
buf.MarshalStringSlice(commits)
return buf.Identity()
}
func (cp CommitParents) Contains(commitID CommitID) bool {
for _, c := range cp {
if c == commitID {
return true
}
}
return false
}
func (cp CommitParents) AsStringSlice() []string {
stringSlice := make([]string, len(cp))
for i, p := range cp {
stringSlice[i] = string(p)
}
return stringSlice
}
// FirstCommitMsg is the message of the first (zero) commit of a lakeFS repository
const FirstCommitMsg = "Repository created"
// CommitVersion used to track changes in Commit schema. Each version is change that a constant describes.
type CommitVersion int
const (
CommitVersionInitial CommitVersion = iota
CommitVersionParentSwitch
CurrentCommitVersion = CommitVersionParentSwitch
)
// Change it if a change was applied to the schema of the Commit struct.
// Commit represents commit metadata (author, time, MetaRangeID)
type Commit struct {
Version CommitVersion `db:"version"`
Committer string `db:"committer"`
Message string `db:"message"`
MetaRangeID MetaRangeID `db:"meta_range_id"`
CreationDate time.Time `db:"creation_date"`
Parents CommitParents `db:"parents"`
Metadata Metadata `db:"metadata"`
Generation int `db:"generation"`
}
func NewCommit() Commit {
return Commit{
Version: CurrentCommitVersion,
CreationDate: time.Now(),
}
}
func (c Commit) Identity() []byte {
b := ident.NewAddressWriter()
b.MarshalString("commit:v1")
b.MarshalString(c.Committer)
b.MarshalString(c.Message)
b.MarshalString(string(c.MetaRangeID))
b.MarshalInt64(c.CreationDate.Unix())
b.MarshalStringMap(c.Metadata)
b.MarshalIdentifiable(c.Parents)
return b.Identity()
}
// CommitRecord holds CommitID with the associated Commit data
type CommitRecord struct {
CommitID CommitID `db:"id"`
*Commit
}
// Branch is a pointer to a commit
type Branch struct {
CommitID CommitID
StagingToken StagingToken
// SealedTokens - Staging tokens are appended to the front, this allows building the diff iterator easily
SealedTokens []StagingToken
}
// BranchRecord holds BranchID with the associated Branch data
type BranchRecord struct {
BranchID BranchID `db:"id"`
*Branch
}
// BranchUpdateFunc Used to pass validation call back to ref manager for UpdateBranch flow
type BranchUpdateFunc func(*Branch) (*Branch, error)
// ValueUpdateFunc Used to pass validation call back to staging manager for UpdateValue flow
type ValueUpdateFunc func(*Value) (*Value, error)
// TagRecord holds TagID with the associated Tag data
type TagRecord struct {
TagID TagID `db:"id"`
CommitID CommitID
}
// Diff represents a change in value based on key
type Diff struct {
Type DiffType
Key Key
Value *Value
LeftIdentity []byte // the Identity of the value on the left side of the diff
}
func (d *Diff) Copy() *Diff {
return &Diff{
Type: d.Type,
Key: d.Key.Copy(),
Value: d.Value,
LeftIdentity: append([]byte(nil), d.LeftIdentity...),
}
}
type CommitParams struct {
Committer string
Message string
// Date (Unix Epoch in seconds) is used to override commits creation date
Date *int64
Metadata Metadata
// SourceMetaRange - If exists, use it directly. Fail if branch has uncommitted changes
SourceMetaRange *MetaRangeID
}
type KeyValueStore interface {
// Get returns value from repository / reference by key, nil value is a valid value for tombstone
// returns error if value does not exist
Get(ctx context.Context, repository *RepositoryRecord, ref Ref, key Key) (*Value, error)
// GetByCommitID returns value from repository / commit by key and error if value does not exist
GetByCommitID(ctx context.Context, repository *RepositoryRecord, commitID CommitID, key Key) (*Value, error)
// Set stores value on repository / branch by key. nil value is a valid value for tombstone
Set(ctx context.Context, repository *RepositoryRecord, branchID BranchID, key Key, value Value, writeConditions ...WriteConditionOption) error
// Delete value from repository / branch by key
Delete(ctx context.Context, repository *RepositoryRecord, branchID BranchID, key Key) error
// List lists values on repository / ref
List(ctx context.Context, repository *RepositoryRecord, ref Ref) (ValueIterator, error)
}
type VersionController interface {
// GetRepository returns the Repository metadata object for the given RepositoryID
GetRepository(ctx context.Context, repositoryID RepositoryID) (*RepositoryRecord, error)
// CreateRepository stores a new Repository under RepositoryID with the given Branch as default branch
CreateRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, branchID BranchID) (*RepositoryRecord, error)
// CreateBareRepository stores a new Repository under RepositoryID with no initial branch or commit
CreateBareRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, defaultBranchID BranchID) (*RepositoryRecord, error)
// ListRepositories returns iterator to scan repositories
ListRepositories(ctx context.Context) (RepositoryIterator, error)
// DeleteRepository deletes the repository
DeleteRepository(ctx context.Context, repositoryID RepositoryID) error
// CreateBranch creates branch on repository pointing to ref
CreateBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, ref Ref) (*Branch, error)
// UpdateBranch updates branch on repository pointing to ref
UpdateBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, ref Ref) (*Branch, error)
// GetBranch gets branch information by branch / repository id
GetBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID) (*Branch, error)
// GetTag gets tag's commit id
GetTag(ctx context.Context, repository *RepositoryRecord, tagID TagID) (*CommitID, error)
// CreateTag creates tag on a repository pointing to a commit id
CreateTag(ctx context.Context, repository *RepositoryRecord, tagID TagID, commitID CommitID) error
// DeleteTag remove tag from a repository
DeleteTag(ctx context.Context, repository *RepositoryRecord, tagID TagID) error
// ListTags lists tags on a repository
ListTags(ctx context.Context, repository *RepositoryRecord) (TagIterator, error)
// Log returns an iterator starting at commit ID up to repository root
Log(ctx context.Context, repository *RepositoryRecord, commitID CommitID) (CommitIterator, error)
// ListBranches lists branches on repositories
ListBranches(ctx context.Context, repository *RepositoryRecord) (BranchIterator, error)
// DeleteBranch deletes branch from repository
DeleteBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID) error
// Commit the staged data and returns a commit ID that references that change
// ErrNothingToCommit in case there is no data in stage
Commit(ctx context.Context, repository *RepositoryRecord, branchID BranchID, commitParams CommitParams) (CommitID, error)
// WriteMetaRangeByIterator accepts a ValueIterator and writes the entire iterator to a new MetaRange
// and returns the result ID.
WriteMetaRangeByIterator(ctx context.Context, repository *RepositoryRecord, it ValueIterator) (*MetaRangeID, error)
// AddCommitToBranchHead creates a commit in the branch from the given pre-existing tree.
// Returns ErrMetaRangeNotFound if the referenced metaRangeID doesn't exist.
// Returns ErrCommitNotHeadBranch if the branch is no longer referencing to the parentCommit
AddCommitToBranchHead(ctx context.Context, repository *RepositoryRecord, branchID BranchID, commit Commit) (CommitID, error)
// AddCommit creates a dangling (no referencing branch) commit in the repo from the pre-existing commit.
// Returns ErrMetaRangeNotFound if the referenced metaRangeID doesn't exist.
AddCommit(ctx context.Context, repository *RepositoryRecord, commit Commit) (CommitID, error)
// GetCommit returns the Commit metadata object for the given CommitID
GetCommit(ctx context.Context, repository *RepositoryRecord, commitID CommitID) (*Commit, error)
// Dereference returns the resolved ref information based on 'ref' reference
Dereference(ctx context.Context, repository *RepositoryRecord, ref Ref) (*ResolvedRef, error)
// ParseRef returns parsed 'ref' information as raw reference
ParseRef(ref Ref) (RawRef, error)
// ResolveRawRef returns the ResolvedRef matching the given RawRef
ResolveRawRef(ctx context.Context, repository *RepositoryRecord, rawRef RawRef) (*ResolvedRef, error)
// Reset throws all staged data on the repository / branch
Reset(ctx context.Context, repository *RepositoryRecord, branchID BranchID) error
// ResetKey throws all staged data under the specified key on the repository / branch
ResetKey(ctx context.Context, repository *RepositoryRecord, branchID BranchID, key Key) error
// ResetPrefix throws all staged data starting with the given prefix on the repository / branch
ResetPrefix(ctx context.Context, repository *RepositoryRecord, branchID BranchID, key Key) error
// Revert creates a reverse patch to the commit given as 'ref', and applies it as a new commit on the given branch.
Revert(ctx context.Context, repository *RepositoryRecord, branchID BranchID, ref Ref, parentNumber int, commitParams CommitParams) (CommitID, error)
// Merge merges 'source' into 'destination' and returns the commit id for the created merge commit.
Merge(ctx context.Context, repository *RepositoryRecord, destination BranchID, source Ref, commitParams CommitParams, strategy string) (CommitID, error)
// DiffUncommitted returns iterator to scan the changes made on the branch
DiffUncommitted(ctx context.Context, repository *RepositoryRecord, branchID BranchID) (DiffIterator, error)
// Diff returns the changes between 'left' and 'right' ref.
// This is similar to a two-dot (left..right) diff in git.
Diff(ctx context.Context, repository *RepositoryRecord, left, right Ref) (DiffIterator, error)
// Compare returns the difference between the commit where 'left' was last synced into 'right', and the most recent commit of `right`.
// This is similar to a three-dot (from...to) diff in git.
Compare(ctx context.Context, repository *RepositoryRecord, left, right Ref) (DiffIterator, error)
// SetHooksHandler set handler for all graveler hooks
SetHooksHandler(handler HooksHandler)
// GetStagingToken returns the token identifying current staging for branchID of
// repositoryID.
GetStagingToken(ctx context.Context, repository *RepositoryRecord, branchID BranchID) (*StagingToken, error)
GetGarbageCollectionRules(ctx context.Context, repository *RepositoryRecord) (*GarbageCollectionRules, error)
SetGarbageCollectionRules(ctx context.Context, repository *RepositoryRecord, rules *GarbageCollectionRules) error
// SaveGarbageCollectionCommits saves the sets of active and expired commits, according to the branch rules for garbage collection.
// Returns
// - run id which can later be used to retrieve the set of commits.
// - location where the expired/active commit information was saved
// - location where the information of addresses to be removed should be saved
// If a previousRunID is specified, commits that were already expired and their ancestors will not be considered as expired/active.
// Note: Ancestors of previously expired commits may still be considered if they can be reached from a non-expired commit.
SaveGarbageCollectionCommits(ctx context.Context, repository *RepositoryRecord, previousRunID string) (garbageCollectionRunMetadata *GarbageCollectionRunMetadata, err error)
// GetBranchProtectionRules return all branch protection rules for the repository
GetBranchProtectionRules(ctx context.Context, repository *RepositoryRecord) (*BranchProtectionRules, error)
// DeleteBranchProtectionRule deletes the branch protection rule for the given pattern,
// or return ErrRuleNotExists if no such rule exists.
DeleteBranchProtectionRule(ctx context.Context, repository *RepositoryRecord, pattern string) error
// CreateBranchProtectionRule creates a rule for the given name pattern,
// or returns ErrRuleAlreadyExists if there is already a rule for the pattern.
CreateBranchProtectionRule(ctx context.Context, repository *RepositoryRecord, pattern string, blockedActions []BranchProtectionBlockedAction) error
}
// Plumbing includes commands for fiddling more directly with graveler implementation
// internals.
type Plumbing interface {
// GetMetaRange returns information where metarangeID is stored.
GetMetaRange(ctx context.Context, repository *RepositoryRecord, metaRangeID MetaRangeID) (MetaRangeAddress, error)
// GetRange returns information where rangeID is stored.
GetRange(ctx context.Context, repository *RepositoryRecord, rangeID RangeID) (RangeAddress, error)
// WriteRange creates a new Range from the iterator values.
// Keeps Range closing logic, so might not flush all values to the range.
WriteRange(ctx context.Context, repository *RepositoryRecord, it ValueIterator) (*RangeInfo, error)
// WriteMetaRange creates a new MetaRange from the given Ranges.
WriteMetaRange(ctx context.Context, repository *RepositoryRecord, ranges []*RangeInfo) (*MetaRangeInfo, error)
}
type Dumper interface {
// DumpCommits iterates through all commits and dumps them in Graveler format
DumpCommits(ctx context.Context, repository *RepositoryRecord) (*MetaRangeID, error)
// DumpBranches iterates through all branches and dumps them in Graveler format
DumpBranches(ctx context.Context, repository *RepositoryRecord) (*MetaRangeID, error)
// DumpTags iterates through all tags and dumps them in Graveler format
DumpTags(ctx context.Context, repository *RepositoryRecord) (*MetaRangeID, error)
}
type Loader interface {
// LoadCommits iterates through all commits in Graveler format and loads them into repositoryID
LoadCommits(ctx context.Context, repository *RepositoryRecord, metaRangeID MetaRangeID) error
// LoadBranches iterates through all branches in Graveler format and loads them into repositoryID
LoadBranches(ctx context.Context, repository *RepositoryRecord, metaRangeID MetaRangeID) error
// LoadTags iterates through all tags in Graveler format and loads them into repositoryID
LoadTags(ctx context.Context, repository *RepositoryRecord, metaRangeID MetaRangeID) error
}
// Internal structures used by Graveler
// xxxIterator used as follows:
// ```
// it := NewXXXIterator(data)
// for it.Next() {
// data := it.Value()
// process(data)
// }
// if it.Err() {
// return fmt.Errorf("stopped because of an error %w", it.Err())
// }
// ```
// 'Value()' should only be called after `Next()` returns true.
// In case `Next()` returns false, `Value()` returns nil and `Err()` should be checked.
// nil error means we reached the end of the input.
// `SeekGE()` behaviour is like as starting a new iterator - `Value()` returns nothing until the first `Next()`.
type RepositoryIterator interface {
Next() bool
SeekGE(id RepositoryID)
Value() *RepositoryRecord
Err() error
Close()
}
type ValueIterator interface {
Next() bool
SeekGE(id Key)
Value() *ValueRecord
Err() error
Close()
}
type DiffIterator interface {
Next() bool
SeekGE(id Key)
Value() *Diff
Err() error
Close()
}
type BranchIterator interface {
Next() bool
SeekGE(id BranchID)
Value() *BranchRecord
Err() error
Close()
}
type TagIterator interface {
Next() bool
SeekGE(id TagID)
Value() *TagRecord
Err() error
Close()
}
type CommitIterator interface {
Next() bool
SeekGE(id CommitID)
Value() *CommitRecord
Err() error
Close()
}
// These are the more complex internal components that compose the functionality of the Graveler
// RefManager handles references: branches, commits, probably tags in the future
// it also handles the structure of the commit graph and its traversal (notably, merge-base and log)
type RefManager interface {
// GetRepository returns the Repository metadata object for the given RepositoryID
GetRepository(ctx context.Context, repositoryID RepositoryID) (*RepositoryRecord, error)
// CreateRepository stores a new Repository under RepositoryID with the given Branch as default branch
CreateRepository(ctx context.Context, repositoryID RepositoryID, repository Repository) (*RepositoryRecord, error)
// CreateBareRepository stores a new repository under RepositoryID without creating an initial commit and branch
CreateBareRepository(ctx context.Context, repositoryID RepositoryID, repository Repository) (*RepositoryRecord, error)
// ListRepositories lists repositories
ListRepositories(ctx context.Context) (RepositoryIterator, error)
// DeleteRepository deletes the repository
DeleteRepository(ctx context.Context, repositoryID RepositoryID) error
// ParseRef returns parsed 'ref' information as RawRef
ParseRef(ref Ref) (RawRef, error)
// ResolveRawRef returns the ResolvedRef matching the given RawRef
ResolveRawRef(ctx context.Context, repository *RepositoryRecord, rawRef RawRef) (*ResolvedRef, error)
// GetBranch returns the Branch metadata object for the given BranchID
GetBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID) (*Branch, error)
// CreateBranch creates a branch with the given id and Branch metadata
CreateBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, branch Branch) error
// SetBranch points the given BranchID at the given Branch metadata
SetBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, branch Branch) error
// BranchUpdate Conditional set of branch with validation callback
BranchUpdate(ctx context.Context, repository *RepositoryRecord, branchID BranchID, f BranchUpdateFunc) error
// DeleteBranch deletes the branch
DeleteBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID) error
// ListBranches lists branches
ListBranches(ctx context.Context, repository *RepositoryRecord) (BranchIterator, error)
// GCBranchIterator TODO (niro): Remove when DB implementation is deleted
// GCBranchIterator temporary WA to support both DB and KV GC BranchIterator, which iterates over branches by order of commit ID
GCBranchIterator(ctx context.Context, repository *RepositoryRecord) (BranchIterator, error)
// GetTag returns the Tag metadata object for the given TagID
GetTag(ctx context.Context, repository *RepositoryRecord, tagID TagID) (*CommitID, error)
// CreateTag create a given tag pointing to a commit
CreateTag(ctx context.Context, repository *RepositoryRecord, tagID TagID, commitID CommitID) error
// DeleteTag deletes the tag
DeleteTag(ctx context.Context, repository *RepositoryRecord, tagID TagID) error
// ListTags lists tags
ListTags(ctx context.Context, repository *RepositoryRecord) (TagIterator, error)
// GetCommit returns the Commit metadata object for the given CommitID.
GetCommit(ctx context.Context, repository *RepositoryRecord, commitID CommitID) (*Commit, error)
// GetCommitByPrefix returns the Commit metadata object for the given prefix CommitID.
// if more than 1 commit starts with the ID prefix returns error
GetCommitByPrefix(ctx context.Context, repository *RepositoryRecord, prefix CommitID) (*Commit, error)
// AddCommit stores the Commit object, returning its ID
AddCommit(ctx context.Context, repository *RepositoryRecord, commit Commit) (CommitID, error)
// RemoveCommit deletes commit from store - used for repository cleanup
RemoveCommit(ctx context.Context, repository *RepositoryRecord, commitID CommitID) error
// FindMergeBase returns the merge-base for the given CommitIDs
// see: https://git-scm.com/docs/git-merge-base
// and internally: https://github.com/treeverse/lakeFS/blob/09954804baeb36ada74fa17d8fdc13a38552394e/index/dag/commits.go
FindMergeBase(ctx context.Context, repository *RepositoryRecord, commitIDs ...CommitID) (*Commit, error)
// Log returns an iterator starting at commit ID up to repository root
Log(ctx context.Context, repository *RepositoryRecord, commitID CommitID) (CommitIterator, error)
// ListCommits returns an iterator over all known commits, ordered by their commit ID
ListCommits(ctx context.Context, repository *RepositoryRecord) (CommitIterator, error)
// GCCommitIterator TODO (niro): Remove when DB implementation is deleted
// GCCommitIterator temporary WA to support both DB and KV GC CommitIterator
GCCommitIterator(ctx context.Context, repository *RepositoryRecord) (CommitIterator, error)
}
// CommittedManager reads and applies committed snapshots
// it is responsible for de-duping them, persisting them and providing basic diff, merge and list capabilities
type CommittedManager interface {
// Get returns the provided key, if exists, from the provided MetaRangeID
Get(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, key Key) (*Value, error)
// Exists returns true if a MetaRange matching ID exists in namespace ns.
Exists(ctx context.Context, ns StorageNamespace, id MetaRangeID) (bool, error)
// WriteMetaRangeByIterator flushes the iterator to a new MetaRange and returns the created ID.
WriteMetaRangeByIterator(ctx context.Context, ns StorageNamespace, it ValueIterator, metadata Metadata) (*MetaRangeID, error)
// WriteRange creates a new Range from the iterator values.
// Keeps Range closing logic, so might not exhaust the iterator.
WriteRange(ctx context.Context, ns StorageNamespace, it ValueIterator) (*RangeInfo, error)
// WriteMetaRange creates a new MetaRange from the given Ranges.
WriteMetaRange(ctx context.Context, ns StorageNamespace, ranges []*RangeInfo) (*MetaRangeInfo, error)
// List takes a given tree and returns an ValueIterator
List(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID) (ValueIterator, error)
// Diff receives two metaRanges and returns a DiffIterator describing all differences between them.
// This is similar to a two-dot diff in git (left..right)
Diff(ctx context.Context, ns StorageNamespace, left, right MetaRangeID) (DiffIterator, error)
// Compare returns the difference between 'source' and 'destination', relative to a merge base 'base'.
// This is similar to a three-dot diff in git.
Compare(ctx context.Context, ns StorageNamespace, destination, source, base MetaRangeID) (DiffIterator, error)
// Merge applies changes from 'source' to 'destination', relative to a merge base 'base' and
// returns the ID of the new metarange. This is similar to a git merge operation.
// The resulting tree is expected to be immediately addressable.
Merge(ctx context.Context, ns StorageNamespace, destination, source, base MetaRangeID, strategy MergeStrategy) (MetaRangeID, error)
// Commit is the act of taking an existing metaRange (snapshot) and applying a set of changes to it.
// A change is either an entity to write/overwrite, or a tombstone to mark a deletion
// it returns a new MetaRangeID that is expected to be immediately addressable
Commit(ctx context.Context, ns StorageNamespace, baseMetaRangeID MetaRangeID, changes ValueIterator) (MetaRangeID, DiffSummary, error)
// GetMetaRange returns information where metarangeID is stored.
GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeAddress, error)
// GetRange returns information where rangeID is stored.
GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeAddress, error)
}
// StagingManager manages entries in a staging area, denoted by a staging token
type StagingManager interface {
// Get returns the value for the provided staging token and key
// Returns ErrNotFound if no value found on key.
Get(ctx context.Context, st StagingToken, key Key) (*Value, error)
// Set writes a (possibly nil) value under the given staging token and key.
Set(ctx context.Context, st StagingToken, key Key, value *Value, overwrite bool) error
// Update updates a (possibly nil) value under the given staging token and key.
Update(ctx context.Context, st StagingToken, key Key, updateFunc ValueUpdateFunc) error
// List returns a ValueIterator for the given staging token
List(ctx context.Context, st StagingToken, batchSize int) (ValueIterator, error)
// DropKey clears a value by staging token and key
DropKey(ctx context.Context, st StagingToken, key Key) error
// Drop clears the given staging area
Drop(ctx context.Context, st StagingToken) error
// DropAsync clears the given staging area eventually. Keys may still exist under the token
// for a short period of time after the request was made.
DropAsync(ctx context.Context, st StagingToken) error
// DropByPrefix drops all keys starting with the given prefix, from the given staging area
DropByPrefix(ctx context.Context, st StagingToken, prefix Key) error
}
// BranchLockerFunc callback function when branch is locked for operation (ex: writer or metadata updater)
type BranchLockerFunc func() (interface{}, error)
type BranchLocker interface {
Writer(ctx context.Context, repository *RepositoryRecord, branchID BranchID, lockedFn BranchLockerFunc) (interface{}, error)
MetadataUpdater(ctx context.Context, repository *RepositoryRecord, branchID BranchID, lockeFn BranchLockerFunc) (interface{}, error)
}
func (id RepositoryID) String() string {
return string(id)
}
func (ns StorageNamespace) String() string {
return string(ns)
}
func (id BranchID) String() string {
return string(id)
}
func (id BranchID) Ref() Ref {
return Ref(id)
}
func (id Ref) String() string {
return string(id)
}
func (id Key) Copy() Key {
keyCopy := make(Key, len(id))
copy(keyCopy, id)
return keyCopy
}
func (id Key) String() string {
return string(id)
}
func (id CommitID) String() string {
return string(id)
}
func (id CommitID) Ref() Ref {
return Ref(id)
}
func (id TagID) String() string {
return string(id)
}
func (id StagingToken) String() string {
return string(id)
}
func (id MetaRangeID) String() string {
return string(id)
}
type KVGraveler struct {
hooks HooksHandler
CommittedManager CommittedManager
RefManager RefManager
StagingManager StagingManager
protectedBranchesManager ProtectedBranchesManager
garbageCollectionManager GarbageCollectionManager
log logging.Logger
}
func NewKVGraveler(committedManager CommittedManager, stagingManager StagingManager, refManager RefManager, gcManager GarbageCollectionManager, protectedBranchesManager ProtectedBranchesManager) *KVGraveler {
return &KVGraveler{
hooks: &HooksNoOp{},
CommittedManager: committedManager,
RefManager: refManager,
StagingManager: stagingManager,
protectedBranchesManager: protectedBranchesManager,
garbageCollectionManager: gcManager,
log: logging.Default().WithField("service_name", "graveler_graveler"),
}
}
func (g *KVGraveler) GetRepository(ctx context.Context, repositoryID RepositoryID) (*RepositoryRecord, error) {
return g.RefManager.GetRepository(ctx, repositoryID)
}
func (g *KVGraveler) CreateRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, branchID BranchID) (*RepositoryRecord, error) {
_, err := g.RefManager.GetRepository(ctx, repositoryID)
if err != nil && !errors.Is(err, ErrRepositoryNotFound) {
return nil, err
}
repo := NewRepository(storageNamespace, branchID)
repository, err := g.RefManager.CreateRepository(ctx, repositoryID, repo)
if err != nil {
return nil, err
}
return repository, nil
}
func (g *KVGraveler) CreateBareRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, defaultBranchID BranchID) (*RepositoryRecord, error) {
_, err := g.RefManager.GetRepository(ctx, repositoryID)
if err != nil && !errors.Is(err, ErrRepositoryNotFound) {
return nil, err
}
repo := NewRepository(storageNamespace, defaultBranchID)
repository, err := g.RefManager.CreateBareRepository(ctx, repositoryID, repo)
if err != nil {
return nil, err
}
return repository, nil
}
func (g *KVGraveler) ListRepositories(ctx context.Context) (RepositoryIterator, error) {
return g.RefManager.ListRepositories(ctx)
}
func (g *KVGraveler) WriteRange(ctx context.Context, repository *RepositoryRecord, it ValueIterator) (*RangeInfo, error) {
return g.CommittedManager.WriteRange(ctx, repository.StorageNamespace, it)
}
func (g *KVGraveler) WriteMetaRange(ctx context.Context, repository *RepositoryRecord, ranges []*RangeInfo) (*MetaRangeInfo, error) {
return g.CommittedManager.WriteMetaRange(ctx, repository.StorageNamespace, ranges)
}
func (g *KVGraveler) WriteMetaRangeByIterator(ctx context.Context, repository *RepositoryRecord, it ValueIterator) (*MetaRangeID, error) {
return g.CommittedManager.WriteMetaRangeByIterator(ctx, repository.StorageNamespace, it, nil)
}
func (g *KVGraveler) DeleteRepository(ctx context.Context, repositoryID RepositoryID) error {
return g.RefManager.DeleteRepository(ctx, repositoryID)
}
func (g *KVGraveler) GetCommit(ctx context.Context, repository *RepositoryRecord, commitID CommitID) (*Commit, error) {
return g.RefManager.GetCommit(ctx, repository, commitID)
}
func GenerateStagingToken(repositoryID RepositoryID, branchID BranchID) StagingToken {
uid := uuid.New().String()
return StagingToken(fmt.Sprintf("%s-%s:%s", repositoryID, branchID, uid))
}
func (g *KVGraveler) CreateBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, ref Ref) (*Branch, error) {
reference, err := g.Dereference(ctx, repository, ref)
if err != nil {
return nil, fmt.Errorf("source reference '%s': %w", ref, err)
}
if reference.ResolvedBranchModifier == ResolvedBranchModifierStaging {
return nil, fmt.Errorf("source reference '%s': %w", ref, ErrCreateBranchNoCommit)
}
_, err = g.RefManager.GetBranch(ctx, repository, branchID)
if err == nil {
return nil, ErrBranchExists
}
if !errors.Is(err, ErrBranchNotFound) {
return nil, err
}
newBranch := Branch{
CommitID: reference.CommitID,
StagingToken: GenerateStagingToken(repository.RepositoryID, branchID),
SealedTokens: make([]StagingToken, 0),
}
storageNamespace := repository.StorageNamespace
preRunID := g.hooks.NewRunID()
err = g.hooks.PreCreateBranchHook(ctx, HookRecord{
RunID: preRunID,
StorageNamespace: storageNamespace,
EventType: EventTypePreCreateBranch,
SourceRef: ref,
RepositoryID: repository.RepositoryID,
BranchID: branchID,
CommitID: reference.CommitID,
})
if err != nil {
return nil, &HookAbortError{
EventType: EventTypePreCreateBranch,
RunID: preRunID,
Err: err,
}
}
err = g.RefManager.CreateBranch(ctx, repository, branchID, newBranch)
if err != nil {
return nil, fmt.Errorf("set branch '%s' to '%v': %w", branchID, newBranch, err)
}
postRunID := g.hooks.NewRunID()
g.hooks.PostCreateBranchHook(ctx, HookRecord{
RunID: postRunID,
StorageNamespace: storageNamespace,
EventType: EventTypePostCreateBranch,
SourceRef: ref,
RepositoryID: repository.RepositoryID,
BranchID: branchID,
CommitID: reference.CommitID,
PreRunID: preRunID,
})
return &newBranch, nil
}
func (g *KVGraveler) UpdateBranch(ctx context.Context, repository *RepositoryRecord, branchID BranchID, ref Ref) (*Branch, error) {
reference, err := g.Dereference(ctx, repository, ref)
if err != nil {
return nil, err
}
if reference.ResolvedBranchModifier == ResolvedBranchModifierStaging {
return nil, fmt.Errorf("reference '%s': %w", ref, ErrDereferenceCommitWithStaging)
}
if err := g.prepareForCommitIDUpdate(ctx, repository, branchID); err != nil {
return nil, err