Skip to content
Permalink
Browse files

Merge pull request #1180 from orisano/feat-improve-patch-delta

feat: improve patch delta performance
  • Loading branch information...
mcuadros committed Aug 11, 2019
2 parents 9feb9fe + d456ce9 commit 883c226587bef98b0e30cd9b0d424f4cb08ebbec
Showing with 89 additions and 63 deletions.
  1. +53 −46 plumbing/format/packfile/parser.go
  2. +36 −17 plumbing/format/packfile/patch_delta.go
@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"io"
"io/ioutil"

"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/cache"
@@ -263,11 +264,14 @@ func (p *Parser) indexObjects() error {
}

func (p *Parser) resolveDeltas() error {
buf := &bytes.Buffer{}
for _, obj := range p.oi {
content, err := p.get(obj)
buf.Reset()
err := p.get(obj, buf)
if err != nil {
return err
}
content := buf.Bytes()

if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
return err
@@ -279,7 +283,7 @@ func (p *Parser) resolveDeltas() error {

if !obj.IsDelta() && len(obj.Children) > 0 {
for _, child := range obj.Children {
if _, err := p.resolveObject(child, content); err != nil {
if err := p.resolveObject(ioutil.Discard, child, content); err != nil {
return err
}
}
@@ -294,120 +298,123 @@ func (p *Parser) resolveDeltas() error {
return nil
}

func (p *Parser) get(o *objectInfo) (b []byte, err error) {
var ok bool
func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) error {
if !o.ExternalRef { // skip cache check for placeholder parents
b, ok = p.cache.Get(o.Offset)
b, ok := p.cache.Get(o.Offset)
if ok {
_, err := buf.Write(b)
return err
}
}

// If it's not on the cache and is not a delta we can try to find it in the
// storage, if there's one. External refs must enter here.
if !ok && p.storage != nil && !o.Type.IsDelta() {
if p.storage != nil && !o.Type.IsDelta() {
e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1)
if err != nil {
return nil, err
return err
}
o.Type = e.Type()

r, err := e.Reader()
if err != nil {
return nil, err
}

b = make([]byte, e.Size())
if _, err = r.Read(b); err != nil {
return nil, err
return err
}
}

if b != nil {
return b, nil
_, err = buf.ReadFrom(io.LimitReader(r, e.Size()))
return err
}

if o.ExternalRef {
// we were not able to resolve a ref in a thin pack
return nil, ErrReferenceDeltaNotFound
return ErrReferenceDeltaNotFound
}

var data []byte
if o.DiskType.IsDelta() {
base, err := p.get(o.Parent)
b := bufPool.Get().(*bytes.Buffer)
defer bufPool.Put(b)
b.Reset()
err := p.get(o.Parent, b)
if err != nil {
return nil, err
return err
}
base := b.Bytes()

data, err = p.resolveObject(o, base)
err = p.resolveObject(buf, o, base)
if err != nil {
return nil, err
return err
}
} else {
data, err = p.readData(o)
err := p.readData(buf, o)
if err != nil {
return nil, err
return err
}
}

if len(o.Children) > 0 {
data := make([]byte, buf.Len())
copy(data, buf.Bytes())
p.cache.Put(o.Offset, data)
}

return data, nil
return nil
}

func (p *Parser) resolveObject(
w io.Writer,
o *objectInfo,
base []byte,
) ([]byte, error) {
) error {
if !o.DiskType.IsDelta() {
return nil, nil
return nil
}

data, err := p.readData(o)
buf := bufPool.Get().(*bytes.Buffer)
defer bufPool.Put(buf)
buf.Reset()
err := p.readData(buf, o)
if err != nil {
return nil, err
return err
}
data := buf.Bytes()

data, err = applyPatchBase(o, data, base)
if err != nil {
return nil, err
return err
}

if p.storage != nil {
obj := new(plumbing.MemoryObject)
obj.SetSize(o.Size())
obj.SetType(o.Type)
if _, err := obj.Write(data); err != nil {
return nil, err
return err
}

if _, err := p.storage.SetEncodedObject(obj); err != nil {
return nil, err
return err
}
}

return data, nil
_, err = w.Write(data)
return err
}

func (p *Parser) readData(o *objectInfo) ([]byte, error) {
func (p *Parser) readData(w io.Writer, o *objectInfo) error {
if !p.scanner.IsSeekable && o.DiskType.IsDelta() {
data, ok := p.deltas[o.Offset]
if !ok {
return nil, ErrDeltaNotCached
return ErrDeltaNotCached
}

return data, nil
_, err := w.Write(data)
return err
}

if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil {
return nil, err
return err
}

buf := new(bytes.Buffer)
if _, _, err := p.scanner.NextObject(buf); err != nil {
return nil, err
if _, _, err := p.scanner.NextObject(w); err != nil {
return err
}

return buf.Bytes(), nil
return nil
}

func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
@@ -1,8 +1,9 @@
package packfile

import (
"bytes"
"errors"
"io/ioutil"
"io"

"gopkg.in/src-d/go-git.v4/plumbing"
)
@@ -26,19 +27,29 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error {
return err
}

src, err := ioutil.ReadAll(r)
buf := bufPool.Get().(*bytes.Buffer)
defer bufPool.Put(buf)
buf.Reset()
_, err = buf.ReadFrom(r)
if err != nil {
return err
}
src := buf.Bytes()

dst, err := PatchDelta(src, delta)
dst := bufPool.Get().(*bytes.Buffer)
defer bufPool.Put(dst)
dst.Reset()
err = patchDelta(dst, src, delta)
if err != nil {
return err
}

target.SetSize(int64(len(dst)))

_, err = w.Write(dst)
target.SetSize(int64(dst.Len()))

b := byteSlicePool.Get().([]byte)
_, err = io.CopyBuffer(w, dst, b)
byteSlicePool.Put(b)
return err
}

@@ -51,23 +62,31 @@ var (
// An error will be returned if delta is corrupted (ErrDeltaLen) or an action command
// is not copy from source or copy from delta (ErrDeltaCmd).
func PatchDelta(src, delta []byte) ([]byte, error) {
b := &bytes.Buffer{}
if err := patchDelta(b, src, delta); err != nil {
return nil, err
}
return b.Bytes(), nil
}

func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
if len(delta) < deltaSizeMin {
return nil, ErrInvalidDelta
return ErrInvalidDelta
}

srcSz, delta := decodeLEB128(delta)
if srcSz != uint(len(src)) {
return nil, ErrInvalidDelta
return ErrInvalidDelta
}

targetSz, delta := decodeLEB128(delta)
remainingTargetSz := targetSz

var cmd byte
dest := make([]byte, 0, targetSz)
dst.Grow(int(targetSz))
for {
if len(delta) == 0 {
return nil, ErrInvalidDelta
return ErrInvalidDelta
}

cmd = delta[0]
@@ -77,43 +96,43 @@ func PatchDelta(src, delta []byte) ([]byte, error) {
var err error
offset, delta, err = decodeOffset(cmd, delta)
if err != nil {
return nil, err
return err
}

sz, delta, err = decodeSize(cmd, delta)
if err != nil {
return nil, err
return err
}

if invalidSize(sz, targetSz) ||
invalidOffsetSize(offset, sz, srcSz) {
break
}
dest = append(dest, src[offset:offset+sz]...)
dst.Write(src[offset:offset+sz])
remainingTargetSz -= sz
} else if isCopyFromDelta(cmd) {
sz := uint(cmd) // cmd is the size itself
if invalidSize(sz, targetSz) {
return nil, ErrInvalidDelta
return ErrInvalidDelta
}

if uint(len(delta)) < sz {
return nil, ErrInvalidDelta
return ErrInvalidDelta
}

dest = append(dest, delta[0:sz]...)
dst.Write(delta[0:sz])
remainingTargetSz -= sz
delta = delta[sz:]
} else {
return nil, ErrDeltaCmd
return ErrDeltaCmd
}

if remainingTargetSz <= 0 {
break
}
}

return dest, nil
return nil
}

// Decodes a number encoded as an unsigned LEB128 at the start of some

0 comments on commit 883c226

Please sign in to comment.
You can’t perform that action at this time.