Wrap lobjs so they can be deduped w/ other IPFS objects, fixes ipfs-s…

…hipyard#12
sameer · Jan 7, 2019 · ca3d565 · ca3d565
1 parent 61547fd
commit ca3d565
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 6 deletions.
diff --git a/cmd/git-remote-ipld/ipld.go b/cmd/git-remote-ipld/ipld.go
@@ -7,11 +7,13 @@ import (
 	"io/ioutil"
 	"path"
 	"strings"
+	"sync"
 
-	core "github.com/ipfs-shipyard/git-remote-ipld/core"
+	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	ipfs "gx/ipfs/QmabBPe1QjKzxHkvoxZmQJYVGE1FUJXE99pyVnkVemf41z/go-ipfs-api"
 
-	"gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
+	core "github.com/ipfs-shipyard/git-remote-ipld/core"
+
 	"gx/ipfs/QmfRYHUcz9QtXq1KK9dQFqprHcpqCVDjswgZDpbHdTzUUW/go-git.v4/plumbing"
 )
 
@@ -242,16 +244,56 @@ func (h *IpnsHandler) Push(remote *core.Remote, local string, remoteRef string)
 func (h *IpnsHandler) bigNodePatcher(tracker *core.Tracker) func(cid.Cid, []byte) error {
 	return func(hash cid.Cid, data []byte) error {
 		if len(data) > (1 << 21) {
-			c, err := h.api.Add(bytes.NewReader(data))
+			endOfHeader := -1
+			for i := 0; i < len(data); i++ {
+				if data[i] == 0 {
+					endOfHeader = i + 1
+					break
+				}
+			}
+
+			var objectHash string
+			var err error
+			if endOfHeader > 0 {
+				wg := sync.WaitGroup{}
+				wg.Add(2)
+
+				var headerHash string
+				var headerHashErr error
+				go func() {
+					headerHash, headerHashErr = h.api.Add(bytes.NewReader(data[0:endOfHeader]))
+					wg.Done()
+				}()
+
+				var dataHash string
+				var dataHashErr error
+				go func() {
+					dataHash, dataHashErr = h.api.Add(bytes.NewReader(data[endOfHeader:]))
+					wg.Done()
+				}()
+
+				wg.Wait()
+				if headerHashErr != nil {
+					return headerHashErr
+				}
+				if dataHashErr != nil {
+					return dataHashErr
+				}
+
+				objectHash, err = h.api.PatchLink(headerHash, "0", dataHash, false)
+			} else {
+				objectHash, err = h.api.Add(bytes.NewReader(data))
+			}
+
 			if err != nil {
 				return err
 			}
 
-			if err := tracker.Set(LOBJ_TRACKER_PRIFIX+"/"+hash.String(), []byte(c)); err != nil {
+			if err := tracker.Set(LOBJ_TRACKER_PRIFIX+"/"+hash.String(), []byte(objectHash)); err != nil {
 				return err
 			}
 
-			h.currentHash, err = h.api.PatchLink(h.currentHash, "objects/"+hash.String(), c, true)
+			h.currentHash, err = h.api.PatchLink(h.currentHash, "objects/"+hash.String(), objectHash, true)
 			if err != nil {
 				return err
 			}

diff --git a/cmd/git-remote-ipld/ipld_test.go b/cmd/git-remote-ipld/ipld_test.go
@@ -2,14 +2,15 @@ package main
 
 import (
 	"bytes"
-	"github.com/ipfs-shipyard/git-remote-ipld/util"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/ipfs-shipyard/git-remote-ipld/util"
 )
 
 func TestCapabilities(t *testing.T) {
@@ -26,6 +27,7 @@ func TestCapabilities(t *testing.T) {
 	listForPushExp := []string{
 		"0000000000000000000000000000000000000000 refs/heads/french",
 		"0000000000000000000000000000000000000000 refs/heads/italian",
+		"0000000000000000000000000000000000000000 refs/heads/lobj",
 		"d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 refs/heads/master",
 	}
 

diff --git a/mock/git/index b/mock/git/index
diff --git a/mock/git/logs/HEAD b/mock/git/logs/HEAD
@@ -6,3 +6,4 @@ d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 31f087b9bf39d5bcbba5d4e80b2b4ff19a71dc0
 d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 Dirk McCormick <dirkmdev@gmail.com> 1517945747 -0500	checkout: moving from master to italian
 d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 78a77abd233c24d8e6a0d0d040c79ae569fc7a19 Dirk McCormick <dirkmdev@gmail.com> 1517945768 -0500	commit: English -> Italian
 78a77abd233c24d8e6a0d0d040c79ae569fc7a19 d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 Dirk McCormick <dirkmdev@gmail.com> 1517945774 -0500	checkout: moving from italian to master
+d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 8f329544bb70a739a8a65eda1c0aa949a971dede Sameer Puri <sameer@users.noreply.github.com> 1546885786 -0600	commit: Add lobj file
diff --git a/mock/git/logs/refs/heads/lobj b/mock/git/logs/refs/heads/lobj
@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 Sameer Puri <sameer@users.noreply.github.com> 1546885771 -0600	branch: Created from HEAD
+d5b0d08c180fd7a9bf4f684a37e60ceeb4d25ec8 8f329544bb70a739a8a65eda1c0aa949a971dede Sameer Puri <sameer@users.noreply.github.com> 1546885786 -0600	commit: Add lobj file
diff --git a/mock/git/objects/23/bdd995170af51795cefb76f267a9a1692f2c5e b/mock/git/objects/23/bdd995170af51795cefb76f267a9a1692f2c5e
diff --git a/mock/git/objects/8f/329544bb70a739a8a65eda1c0aa949a971dede b/mock/git/objects/8f/329544bb70a739a8a65eda1c0aa949a971dede
diff --git a/mock/git/objects/9e/62b9581645e40f4ed6985902eff1e9bc1d326c b/mock/git/objects/9e/62b9581645e40f4ed6985902eff1e9bc1d326c
diff --git a/mock/git/refs/heads/lobj b/mock/git/refs/heads/lobj
@@ -0,0 +1 @@
+8f329544bb70a739a8a65eda1c0aa949a971dede