Incremental loading for sets.

Issue #11

types/incremental_test.go

@@ -7,11 +7,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestIncrementalLoad(t *testing.T) {
-	assert := assert.New(t)
-	cs := &testStore{ChunkStore: &chunks.MemoryStore{}}
-
-	expected := NewList(
+var (
+	testVals = []Value{
 		Bool(true),
 		Int16(1),
 		Int32(1),
@@ -25,34 +22,62 @@ func TestIncrementalLoad(t *testing.T) {
 		NewBlob([]byte("hi")),
 		NewSet(NewString("hi")),
 		NewList(NewString("hi")),
-		NewMap(NewString("hi"), NewString("hi")))
+		NewMap(NewString("hi"), NewString("hi")),
+	}
+)
+
+func isEncodedOutOfLine(v Value) int {
+	switch v.(type) {
+	case Blob, Set, List, Map:
+		return 1
+	}
+	return 0
+}
+
+func TestIncrementalLoadList(t *testing.T) {
+	assert := assert.New(t)
+	cs := &testStore{ChunkStore: &chunks.MemoryStore{}}
 
+	expected := NewList(testVals...)
 	ref, err := WriteValue(expected, cs)
 	assert.NoError(err)
 
 	actualVar, err := ReadValue(ref, cs)
 	assert.NoError(err)
 	actual := actualVar.(List)
 
-	prev := cs.count
-	assert.Equal(1, prev)
+	expectedCount := cs.count
+	assert.Equal(1, expectedCount)
 	for i := uint64(0); i < expected.Len(); i++ {
 		v := actual.Get(i)
 		assert.True(expected.Get(i).Equals(v))
 
-		next := prev
-		switch v.(type) {
-		case Blob, Set, List, Map:
-			// These are the types that are out-of-line in our current encoding. So we expect them to cause a load.
-			next += 1
-		}
-
-		assert.Equal(next, cs.count)
+		expectedCount += isEncodedOutOfLine(v)
+		assert.Equal(expectedCount, cs.count)
 
 		// Do it again to make sure multiple derefs don't do multiple loads.
 		v = actual.Get(i)
-		assert.Equal(next, cs.count)
-
-		prev = next
+		assert.Equal(expectedCount, cs.count)
 	}
 }
+
+func TestIncrementalLoadSet(t *testing.T) {
+	assert := assert.New(t)
+	cs := &testStore{ChunkStore: &chunks.MemoryStore{}}
+
+	expected := NewSet(testVals...)
+	ref, err := WriteValue(expected, cs)
+	assert.NoError(err)
+
+	actualVar, err := ReadValue(ref, cs)
+	assert.NoError(err)
+	actual := actualVar.(Set)
+
+	expectedCount := cs.count
+	assert.Equal(1, expectedCount)
+	actual.Iter(func(v Value) (stop bool) {
+		expectedCount += isEncodedOutOfLine(v)
+		assert.Equal(expectedCount, cs.count)
+		return
+	})
+}

types/json_decode.go

@@ -113,19 +113,15 @@ func jsonDecodeList(input []interface{}, s chunks.ChunkSource) (future, error) {
 }
 
 func jsonDecodeSet(input []interface{}, s chunks.ChunkSource) (future, error) {
-	vals := []Value{}
+	output := []future{}
 	for _, inVal := range input {
 		f, err := jsonDecodeValue(inVal, s)
 		if err != nil {
 			return nil, err
 		}
-		outVal, err := f.Deref(s)
-		if err != nil {
-			return nil, err
-		}
-		vals = append(vals, outVal)
+		output = append(output, f)
 	}
-	return futureFromValue(NewSet(vals...)), nil
+	return futureFromValue(setFromFutures(output, s)), nil
 }
 
 func jsonDecodeMap(input []interface{}, s chunks.ChunkSource) (future, error) {

types/set.go

@@ -1,18 +1,25 @@
 package types
 
 import (
+	"github.com/attic-labs/noms/chunks"
+	. "github.com/attic-labs/noms/dbg"
 	"github.com/attic-labs/noms/ref"
 )
 
-type setData map[ref.Ref]Value
+type setData map[ref.Ref]future
 
 type Set struct {
 	m  setData
+	cs chunks.ChunkSource
 	cr *cachedRef
 }
 
 func NewSet(v ...Value) Set {
-	return newSetFromData(buildSetData(setData{}, v))
+	return newSetFromData(buildSetData(setData{}, valuesToFutures(v)), nil)
+}
+
+func setFromFutures(f []future, cs chunks.ChunkSource) Set {
+	return newSetFromData(buildSetData(setData{}, f), cs)
 }
 
 func (fs Set) Empty() bool {
@@ -29,7 +36,7 @@ func (fs Set) Has(v Value) bool {
 }
 
 func (fs Set) Insert(values ...Value) Set {
-	return newSetFromData(buildSetData(fs.m, values))
+	return newSetFromData(buildSetData(fs.m, valuesToFutures(values)), fs.cs)
 }
 
 func (fs Set) Remove(values ...Value) Set {
@@ -39,7 +46,7 @@ func (fs Set) Remove(values ...Value) Set {
 			delete(m2, v.Ref())
 		}
 	}
-	return newSetFromData(m2)
+	return newSetFromData(m2, fs.cs)
 }
 
 func (fs Set) Union(others ...Set) (result Set) {
@@ -68,15 +75,19 @@ type setIterCallback func(v Value) bool
 
 func (fm Set) Iter(cb setIterCallback) {
 	// TODO: sort iteration order
-	for _, v := range fm.m {
+	for _, f := range fm.m {
+		v, err := f.Deref(fm.cs)
+		Chk.NoError(err)
 		if cb(v) {
 			break
 		}
 	}
 }
 
 func (fm Set) Any() Value {
-	for _, v := range fm.m {
+	for _, f := range fm.m {
+		v, err := f.Deref(fm.cs)
+		Chk.NoError(err)
 		return v
 	}
 	return nil
@@ -94,25 +105,22 @@ func (fs Set) Equals(other Value) bool {
 	}
 }
 
-func newSetFromData(m setData) Set {
-	return Set{
-		m:  m,
-		cr: &cachedRef{},
-	}
+func newSetFromData(m setData, cs chunks.ChunkSource) Set {
+	return Set{m, cs, &cachedRef{}}
 }
 
 func copySetData(m setData) setData {
 	r := setData{}
-	for k, v := range m {
-		r[k] = v
+	for k, f := range m {
+		r[k] = f
 	}
 	return r
 }
 
-func buildSetData(old setData, values []Value) setData {
+func buildSetData(old setData, futures []future) setData {
 	m := copySetData(old)
-	for _, v := range values {
-		m[v.Ref()] = v
+	for _, f := range futures {
+		m[f.Ref()] = f
 	}
 	return m
 }