Merge pull request #2 from levenlabs/master

Added Merge, MarshalBinary, UnmarshalBinary

Makefile

@@ -1,8 +1,8 @@
 test:
-	GOCACHE=off go test -v ring_test.go
+	go test -v ring_test.go
 
 coverage:
-	GOCACHE=off go test -covermode=count -coverprofile=count.out ./...
+	go test -covermode=count -coverprofile=count.out ./...
 	go tool cover -func=count.out
 	rm count.out
 

README.md

@@ -26,17 +26,23 @@ the target.
 === RUN   TestBadParameters
 --- PASS: TestBadParameters (0.00s)
 === RUN   TestReset
---- PASS: TestReset (0.30s)
+--- PASS: TestReset (0.26s)
 === RUN   TestData
---- PASS: TestData (17.27s)
+--- PASS: TestData (14.07s)
+=== RUN   TestMerge
+--- PASS: TestMerge (13.78s)
+=== RUN   TestMarshal
+--- PASS: TestMarshal (14.48s)
 PASS
 >> Number of elements:  1000000
 >> Target false positive rate:  0.001000
->> Number of false positives:  137
->> Actual false positive rate:  0.000137
->> Benchmark Add():   5000000          257 ns/op
->> Benchmark Test():  10000000         175 ns/op
-ok      command-line-arguments  21.089s
+>> Number of false positives:  99
+>> Actual false positive rate:  0.000099
+>> Number of false negatives:  0
+>> Actual false negative rate:  0.000000
+>> Benchmark Add():  10000000          158 ns/op
+>> Benchmark Test():  10000000         173 ns/op
+ok      command-line-arguments  47.914s
 ```
 
 ## License

ring.go

@@ -5,7 +5,9 @@
 package ring
 
 import (
+	"encoding/binary"
 	"errors"
+	"fmt"
 	"math"
 	"sync"
 )
@@ -72,14 +74,66 @@ func (r *Ring) Test(data []byte) bool {
 	// generate hashes
 	hash := generateMultiHash(data)
 	r.mutex.RLock()
+	defer r.mutex.RUnlock()
 	for i := uint64(0); i < uint64(r.hash); i++ {
 		index := getRound(hash, i) % r.size
 		// check if index%8-th bit is not active
 		if (r.bits[index/8] & (1 << (index % 8))) == 0 {
-			r.mutex.RUnlock()
 			return false
 		}
 	}
-	r.mutex.RUnlock()
 	return true
 }
+
+// Merges the sent Ring into itself.
+func (r *Ring) Merge(m *Ring) error {
+	if r.size != m.size || r.hash != m.hash {
+		return errors.New("rings must have the same m/k parameters")
+	}
+
+	r.mutex.Lock()
+	m.mutex.RLock()
+	for i := 0; i < len(m.bits); i++ {
+		r.bits[i] |= m.bits[i]
+	}
+	r.mutex.Unlock()
+	m.mutex.RUnlock()
+	return nil
+}
+
+// MarshalBinary implements the encoding.BinaryMarshaler interface.
+func (r *Ring) MarshalBinary() ([]byte, error) {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	out := make([]byte, len(r.bits)+17)
+	// store a version for future compatibility
+	out[0] = 1
+	binary.BigEndian.PutUint64(out[1:9], r.size)
+	binary.BigEndian.PutUint64(out[9:17], r.hash)
+	copy(out[17:], r.bits)
+	return out, nil
+}
+
+// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
+func (r *Ring) UnmarshalBinary(data []byte) error {
+	// 17 bytes for version + size + hash and 1 byte at least for bits
+	if len(data) < 17+1 {
+		return fmt.Errorf("incorrect length: %d", len(data))
+	}
+	if data[0] != 1 {
+		return fmt.Errorf("unexpected version: %d", data[0])
+	}
+	if r.mutex == nil {
+		r.mutex = new(sync.RWMutex)
+	}
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.size = binary.BigEndian.Uint64(data[1:9])
+	r.hash = binary.BigEndian.Uint64(data[9:17])
+	// sanity check against the bits being the wrong size
+	if len(r.bits) != int(r.size/8+1) {
+		r.bits = make([]uint8, r.size/8+1)
+	}
+	copy(r.bits, data[17:])
+	return nil
+}

ring_test.go

@@ -24,8 +24,10 @@ var (
 	r, _ = ring.Init(tests, fpRate)
 	// benchmark
 	rBench, _ = ring.Init(tests, fpRate)
-	// error count
-	errorCount = 0
+	// false positive count
+	positiveCount = 0
+	// false negative count
+	negativeCount = 0
 )
 
 // TestMain performs unit tests and benchmarks.
@@ -37,8 +39,10 @@ func TestMain(m *testing.M) {
 	// print stats
 	fmt.Printf(">> Number of elements:  %d\n", tests)
 	fmt.Printf(">> Target false positive rate:  %f\n", fpRate)
-	fmt.Printf(">> Number of false positives:  %d\n", errorCount)
-	fmt.Printf(">> Actual false positive rate:  %f\n", float64(errorCount)/tests)
+	fmt.Printf(">> Number of false positives:  %d\n", positiveCount)
+	fmt.Printf(">> Actual false positive rate:  %f\n", float64(positiveCount)/tests)
+	fmt.Printf(">> Number of false negatives:  %d\n", negativeCount)
+	fmt.Printf(">> Actual false negative rate:  %f\n", float64(negativeCount)/tests)
 
 	// benchmarks
 	fmt.Printf(">> Benchmark Add():  %s\n", testing.Benchmark(BenchmarkAdd))
@@ -47,9 +51,12 @@ func TestMain(m *testing.M) {
 	// actual failure if actual exceeds desired false positive rate
 	if ret != 0 {
 		os.Exit(ret)
-	} else if float64(errorCount)/tests > fpRate {
+	} else if float64(positiveCount)/tests > fpRate {
 		fmt.Printf("False positive threshold exceeded !!\n")
 		os.Exit(1)
+	} else if negativeCount > 0 {
+		fmt.Printf("False negative threshold exceeded !!\n")
+		os.Exit(1)
 	} else {
 		os.Exit(0)
 	}
@@ -134,13 +141,114 @@ func TestData(t *testing.T) {
 
 		// test before adding
 		if r.Test(token) {
-			errorCount++
+			positiveCount++
 		}
 		r.Add(token)
 		// test after adding
 		if !r.Test(token) {
-			errorCount++
+			negativeCount++
+		}
+	}
+}
+
+// TestMerge ensures that a Merge produces the right Ring.
+func TestMerge(t *testing.T) {
+	var token []byte
+	// byte range of random data
+	min, max := 8, 8192
+	// test a range of sizes
+	for i := uint(0); i < 20; i++ {
+		innerCount := 1 << i
+		elems := make([][]byte, innerCount)
+		r, _ := ring.Init(tests, fpRate)
+		r2, _ := ring.Init(tests, fpRate)
+		for j := 0; j < innerCount; j++ {
+			// generate random data
+			size := rand.Intn(max-min) + min
+			token = make([]byte, size)
+			rand.Read(token)
+			elems[j] = token
+			if size&2 == 0 {
+				r.Add(token)
+			} else {
+				r2.Add(token)
+			}
+		}
+		if err := r.Merge(r2); err != nil {
+			t.Errorf("Error calling Merge: %v", err)
+			break
 		}
+		notFound := 0
+		for j := 0; j < innerCount; j++ {
+			if !r.Test(elems[j]) {
+				notFound++
+			}
+		}
+		if notFound > 0 {
+			t.Errorf("Unexpected number of tokens not found: %v", notFound)
+			break
+		}
+	}
+
+	r, _ := ring.Init(tests, fpRate)
+	// different params should fail to merge
+	r2, _ := ring.Init(tests, 0.1)
+	if r.Merge(r2) == nil {
+		t.Errorf("Expected error calling Merge with different size")
+	}
+	r2, _ = ring.Init(100, fpRate)
+	if r.Merge(r2) == nil {
+		t.Errorf("Expected error calling Merge with different fp")
+	}
+}
+
+// TestMarshal ensures that the Marshal and Unmarshal methods produce
+// duplicate Ring's.
+func TestMarshal(t *testing.T) {
+	// Travis CI has strict memory limits that we hit if too high
+	size := tests / 100
+	r, _ := ring.Init(size, fpRate)
+	elems := make([][]byte, size)
+	var token []byte
+	// byte range of random data
+	min, max := 8, 8192
+	// test a range of sizes
+	for i := uint(0); i < uint(size); i++ {
+		// generate random data
+		size := rand.Intn(max-min) + min
+		token = make([]byte, size)
+		rand.Read(token)
+		elems[i] = token
+		r.Add(token)
+	}
+
+	out, err := r.MarshalBinary()
+	if err != nil {
+		t.Errorf("Unexpected error from MarshalBinary: %v", err)
+		return
+	}
+
+	r2 := new(ring.Ring)
+	r2.UnmarshalBinary(out)
+
+	notFound := 0
+	for _, el := range elems {
+		if !r.Test(el) {
+			notFound++
+		}
+	}
+	if notFound > 0 {
+		t.Errorf("Unexpected number of tokens not found: %v", notFound)
+	}
+
+	// unexpected length should error
+	if r2.UnmarshalBinary(nil) == nil {
+		t.Errorf("Expected error calling UnmarshalBinary with nil")
+	}
+	// unexpected version should error
+	out[0] = 0
+	if r2.UnmarshalBinary(out) == nil {
+		t.Errorf("Expected error calling UnmarshalBinary with wrong version")
 	}
 }