Skip to content
This repository has been archived by the owner on Jul 16, 2021. It is now read-only.

Commit

Permalink
MutationStorage integration test
Browse files Browse the repository at this point in the history
  • Loading branch information
@gdbelvin
gdbelvin committed Oct 3, 2019
1 parent 1127fa4 commit a03b5f1
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 93 deletions.
126 changes: 126 additions & 0 deletions core/integration/storagetest/queue.go
View file
@@ -0,0 +1,126 @@
package storagetest

import (
"context"
"testing"
"time"

"github.com/golang/protobuf/proto"
"github.com/google/go-cmp/cmp"
"github.com/google/keytransparency/core/adminserver"
"github.com/google/keytransparency/core/keyserver"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
)

// Queuer writes items to the queue.
type Queuer interface {
keyserver.MutationLogs
adminserver.LogsAdmin
}

type QueueStorageFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) Queuer

type QueueStorageTest func(ctx context.Context, t *testing.T, f QueueStorageFactory)

// RunQueueStorageTests runs all the batch storage tests against the provided map storage implementation.
func RunQueueStorageTests(t *testing.T, factory QueueStorageFactory) {
ctx := context.Background()
b := &QueueTests{}
for name, f := range map[string]QueueStorageTest{
// TODO(gbelvin): Discover test methods via reflection.
"TestSetWritable": b.TestSetWritable,
"TestReadLog": b.TestReadLog,
} {
t.Run(name, func(t *testing.T) { f(ctx, t, factory) })
}
}

// QueueTests is a suite of tests to run against
type QueueTests struct{}

func (QueueTests) TestSetWritable(ctx context.Context, t *testing.T, f QueueStorageFactory) {
directoryID := "TestSetWritable"
for _, tc := range []struct {
desc string
logIDs []int64
set map[int64]bool
wantLogIDs []int64
wantCode codes.Code
}{
{desc: "one row", logIDs: []int64{10}, wantLogIDs: []int64{10}},
{desc: "one row disabled", logIDs: []int64{10}, set: map[int64]bool{10: false}, wantCode: codes.NotFound},
{desc: "one row enabled", logIDs: []int64{1, 2, 3}, set: map[int64]bool{1: false, 2: false}, wantLogIDs: []int64{3}},
{desc: "multi", logIDs: []int64{1, 2, 3}, set: map[int64]bool{1: true, 2: false}, wantLogIDs: []int64{1, 3}},
} {
t.Run(tc.desc, func(t *testing.T) {
m := f(ctx, t, directoryID, tc.logIDs...)
wantLogs := make(map[int64]bool)
for _, logID := range tc.wantLogIDs {
wantLogs[logID] = true
}

for logID, enabled := range tc.set {
if err := m.SetWritable(ctx, directoryID, logID, enabled); err != nil {
t.Errorf("SetWritable(): %v", err)
}
}

logIDs, err := m.ListLogs(ctx, directoryID, true /* Only Writable */)
if status.Code(err) != tc.wantCode {
t.Errorf("ListLogs(): %v, want %v", err, tc.wantCode)
}
if err != nil {
return
}
logs := make(map[int64]bool)
for _, log := range logIDs {
logs[log] = true
}
if got, want := logs, wantLogs; !cmp.Equal(got, want) {
t.Errorf("randLog(): %v, want %v", got, want)
}
})
}
}

func mustMarshal(t *testing.T, p proto.Message) []byte {
t.Helper()
b, err := proto.Marshal(p)
if err != nil {
t.Fatalf("proto.Marshal(): %v", err)
}
return b
}

func (QueueTests) TestReadLog(ctx context.Context, t *testing.T, newForTest QueueStorageFactory) {
directoryID := "TestReadLog"
logID := int64(5)
m := newForTest(ctx, t, directoryID, logID)
for i := byte(0); i < 10; i++ {
entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
if _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil {
t.Fatalf("Send(): %v", err)
}
}

for _, tc := range []struct {
batchSize int32
count int
}{
{batchSize: 0, count: 0},
{batchSize: 1, count: 3},
{batchSize: 4, count: 6},
{batchSize: 100, count: 30},
} {
rows, err := m.ReadLog(ctx, directoryID, logID, 0, time.Now().UnixNano(), tc.batchSize)
if err != nil {
t.Fatalf("ReadLog(%v): %v", tc.batchSize, err)
}
if got, want := len(rows), tc.count; got != want {
t.Fatalf("ReadLog(%v): len: %v, want %v", tc.batchSize, got, want)
}
}
}
110 changes: 17 additions & 93 deletions impl/sql/mutationstorage/queue_test.go
View file
Expand Up @@ -20,77 +20,37 @@ import (
"testing"
"time"

"github.com/golang/protobuf/proto"
"github.com/google/go-cmp/cmp"
"github.com/google/keytransparency/core/integration/storagetest"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
_ "github.com/mattn/go-sqlite3"
)

const directoryID = "default"
func queueFactory(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) storagetest.Queuer {
return newForTest(ctx, t, dirID, logIDs...)
}

func newForTest(ctx context.Context, t testing.TB, logIDs ...int64) *Mutations {
func newForTest(ctx context.Context, t testing.TB, dirID string, logIDs ...int64) *Mutations {
m, err := New(newDB(t))
if err != nil {
t.Fatalf("Failed to create Mutations: %v", err)
t.Fatalf("Failed to create mutations: %v", err)
}
if err := m.AddLogs(ctx, directoryID, logIDs...); err != nil {
if err := m.AddLogs(ctx, dirID, logIDs...); err != nil {
t.Fatalf("AddLogs(): %v", err)
}
return m
}

func TestSetWritable(t *testing.T) {
ctx := context.Background()

for _, tc := range []struct {
desc string
logIDs []int64
set map[int64]bool
wantLogIDs []int64
wantCode codes.Code
}{
{desc: "one row", logIDs: []int64{10}, wantLogIDs: []int64{10}},
{desc: "one row disabled", logIDs: []int64{10}, set: map[int64]bool{10: false}, wantCode: codes.NotFound},
{desc: "one row enabled", logIDs: []int64{1, 2, 3}, set: map[int64]bool{1: false, 2: false}, wantLogIDs: []int64{3}},
{desc: "multi", logIDs: []int64{1, 2, 3}, set: map[int64]bool{1: true, 2: false}, wantLogIDs: []int64{1, 3}},
} {
t.Run(tc.desc, func(t *testing.T) {
m := newForTest(ctx, t, tc.logIDs...)
wantLogs := make(map[int64]bool)
for _, logID := range tc.wantLogIDs {
wantLogs[logID] = true
}

for logID, enabled := range tc.set {
if err := m.SetWritable(ctx, directoryID, logID, enabled); err != nil {
t.Errorf("SetWritable(): %v", err)
}
}

logs := make(map[int64]bool)
// Run enough times to ensure that random sampling will get us all logs.
for i := 0; i < 10*len(tc.logIDs); i++ {
log, err := m.randLog(ctx, directoryID)
if status.Code(err) != tc.wantCode {
t.Errorf("randLog(): %v, want %v", err, tc.wantCode)
}
if err != nil {
break
}
logs[log] = true
}
if got, want := logs, wantLogs; !cmp.Equal(got, want) {
t.Errorf("randLog(): %v, want %v", got, want)
}
})
}
func TestQueueIntegration(t *testing.T) {
storagetest.RunQueueStorageTests(t, queueFactory)
}

func TestRandLog(t *testing.T) {
ctx := context.Background()
directoryID := "TestRandLog"

for _, tc := range []struct {
desc string
Expand All @@ -107,7 +67,7 @@ func TestRandLog(t *testing.T) {
}},
} {
t.Run(tc.desc, func(t *testing.T) {
m := newForTest(ctx, t, tc.send...)
m := newForTest(ctx, t, directoryID, tc.send...)
logs := make(map[int64]bool)
for i := 0; i < 10*len(tc.wantLogs); i++ {
logID, err := m.randLog(ctx, directoryID)
Expand All @@ -128,8 +88,9 @@ func TestRandLog(t *testing.T) {

func BenchmarkSend(b *testing.B) {
ctx := context.Background()
directoryID := "BenchmarkSend"
logID := int64(1)
m := newForTest(ctx, b, logID)
m := newForTest(ctx, b, directoryID, logID)

update := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte("xxxxxxxxxxxxxxxxxx")}}
for _, tc := range []struct {
Expand Down Expand Up @@ -162,7 +123,8 @@ func BenchmarkSend(b *testing.B) {
func TestSend(t *testing.T) {
ctx := context.Background()

m := newForTest(ctx, t, 1, 2)
directoryID := "TestSend"
m := newForTest(ctx, t, directoryID, 1, 2)
update := []byte("bar")
ts1 := time.Now()
ts2 := ts1.Add(time.Duration(1))
Expand Down Expand Up @@ -190,8 +152,9 @@ func TestSend(t *testing.T) {

func TestWatermark(t *testing.T) {
ctx := context.Background()
directoryID := "TestWatermark"
logIDs := []int64{1, 2}
m := newForTest(ctx, t, logIDs...)
m := newForTest(ctx, t, directoryID, logIDs...)
update := []byte("bar")

startTS := time.Now()
Expand Down Expand Up @@ -234,42 +197,3 @@ func TestWatermark(t *testing.T) {
})
}
}

func mustMarshal(t *testing.T, p proto.Message) []byte {
t.Helper()
b, err := proto.Marshal(p)
if err != nil {
t.Fatalf("proto.Marshal(): %v", err)
}
return b
}

func TestReadLog(t *testing.T) {
ctx := context.Background()
logID := int64(5)
m := newForTest(ctx, t, logID)
for i := byte(0); i < 10; i++ {
entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
if _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil {
t.Fatalf("Send(): %v", err)
}
}

for _, tc := range []struct {
batchSize int32
count int
}{
{batchSize: 0, count: 0},
{batchSize: 1, count: 3},
{batchSize: 4, count: 6},
{batchSize: 100, count: 30},
} {
rows, err := m.ReadLog(ctx, directoryID, logID, 0, time.Now().UnixNano(), tc.batchSize)
if err != nil {
t.Fatalf("ReadLog(%v): %v", tc.batchSize, err)
}
if got, want := len(rows), tc.count; got != want {
t.Fatalf("ReadLog(%v): len: %v, want %v", tc.batchSize, got, want)
}
}
}

0 comments on commit a03b5f1

Please sign in to comment.