Generating a test suite with breakdancer.

There are still a few ops not handled, but many are.

cbugg: close bug-453

.gitignore

@@ -1,9 +1,11 @@
 #*
 *.pprof
+*.pyc
 *.test
 *~
 .#*
+/generated_suite_test.json
+/tools/createbuckets/createbuckets
 _*
 cbgb/cbgb
 tmp
-/tools/createbuckets/createbuckets

.travis.yml

@@ -2,6 +2,7 @@ language: go
 
 script:
   - go get github.com/dustin/go-jsonpointer
+  - ./gen_ops_tests.py
   - go test -v ./...
   - go vet ./...
 

break_test.go

@@ -0,0 +1,203 @@
+package cbgb
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/dustin/gomemcached"
+)
+
+const tmpdirName = "break-tmp"
+const testKey = "somekey"
+const expTime = 3600
+
+type testItem struct {
+	Op    string
+	Val   *string
+	Error bool
+}
+
+type testDef map[string][]testItem
+
+/*
+  "addaddaddget": [
+      {"op": "add", "val": "0", "error": false},
+      {"op": "add", "val": "0", "error": true},
+      {"op": "add", "val": "0", "error": true},
+      {"op": "get", "val": "0", "error": false},
+      {"op": "assert", "val": "0"}
+  ]
+*/
+
+type op func(b *vbucket, memo interface{}) (interface{}, error)
+
+func dispatchTestCommand(v *vbucket, cas uint64,
+	cmd gomemcached.CommandCode) (*gomemcached.MCResponse, error) {
+
+	req := &gomemcached.MCRequest{
+		Opcode: cmd,
+		Key:    []byte(testKey),
+		Cas:    cas,
+	}
+	switch cmd {
+	case gomemcached.ADD, gomemcached.SET:
+		req.Extras = make([]byte, 8)
+		binary.BigEndian.PutUint64(req.Extras, uint64(0)<<32|uint64(expTime))
+	}
+	res := v.Dispatch(ioutil.Discard, req)
+	var err error
+	if res.Status != 0 {
+		err = res
+	}
+	return res, err
+}
+
+func shortTestDispatch(v *vbucket, cmd gomemcached.CommandCode) error {
+	_, err := dispatchTestCommand(v, 0, cmd)
+	return err
+}
+
+var opMap = map[string]op{
+	"add": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, shortTestDispatch(v, gomemcached.ADD)
+	},
+	"set": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, shortTestDispatch(v, gomemcached.SET)
+	},
+	"setRetainCAS": func(v *vbucket, memo interface{}) (interface{}, error) {
+		res, err := dispatchTestCommand(v, 0, gomemcached.SET)
+		if err != nil {
+			return nil, err
+		}
+		return res.Cas, err
+	},
+	"setUsingCAS": func(v *vbucket, memo interface{}) (interface{}, error) {
+		casid, ok := memo.(uint64)
+		if !ok {
+			return nil, fmt.Errorf("Memo doesn't contain a CAS: %+v", memo)
+		}
+		_, err := dispatchTestCommand(v, casid, gomemcached.SET)
+		if err != nil {
+			return nil, err
+		}
+		return 0, err
+	},
+	"get": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, shortTestDispatch(v, gomemcached.GET)
+	},
+	"del": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, shortTestDispatch(v, gomemcached.DELETE)
+	},
+	"assert": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, nil
+	},
+	"assertMissing": func(v *vbucket, memo interface{}) (interface{}, error) {
+		return nil, nil
+	},
+}
+
+func runTest(t *testing.T, buckets *Buckets, name string, items []testItem) {
+	for _, i := range items {
+		if _, ok := opMap[i.Op]; !ok {
+			// t.Logf("Skipping %v because of %v", name, i.Op)
+			return
+		}
+	}
+
+	b, err := buckets.New(name, &BucketSettings{MemoryOnly: 2})
+	if err != nil {
+		t.Errorf("Error making bucket: %v", err)
+		return
+	}
+	defer b.Close()
+
+	vb, err := b.CreateVBucket(0)
+	if err != nil {
+		t.Errorf("Error making vbucket: %v", err)
+		return
+	}
+
+	var memo interface{}
+	for n, i := range items {
+		mtmp, err := opMap[i.Op](vb, memo)
+		if mtmp != nil {
+			memo = mtmp
+		}
+		if (err != nil) != i.Error {
+			t.Errorf("Unexpected error state in %v on op %v: %+v: %v",
+				name, n, i, err)
+			return
+		}
+	}
+
+	t.Logf("PASS: %v", name)
+}
+
+func testRunner(t *testing.T, buckets *Buckets,
+	wg *sync.WaitGroup, ch <-chan testDef) {
+
+	defer wg.Done()
+	for td := range ch {
+		for k, seq := range td {
+			runTest(t, buckets, k, seq)
+		}
+	}
+}
+
+func TestAllTheThings(t *testing.T) {
+	os.RemoveAll(tmpdirName)
+	os.Mkdir(tmpdirName, 0777)
+	defer os.RemoveAll(tmpdirName)
+
+	f, err := os.Open("generated_suite_test.json")
+	if err != nil {
+		t.Logf("Error opening test inputs: %v -- skipping", err)
+		return
+	}
+	defer f.Close()
+
+	d := json.NewDecoder(f)
+
+	ch := make(chan testDef)
+	wg := &sync.WaitGroup{}
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+
+		buckets, err := NewBuckets(tmpdirName, &BucketSettings{
+			NumPartitions: 1,
+			MemoryOnly:    2,
+		})
+		if err != nil {
+			t.Fatalf("Error making buckets: %v", err)
+		}
+		defer buckets.CloseAll()
+
+		go testRunner(t, buckets, wg, ch)
+	}
+
+	ran := 0
+	for {
+		aTest := testDef{}
+		err = d.Decode(&aTest)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("Error decoding things: %v", err)
+		}
+
+		ch <- aTest
+		ran++
+	}
+	close(ch)
+	wg.Wait()
+
+	t.Logf("Ran %v tests", ran)
+}

breakdancer.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+import itertools
+
+class Condition(object):
+    """Something asserted to be true during the test.
+
+    A given condition may be used as a precondition or a
+    postcondition."""
+
+    def __call__(self, k, state):
+        """Called with a key and a state.  True if the condition is met."""
+        return True
+
+class Effect(object):
+    """The affect an action will perform."""
+
+    def __call__(self, k, state):
+        """Called with a key and a state.
+
+        The effect modifies the state as appropriate."""
+
+class Action(object):
+    """Actions are the operations that will be permuted into test cases.
+
+    Each action has a collection of preconditions and postconditions
+    that will be evaluated for checking input and output state for the
+    action.
+
+    Action.preconditions is the collection of conditions that must all
+    be true upon input to the action.  If any condition is not true,
+    the effect is not executed and the action state is considered
+    "errored."
+
+    Action.effect is the callable that is expected to alter the state
+    to satisfy the postconditions of the action.
+
+    Action.postconditions is the collection of conditions that must
+    all be true after the effect of the action completes.
+    """
+
+    preconditions = []
+    effect = None
+    postconditions = []
+    enabled = True
+
+    @property
+    def name(self):
+        """The name of this action (default derived from class name)"""
+        n = self.__class__.__name__
+        return n[0].lower() + n[1:]
+
+class Driver(object):
+    """The driver "performs" the test."""
+
+    def newState(self):
+        """Initialize and return the state for a test."""
+        return {}
+
+    def preSuite(self, seq):
+        """Invoked with the sequence of tests before any are run."""
+
+    def startSequence(self, seq):
+        """Invoked with the sequence of actions in a single test
+        before it is performed."""
+
+    def startAction(self, action):
+        """Invoked when before starting an action."""
+
+    def endAction(self, action, state, errored):
+        """Invoked after the action is performed."""
+
+    def endSequence(self, seq, state):
+        """Invoked at the end of a sequence of tests."""
+
+    def postSuite(self, seq):
+        """Invoked with the sequence of tests after all of them are run."""
+
+def runTest(actions, driver, duplicates=3, length=4):
+    """Run a test with the given collection of actions and driver.
+
+    The optional argument `duplicates' specifies how many times a
+    given action may be duplicated in a sequence.
+
+    The optional argument `length` specifies how long each test
+    sequence is.
+    """
+
+    instances = itertools.chain(*itertools.repeat([a() for a in actions],
+                                                  duplicates))
+    tests = set(itertools.permutations(instances, length))
+    driver.preSuite(tests)
+    for seq in sorted(tests):
+        state = driver.newState()
+        driver.startSequence(seq)
+        for a in seq:
+            driver.startAction(a)
+            haserror = not all(p(state) for p in a.preconditions)
+            if not haserror:
+                try:
+                    a.effect(state)
+                    haserror = not all(p(state) for p in a.postconditions)
+                except:
+                    haserror = True
+            driver.endAction(a, state, haserror)
+        driver.endSequence(seq, state)
+    driver.postSuite(tests)
+
+def findActions(classes):
+    """Helper function to extract action subclasses from a collection
+    of classes."""
+
+    actions = []
+    for __t in (t for t in classes if isinstance(type, type(t))):
+        if Action in __t.__mro__ and __t != Action and __t.enabled:
+            actions.append(__t)
+    return actions

bucket.go

@@ -117,7 +117,6 @@ func (b *Buckets) New(name string,
 	if err != nil {
 		return nil, err
 	}
-	log.Printf("allocating bucket: %v, settings: %+v", name, settings)
 
 	if rv, err = NewBucket(bdir, settings); err != nil {
 		return nil, err