Skip to content

Commit

Permalink
Fixed README merge conflict.
Browse files Browse the repository at this point in the history
  • Loading branch information
Jonathan Gillham committed Jul 9, 2014
2 parents 4b49f61 + 19a6c38 commit 0985407
Show file tree
Hide file tree
Showing 14 changed files with 548 additions and 72 deletions.
11 changes: 9 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,12 @@ before_install:
- wget https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.6.zip
- unzip -d $HOME go_appengine_sdk_linux_amd64-1.9.6.zip
- export PATH=$PATH:$HOME/go_appengine
install: goapp get github.com/qedus/nds
script: goapp test
install:
- goapp get -v github.com/qedus/nds
- goapp get -v code.google.com/p/go.tools/cmd/cover
script:
- goapp test -v -covermode=count -coverprofile=profile.cov
after_success:
- goapp get -v github.com/mattn/goveralls
- export PATH=$PATH:$HOME/gopath/bin
- goveralls -coverprofile=profile.cov -service=travis-ci
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# nds

[![Build Status](https://travis-ci.org/qedus/nds.svg?branch=master)](https://travis-ci.org/qedus/nds)
[![Build Status](https://travis-ci.org/qedus/nds.svg?branch=master)](https://travis-ci.org/qedus/nds) [![Coverage Status](https://coveralls.io/repos/qedus/nds/badge.png)](https://coveralls.io/r/qedus/nds)

Package `github.com/qedus/nds` is a datastore API for the Google App Engine (GAE) [Go Runtime Environment](https://developers.google.com/appengine/docs/go/) that uses memcache to cache all datastore requests. This package guarantees strong cache consistency, meaning you will never get data from a stale cache.

Exposed parts of this API are similar to the official one distributed by Google ([`appengine/datastore`](https://developers.google.com/appengine/docs/go/datastore/reference)). However, underneath `github.com/qedus/nds` uses a caching stategy similar to the GAE [Python NDB API](https://developers.google.com/appengine/docs/python/ndb/). In fact the caching strategy used here even fixes one or two of the Python NDB [caching consistency bugs](http://goo.gl/3ByVlA).
Exposed parts of this API are the same as the official one distributed by Google ([`appengine/datastore`](https://developers.google.com/appengine/docs/go/datastore/reference)). However, underneath `github.com/qedus/nds` uses a caching stategy similar to the GAE [Python NDB API](https://developers.google.com/appengine/docs/python/ndb/). In fact the caching strategy used here even fixes one or two of the Python NDB [caching consistency bugs](http://goo.gl/3ByVlA).

You can find the API documentation at [http://godoc.org/github.com/qedus/nds](http://godoc.org/github.com/qedus/nds). This API only exposes `Get`, `Put`, `Delete`, their `*Multi` equivalents and `RunInTransaction` functions as they are the only ones needed to fully make use of caching. You can carry on using `appengine/datastore` for all other datastore operations.
You can find the API documentation at [http://godoc.org/github.com/qedus/nds](http://godoc.org/github.com/qedus/nds).

One other benefit is that the standard `datastore.GetMulti` function only allows you to retrieve a maximum of 1000 entities at a time. The [`GetMulti`](http://godoc.org/github.com/qedus/nds#GetMulti) in this package allows you to get as many as you need (within timeout limits) by concurrently calling the datastore until your entity request is fulfilled.

Expand All @@ -15,4 +15,4 @@ One other benefit is that the standard `datastore.GetMulti` function only allows
You can use this package in *exactly* the same way you would use `appengine/datastore`. However, it is important not to mix usage of functions between `appengine/datastore` and `github.com/qedus/nds` within your app. You will be liable to get stale datastore entities as `github.com/qedus/nds` goes to great lengths to keep caches in sync with the datastore.

## Limitations
Currently `PropertyLoadSaver` is not supported.
`PropertyLoadSaver` is currently not implemented.
14 changes: 8 additions & 6 deletions delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,29 @@ import (

// DeleteMulti works just like datastore.DeleteMulti except it maintains
// cache consistency with other NDS methods.
func DeleteMulti(c appengine.Context, keys []*datastore.Key) error {
func DeleteMulti(c appengine.Context, keys []*Key) error {
return deleteMulti(c, keys)
}

func Delete(c appengine.Context, key *datastore.Key) error {
func Delete(c appengine.Context, key *Key) error {
if key == nil {
return errors.New("nds: key is nil")
}

err := deleteMulti(c, []*datastore.Key{key})
err := deleteMulti(c, []*Key{key})
if me, ok := err.(appengine.MultiError); ok {
return me[0]
}
return err
}

func deleteMulti(c appengine.Context, keys []*datastore.Key) error {
func deleteMulti(c appengine.Context, keys []*Key) error {
// TODO: ensure valid keys.

lockMemcacheItems := []*memcache.Item{}
for _, key := range keys {
if key.Incomplete() {
return datastore.ErrInvalidKey
return ErrInvalidKey
}

item := &memcache.Item{
Expand All @@ -46,5 +48,5 @@ func deleteMulti(c appengine.Context, keys []*datastore.Key) error {
return err
}

return datastore.DeleteMulti(c, keys)
return datastore.DeleteMulti(c, unwrapKeys(keys))
}
9 changes: 4 additions & 5 deletions delete_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package nds_test
import (
"appengine"
"appengine/aetest"
"appengine/datastore"
"github.com/qedus/nds"
"testing"
)
Expand All @@ -19,8 +18,8 @@ func TestDelete(t *testing.T) {
Val int
}

key := datastore.NewKey(c, "Entity", "", 1, nil)
keys := []*datastore.Key{key}
key := nds.NewKey(c, "Entity", "", 1, nil)
keys := []*nds.Key{key}
entities := make([]testEntity, 1)
entities[0].Val = 43

Expand All @@ -41,11 +40,11 @@ func TestDelete(t *testing.T) {
t.Fatal(err)
}

keys = []*datastore.Key{key}
keys = []*nds.Key{key}
entities = make([]testEntity, 1)
err = nds.GetMulti(c, keys, entities)
if me, ok := err.(appengine.MultiError); ok {
if me[0] != datastore.ErrNoSuchEntity {
if me[0] != nds.ErrNoSuchEntity {
t.Fatal("entity should be deleted", entities)
}
} else {
Expand Down
25 changes: 12 additions & 13 deletions get.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ const getMultiLimit = 1000
//
// vals currently only takes slices of structs. It does not take slices of
// pointers, interfaces or datastore.PropertyLoadSaver.
func GetMulti(c appengine.Context,
keys []*datastore.Key, vals interface{}) error {
func GetMulti(c appengine.Context, keys []*Key, vals interface{}) error {

v := reflect.ValueOf(vals)
if err := checkMultiArgs(keys, v); err != nil {
Expand Down Expand Up @@ -74,7 +73,7 @@ func GetMulti(c appengine.Context,
go func() {
if inTransaction(c) {
errs[index] = datastore.GetMulti(c,
keySlice, valSlice.Interface())
unwrapKeys(keySlice), valSlice.Interface())
} else {
errs[index] = getMulti(c, keySlice, valSlice)
}
Expand Down Expand Up @@ -110,14 +109,14 @@ func GetMulti(c appengine.Context,
return groupedErrs
}

func Get(c appengine.Context, key *datastore.Key, val interface{}) error {
func Get(c appengine.Context, key *Key, val interface{}) error {

if err := checkArgs(key, val); err != nil {
return err
}

vals := reflect.ValueOf([]interface{}{val})
err := getMulti(c, []*datastore.Key{key}, vals)
err := getMulti(c, []*Key{key}, vals)
if me, ok := err.(appengine.MultiError); ok {
return me[0]
}
Expand All @@ -134,7 +133,7 @@ const (
)

type cacheItem struct {
key *datastore.Key
key *Key
memcacheKey string

val reflect.Value
Expand All @@ -151,8 +150,7 @@ type cacheItem struct {
// function, datastore or server fails at any point. The caching strategy is
// borrowed from Python ndb with some improvements that eliminate some
// consistency issues surrounding ndb, including http://goo.gl/3ByVlA.
func getMulti(c appengine.Context, keys []*datastore.Key,
vals reflect.Value) error {
func getMulti(c appengine.Context, keys []*Key, vals reflect.Value) error {

cacheItems := make([]cacheItem, len(keys))
for i, key := range keys {
Expand Down Expand Up @@ -216,7 +214,7 @@ func loadMemcache(c appengine.Context, cacheItems []cacheItem) error {
cacheItems[i].state = externalLock
case noneItem:
cacheItems[i].state = done
cacheItems[i].err = datastore.ErrNoSuchEntity
cacheItems[i].err = ErrNoSuchEntity
case entityItem:
err := unmarshal(item.Value, cacheItems[i].val)
if err == nil {
Expand Down Expand Up @@ -286,7 +284,7 @@ func lockMemcache(c appengine.Context, cacheItems []cacheItem) error {
}
case noneItem:
cacheItems[i].state = done
cacheItems[i].err = datastore.ErrNoSuchEntity
cacheItems[i].err = ErrNoSuchEntity
case entityItem:
err := unmarshal(item.Value, cacheItems[i].val)
if err == nil {
Expand All @@ -313,7 +311,7 @@ func lockMemcache(c appengine.Context, cacheItems []cacheItem) error {
func loadDatastore(c appengine.Context, cacheItems []cacheItem,
valsType reflect.Type) error {

keys := make([]*datastore.Key, 0, len(cacheItems))
keys := make([]*Key, 0, len(cacheItems))
vals := reflect.MakeSlice(valsType, 0, len(cacheItems))
cacheItemsIndex := make([]int, 0, len(cacheItems))

Expand All @@ -327,7 +325,8 @@ func loadDatastore(c appengine.Context, cacheItems []cacheItem,
}

var me appengine.MultiError
if err := datastore.GetMulti(c, keys, vals.Interface()); err == nil {
if err := datastore.GetMulti(c, unwrapKeys(keys),
vals.Interface()); err == nil {
me = make(appengine.MultiError, len(keys))
} else if e, ok := err.(appengine.MultiError); ok {
me = e
Expand Down Expand Up @@ -356,7 +355,7 @@ func loadDatastore(c appengine.Context, cacheItems []cacheItem,
cacheItems[index].item.Expiration = 0
cacheItems[index].item.Value = []byte{}
}
cacheItems[index].err = datastore.ErrNoSuchEntity
cacheItems[index].err = ErrNoSuchEntity
default:
cacheItems[index].state = externalLock
cacheItems[index].err = me[i]
Expand Down
89 changes: 89 additions & 0 deletions key.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package nds

import (
"appengine"
"appengine/datastore"
)

type Key struct {
*datastore.Key
}

func unwrapKeys(keys []*Key) []*datastore.Key {
datastoreKeys := make([]*datastore.Key, len(keys))
for i, k := range keys {
datastoreKeys[i] = k.Key
}
return datastoreKeys
}

func wrapKeys(keys []*datastore.Key) []*Key {
ndsKeys := make([]*Key, len(keys))
for i, k := range keys {
ndsKeys[i] = &Key{k}
}
return ndsKeys
}

func NewIncompleteKey(c appengine.Context, kind string, parent *Key) *Key {
var parentKey *datastore.Key
if parent != nil {
parentKey = parent.Key
}
return &Key{datastore.NewIncompleteKey(c, kind, parentKey)}
}

func NewKey(c appengine.Context,
kind, stringID string, intID int64, parent *Key) *Key {
var parentKey *datastore.Key
if parent != nil {
parentKey = parent.Key
}
return &Key{datastore.NewKey(c, kind, stringID, intID, parentKey)}
}

func (k *Key) Parent() *Key {
return &Key{k.Key.Parent()}
}

func (k *Key) Equal(o *Key) bool {
return k.Key.Equal(o.Key)
}

func DecodeKey(encoded string) (*Key, error) {
key, err := datastore.DecodeKey(encoded)
if err != nil {
return nil, err
}
return &Key{key}, err
}

func (k *Key) GobDecode(buf []byte) error {
key := &datastore.Key{}
if err := key.GobDecode(buf); err != nil {
return err
}
k.Key = key
return nil
}

func (k *Key) UnmarshalJSON(buf []byte) error {

key := &datastore.Key{}
if err := key.UnmarshalJSON(buf); err != nil {
return err
}
k.Key = key
return nil
}

func AllocateIDs(c appengine.Context,
kind string, parent *Key, n int) (low, high int64, err error) {

var parentKey *datastore.Key
if parent != nil {
parentKey = parent.Key
}

return datastore.AllocateIDs(c, kind, parentKey, n)
}
Loading

0 comments on commit 0985407

Please sign in to comment.