From fe9ca5ffcc17f3c0737069da089182a853ad793f Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 22:01:02 +0930 Subject: [PATCH 01/24] Fix data race in gremlin timeout handling Fixes issue #95. --- query/gremlin/finals.go | 28 +++++++++++++++++++++------- query/gremlin/session.go | 24 ++++++++++++++++++------ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index 022a39496..a16e661ab 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -148,8 +148,10 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string count := 0 it, _ = it.Optimize() for { - if ses.doHalt { + select { + case <-ses.kill: return nil + default: } _, ok := graph.Next(it) if !ok { @@ -163,8 +165,10 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string break } for it.NextResult() == true { - if ses.doHalt { + select { + case <-ses.kill: return nil + default: } tags := make(map[string]graph.Value) it.TagResults(tags) @@ -184,8 +188,10 @@ func runIteratorToArrayNoTags(it graph.Iterator, ses *Session, limit int) []stri count := 0 it, _ = it.Optimize() for { - if ses.doHalt { + select { + case <-ses.kill: return nil + default: } val, ok := graph.Next(it) if !ok { @@ -205,8 +211,10 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu count := 0 it, _ = it.Optimize() for { - if ses.doHalt { + select { + case <-ses.kill: return + default: } _, ok := graph.Next(it) if !ok { @@ -221,8 +229,10 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu break } for it.NextResult() == true { - if ses.doHalt { + select { + case <-ses.kill: return + default: } tags := make(map[string]graph.Value) it.TagResults(tags) @@ -246,8 +256,10 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { glog.V(2).Infoln(it.DebugString(0)) for { // TODO(barakmich): Better halting. - if ses.doHalt { + select { + case <-ses.kill: return + default: } _, ok := graph.Next(it) if !ok { @@ -260,8 +272,10 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { break } for it.NextResult() == true { - if ses.doHalt { + select { + case <-ses.kill: return + default: } tags := make(map[string]graph.Value) it.TagResults(tags) diff --git a/query/gremlin/session.go b/query/gremlin/session.go index 00532b8a2..e100b43aa 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "sort" + "sync" "time" "github.com/robertkrimen/otto" @@ -30,6 +31,7 @@ type Session struct { ts graph.TripleStore currentChannel chan interface{} env *otto.Otto + envLock sync.Mutex debug bool limit int count int @@ -38,7 +40,7 @@ type Session struct { queryShape map[string]interface{} err error script *otto.Script - doHalt bool + kill chan struct{} timeoutSec time.Duration emptyEnv *otto.Otto } @@ -95,8 +97,10 @@ func (s *Session) SendResult(result *GremlinResult) bool { if s.limit >= 0 && s.limit == s.count { return false } - if s.doHalt { + select { + case <-s.kill: return false + default: } if s.currentChannel != nil { s.currentChannel <- result @@ -113,7 +117,7 @@ func (s *Session) SendResult(result *GremlinResult) bool { var halt = errors.New("Query Timeout") func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { - s.doHalt = false + s.kill = make(chan struct{}) defer func() { if caught := recover(); caught != nil { if caught == halt { @@ -129,7 +133,9 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { if s.timeoutSec != -1 { go func() { time.Sleep(s.timeoutSec * time.Second) // Stop after two seconds - s.doHalt = true + close(s.kill) + s.envLock.Lock() + defer s.envLock.Unlock() if s.env != nil { s.env.Interrupt <- func() { panic(halt) @@ -139,6 +145,8 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { }() } + s.envLock.Lock() + defer s.envLock.Unlock() return s.env.Run(input) // Here be dragons (risky code) } @@ -166,7 +174,9 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { } s.currentChannel = nil s.script = nil + s.envLock.Lock() s.env = s.emptyEnv + s.envLock.Unlock() return } @@ -256,10 +266,12 @@ func (ses *Session) GetJson() ([]interface{}, error) { if ses.err != nil { return nil, ses.err } - if ses.doHalt { + select { + case <-ses.kill: return nil, halt + default: + return ses.dataOutput, nil } - return ses.dataOutput, nil } func (ses *Session) ClearJson() { From 8ccf842518419c3c0fb28c6432282f63580cf729 Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 22:14:24 +0930 Subject: [PATCH 02/24] Improve error handling Export the timeout kill error and use error instead of string in result struct. --- cayley_test.go | 2 +- query/gremlin/environ.go | 2 +- query/gremlin/finals.go | 4 ++-- query/gremlin/session.go | 33 ++++++++++++++------------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/cayley_test.go b/cayley_test.go index af77610a3..55446154f 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -333,7 +333,7 @@ func TestQueries(t *testing.T) { if j == nil && err == nil { continue } - if err != nil && err.Error() == "Query Timeout" { + if err == gremlin.ErrKillTimeout { timedOut = true continue } diff --git a/query/gremlin/environ.go b/query/gremlin/environ.go index 503ad4f61..9e0059436 100644 --- a/query/gremlin/environ.go +++ b/query/gremlin/environ.go @@ -84,7 +84,7 @@ func setupGremlin(env *otto.Otto, ses *Session) { graph.Set("Emit", func(call otto.FunctionCall) otto.Value { value := call.Argument(0) if value.IsDefined() { - ses.SendResult(&GremlinResult{metaresult: false, err: "", val: &value, actualResults: nil}) + ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: &value, actualResults: nil}) } return otto.NullValue() }) diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index a16e661ab..a58f5ac1b 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -267,7 +267,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: nil, actualResults: &tags}) if !cont { break } @@ -279,7 +279,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: nil, actualResults: &tags}) if !cont { break } diff --git a/query/gremlin/session.go b/query/gremlin/session.go index e100b43aa..c0684dc31 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -66,7 +66,7 @@ func NewSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool type GremlinResult struct { metaresult bool - err string + err error val *otto.Value actualResults *map[string]graph.Value } @@ -114,17 +114,17 @@ func (s *Session) SendResult(result *GremlinResult) bool { return false } -var halt = errors.New("Query Timeout") +var ErrKillTimeout = errors.New("query timed out") func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { s.kill = make(chan struct{}) defer func() { - if caught := recover(); caught != nil { - if caught == halt { - s.err = halt + if r := recover(); r != nil { + if r == ErrKillTimeout { + s.err = ErrKillTimeout return } - panic(caught) // Something else happened, repanic! + panic(r) } }() @@ -138,7 +138,7 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { defer s.envLock.Unlock() if s.env != nil { s.env.Interrupt <- func() { - panic(halt) + panic(ErrKillTimeout) } s.env = s.emptyEnv } @@ -161,16 +161,11 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { } else { value, err = s.runUnsafe(s.script) } - if err != nil { - out <- &GremlinResult{metaresult: true, - err: err.Error(), - val: &value, - actualResults: nil} - } else { - out <- &GremlinResult{metaresult: true, - err: "", - val: &value, - actualResults: nil} + out <- &GremlinResult{ + metaresult: true, + err: err, + val: &value, + actualResults: nil, } s.currentChannel = nil s.script = nil @@ -183,7 +178,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { func (s *Session) ToText(result interface{}) string { data := result.(*GremlinResult) if data.metaresult { - if data.err != "" { + if data.err != nil { return fmt.Sprintln("Error: ", data.err) } if data.val != nil { @@ -268,7 +263,7 @@ func (ses *Session) GetJson() ([]interface{}, error) { } select { case <-ses.kill: - return nil, halt + return nil, ErrKillTimeout default: return ses.dataOutput, nil } From 15a45ef0d4b8a2322866a871ffc6ac6f8c499090 Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 22:45:19 +0930 Subject: [PATCH 03/24] Clean up style --- query/gremlin/environ.go | 2 +- query/gremlin/finals.go | 10 ++-- query/gremlin/session.go | 116 ++++++++++++++++++--------------------- 3 files changed, 59 insertions(+), 69 deletions(-) diff --git a/query/gremlin/environ.go b/query/gremlin/environ.go index 9e0059436..670b6b6df 100644 --- a/query/gremlin/environ.go +++ b/query/gremlin/environ.go @@ -84,7 +84,7 @@ func setupGremlin(env *otto.Otto, ses *Session) { graph.Set("Emit", func(call otto.FunctionCall) otto.Value { value := call.Argument(0) if value.IsDefined() { - ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: &value, actualResults: nil}) + ses.SendResult(&GremlinResult{val: &value}) } return otto.NullValue() }) diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index a58f5ac1b..665850262 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -248,8 +248,8 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu } func runIteratorOnSession(it graph.Iterator, ses *Session) { - if ses.lookingForQueryShape { - iterator.OutputQueryShapeForIterator(it, ses.ts, ses.queryShape) + if ses.wantShape { + iterator.OutputQueryShapeForIterator(it, ses.ts, ses.shape) return } it, _ = it.Optimize() @@ -267,8 +267,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: nil, actualResults: &tags}) - if !cont { + if !ses.SendResult(&GremlinResult{actualResults: &tags}) { break } for it.NextResult() == true { @@ -279,8 +278,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: nil, val: nil, actualResults: &tags}) - if !cont { + if !ses.SendResult(&GremlinResult{actualResults: &tags}) { break } } diff --git a/query/gremlin/session.go b/query/gremlin/session.go index c0684dc31..8223c3723 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -27,40 +27,36 @@ import ( "github.com/google/cayley/query" ) +var ErrKillTimeout = errors.New("query timed out") + type Session struct { - ts graph.TripleStore - currentChannel chan interface{} - env *otto.Otto - envLock sync.Mutex - debug bool - limit int - count int - dataOutput []interface{} - lookingForQueryShape bool - queryShape map[string]interface{} - err error - script *otto.Script - kill chan struct{} - timeoutSec time.Duration - emptyEnv *otto.Otto + ts graph.TripleStore + results chan interface{} + env *otto.Otto + envLock sync.Mutex + debug bool + limit int + count int + dataOutput []interface{} + wantShape bool + shape map[string]interface{} + err error + script *otto.Script + kill chan struct{} + timeoutSec time.Duration + emptyEnv *otto.Otto } func NewSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *Session { - var g Session - g.ts = inputTripleStore + g := Session{ + ts: inputTripleStore, + limit: -1, + timeoutSec: time.Duration(timeoutSec), + } g.env = BuildEnviron(&g) - g.limit = -1 - g.count = 0 - g.lookingForQueryShape = false if persist { g.emptyEnv = g.env } - if timeoutSec < 0 { - g.timeoutSec = time.Duration(-1) - } else { - g.timeoutSec = time.Duration(timeoutSec) - } - g.ClearJson() return &g } @@ -75,13 +71,13 @@ func (s *Session) ToggleDebug() { s.debug = !s.debug } -func (s *Session) GetQuery(input string, output_struct chan map[string]interface{}) { - defer close(output_struct) - s.queryShape = make(map[string]interface{}) - s.lookingForQueryShape = true +func (s *Session) GetQuery(input string, out chan map[string]interface{}) { + defer close(out) + s.shape = make(map[string]interface{}) + s.wantShape = true s.env.Run(input) - output_struct <- s.queryShape - s.queryShape = nil + out <- s.shape + s.shape = nil } func (s *Session) InputParses(input string) (query.ParseResult, error) { @@ -102,8 +98,8 @@ func (s *Session) SendResult(result *GremlinResult) bool { return false default: } - if s.currentChannel != nil { - s.currentChannel <- result + if s.results != nil { + s.results <- result s.count++ if s.limit >= 0 && s.limit == s.count { return false @@ -114,8 +110,6 @@ func (s *Session) SendResult(result *GremlinResult) bool { return false } -var ErrKillTimeout = errors.New("query timed out") - func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { s.kill = make(chan struct{}) defer func() { @@ -128,11 +122,12 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { } }() - s.env.Interrupt = make(chan func(), 1) // The buffer prevents blocking + // Use buffered chan to prevent blocking. + s.env.Interrupt = make(chan func(), 1) - if s.timeoutSec != -1 { + if s.timeoutSec >= 0 { go func() { - time.Sleep(s.timeoutSec * time.Second) // Stop after two seconds + time.Sleep(s.timeoutSec * time.Second) close(s.kill) s.envLock.Lock() defer s.envLock.Unlock() @@ -147,13 +142,13 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { s.envLock.Lock() defer s.envLock.Unlock() - return s.env.Run(input) // Here be dragons (risky code) + return s.env.Run(input) } func (s *Session) ExecInput(input string, out chan interface{}, limit int) { defer close(out) s.err = nil - s.currentChannel = out + s.results = out var err error var value otto.Value if s.script == nil { @@ -162,24 +157,22 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { value, err = s.runUnsafe(s.script) } out <- &GremlinResult{ - metaresult: true, - err: err, - val: &value, - actualResults: nil, + metaresult: true, + err: err, + val: &value, } - s.currentChannel = nil + s.results = nil s.script = nil s.envLock.Lock() s.env = s.emptyEnv s.envLock.Unlock() - return } func (s *Session) ToText(result interface{}) string { data := result.(*GremlinResult) if data.metaresult { if data.err != nil { - return fmt.Sprintln("Error: ", data.err) + return fmt.Sprintf("Error: %v\n", data.err) } if data.val != nil { s, _ := data.val.Export() @@ -226,7 +219,7 @@ func (s *Session) ToText(result interface{}) string { } // Web stuff -func (ses *Session) BuildJson(result interface{}) { +func (s *Session) BuildJson(result interface{}) { data := result.(*GremlinResult) if !data.metaresult { if data.val == nil { @@ -240,35 +233,34 @@ func (ses *Session) BuildJson(result interface{}) { } sort.Strings(tagKeys) for _, k := range tagKeys { - obj[k] = ses.ts.NameOf((*tags)[k]) + obj[k] = s.ts.NameOf((*tags)[k]) } - ses.dataOutput = append(ses.dataOutput, obj) + s.dataOutput = append(s.dataOutput, obj) } else { if data.val.IsObject() { export, _ := data.val.Export() - ses.dataOutput = append(ses.dataOutput, export) + s.dataOutput = append(s.dataOutput, export) } else { strVersion, _ := data.val.ToString() - ses.dataOutput = append(ses.dataOutput, strVersion) + s.dataOutput = append(s.dataOutput, strVersion) } } } - } -func (ses *Session) GetJson() ([]interface{}, error) { - defer ses.ClearJson() - if ses.err != nil { - return nil, ses.err +func (s *Session) GetJson() ([]interface{}, error) { + defer s.ClearJson() + if s.err != nil { + return nil, s.err } select { - case <-ses.kill: + case <-s.kill: return nil, ErrKillTimeout default: - return ses.dataOutput, nil + return s.dataOutput, nil } } -func (ses *Session) ClearJson() { - ses.dataOutput = nil +func (s *Session) ClearJson() { + s.dataOutput = nil } From bf3ac2be9eb763d827c678cfd04c4874cbaaee15 Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 22:49:57 +0930 Subject: [PATCH 04/24] Destutter gremlin.GremlinResult --- query/gremlin/environ.go | 2 +- query/gremlin/finals.go | 4 ++-- query/gremlin/gremlin_test.go | 5 +++-- query/gremlin/session.go | 12 ++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/query/gremlin/environ.go b/query/gremlin/environ.go index 670b6b6df..7918a2156 100644 --- a/query/gremlin/environ.go +++ b/query/gremlin/environ.go @@ -84,7 +84,7 @@ func setupGremlin(env *otto.Otto, ses *Session) { graph.Set("Emit", func(call otto.FunctionCall) otto.Value { value := call.Argument(0) if value.IsDefined() { - ses.SendResult(&GremlinResult{val: &value}) + ses.SendResult(&Result{val: &value}) } return otto.NullValue() }) diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index 665850262..42998fa61 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -267,7 +267,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - if !ses.SendResult(&GremlinResult{actualResults: &tags}) { + if !ses.SendResult(&Result{actualResults: &tags}) { break } for it.NextResult() == true { @@ -278,7 +278,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } tags := make(map[string]graph.Value) it.TagResults(tags) - if !ses.SendResult(&GremlinResult{actualResults: &tags}) { + if !ses.SendResult(&Result{actualResults: &tags}) { break } } diff --git a/query/gremlin/gremlin_test.go b/query/gremlin/gremlin_test.go index 453ff812b..dbd115a10 100644 --- a/query/gremlin/gremlin_test.go +++ b/query/gremlin/gremlin_test.go @@ -20,8 +20,9 @@ import ( "testing" "github.com/google/cayley/graph" - _ "github.com/google/cayley/graph/memstore" "github.com/google/cayley/quad" + + _ "github.com/google/cayley/graph/memstore" ) // This is a simple test graph. @@ -252,7 +253,7 @@ func runQueryGetTag(g []*quad.Quad, query string, tag string) []string { var results []string for res := range c { - data := res.(*GremlinResult) + data := res.(*Result) if data.val == nil { val := (*data.actualResults)[tag] if val != nil { diff --git a/query/gremlin/session.go b/query/gremlin/session.go index 8223c3723..e5b346021 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -60,7 +60,7 @@ func NewSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool return &g } -type GremlinResult struct { +type Result struct { metaresult bool err error val *otto.Value @@ -89,7 +89,7 @@ func (s *Session) InputParses(input string) (query.ParseResult, error) { return query.Parsed, nil } -func (s *Session) SendResult(result *GremlinResult) bool { +func (s *Session) SendResult(r *Result) bool { if s.limit >= 0 && s.limit == s.count { return false } @@ -99,7 +99,7 @@ func (s *Session) SendResult(result *GremlinResult) bool { default: } if s.results != nil { - s.results <- result + s.results <- r s.count++ if s.limit >= 0 && s.limit == s.count { return false @@ -156,7 +156,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { } else { value, err = s.runUnsafe(s.script) } - out <- &GremlinResult{ + out <- &Result{ metaresult: true, err: err, val: &value, @@ -169,7 +169,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { } func (s *Session) ToText(result interface{}) string { - data := result.(*GremlinResult) + data := result.(*Result) if data.metaresult { if data.err != nil { return fmt.Sprintf("Error: %v\n", data.err) @@ -220,7 +220,7 @@ func (s *Session) ToText(result interface{}) string { // Web stuff func (s *Session) BuildJson(result interface{}) { - data := result.(*GremlinResult) + data := result.(*Result) if !data.metaresult { if data.val == nil { obj := make(map[string]string) From 2d884f92e9207f78ec3e309506e47701df07509f Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 22:58:53 +0930 Subject: [PATCH 05/24] Name reduction --- query/gremlin/session.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query/gremlin/session.go b/query/gremlin/session.go index e5b346021..3f8bb0bb4 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -47,9 +47,9 @@ type Session struct { emptyEnv *otto.Otto } -func NewSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *Session { +func NewSession(ts graph.TripleStore, timeoutSec int, persist bool) *Session { g := Session{ - ts: inputTripleStore, + ts: ts, limit: -1, timeoutSec: time.Duration(timeoutSec), } From 0fedecd392cf51e5ded5029e32bc6c0a5e28765e Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 23:15:41 +0930 Subject: [PATCH 06/24] Use time.Duration according to the time docs Having a time.Duration measuring seconds is likely to cause problems at a later stage. This change retains configuration compatibility while adding idiomatic duration use. --- cayley_test.go | 3 +- config/config.go | 99 ++++++++++++++++++++++++++++++++++++---- docs/Configuration.md | 4 +- query/gremlin/session.go | 14 +++--- 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/cayley_test.go b/cayley_test.go index 55446154f..f0e85b7dd 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -17,6 +17,7 @@ package main import ( "sync" "testing" + "time" "github.com/google/cayley/config" "github.com/google/cayley/db" @@ -294,7 +295,7 @@ var ( cfg = &config.Config{ DatabasePath: "30kmoviedata.nq.gz", DatabaseType: "memstore", - GremlinTimeout: 300, + GremlinTimeout: 300 * time.Second, } ts graph.TripleStore diff --git a/config/config.go b/config/config.go index d52104356..04867d81e 100644 --- a/config/config.go +++ b/config/config.go @@ -17,29 +17,112 @@ package config import ( "encoding/json" "flag" + "fmt" "os" + "strconv" + "time" "github.com/barakmich/glog" ) type Config struct { + DatabaseType string + DatabasePath string + DatabaseOptions map[string]interface{} + ListenHost string + ListenPort string + ReadOnly bool + GremlinTimeout time.Duration + LoadSize int +} + +type config struct { DatabaseType string `json:"database"` DatabasePath string `json:"db_path"` DatabaseOptions map[string]interface{} `json:"db_options"` ListenHost string `json:"listen_host"` ListenPort string `json:"listen_port"` ReadOnly bool `json:"read_only"` - GremlinTimeout int `json:"gremlin_timeout"` + GremlinTimeout duration `json:"gremlin_timeout"` LoadSize int `json:"load_size"` } -var databasePath = flag.String("dbpath", "/tmp/testdb", "Path to the database.") -var databaseBackend = flag.String("db", "memstore", "Database Backend.") -var host = flag.String("host", "0.0.0.0", "Host to listen on (defaults to all).") -var loadSize = flag.Int("load_size", 10000, "Size of triplesets to load") -var port = flag.String("port", "64210", "Port to listen on.") -var readOnly = flag.Bool("read_only", false, "Disable writing via HTTP.") -var gremlinTimeout = flag.Int("gremlin_timeout", 30, "Number of seconds until an individual query times out.") +func (c *Config) UnmarshalJSON(data []byte) error { + var t config + err := json.Unmarshal(data, &t) + if err != nil { + return err + } + *c = Config{ + DatabaseType: t.DatabaseType, + DatabasePath: t.DatabasePath, + DatabaseOptions: t.DatabaseOptions, + ListenHost: t.ListenHost, + ListenPort: t.ListenPort, + ReadOnly: t.ReadOnly, + GremlinTimeout: time.Duration(t.GremlinTimeout), + LoadSize: t.LoadSize, + } + return nil +} + +func (c *Config) MarshalJSON() ([]byte, error) { + return json.Marshal(config{ + DatabaseType: c.DatabaseType, + DatabasePath: c.DatabasePath, + DatabaseOptions: c.DatabaseOptions, + ListenHost: c.ListenHost, + ListenPort: c.ListenPort, + ReadOnly: c.ReadOnly, + GremlinTimeout: duration(c.GremlinTimeout), + LoadSize: c.LoadSize, + }) +} + +// duration is a time.Duration that satisfies the +// json.UnMarshaler and json.Marshaler interfaces. +type duration time.Duration + +// UnmarshalJSON unmarshals a duration according to the following scheme: +// * If the element is absent the duration is zero. +// * If the element is parsable as a time.Duration, the parsed value is kept. +// * If the element is parsable as a number, that number of seconds is kept. +func (d *duration) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + *d = 0 + return nil + } + text := string(data) + t, err := time.ParseDuration(text) + if err == nil { + *d = duration(t) + return nil + } + i, err := strconv.ParseInt(text, 10, 64) + if err == nil { + *d = duration(time.Duration(i) * time.Second) + return nil + } + // This hack is to get around strconv.ParseFloat + // not handling e-notation for integers. + f, err := strconv.ParseFloat(text, 64) + *d = duration(time.Duration(f) * time.Second) + return err +} + +func (d *duration) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", *d)), nil +} + +var ( + databasePath = flag.String("dbpath", "/tmp/testdb", "Path to the database.") + databaseBackend = flag.String("db", "memstore", "Database Backend.") + host = flag.String("host", "0.0.0.0", "Host to listen on (defaults to all).") + loadSize = flag.Int("load_size", 10000, "Size of triplesets to load") + port = flag.String("port", "64210", "Port to listen on.") + readOnly = flag.Bool("read_only", false, "Disable writing via HTTP.") + gremlinTimeout = flag.Duration("gremlin_timeout", 30*time.Second, "Elapsed time until an individual query times out.") +) func ParseConfigFromFile(filename string) *Config { config := &Config{} diff --git a/docs/Configuration.md b/docs/Configuration.md index 792b47633..37b766906 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -74,10 +74,10 @@ All command line flags take precedence over the configuration file. #### **`gremlin_timeout`** - * Type: Integer + * Type: Integer or String * Default: 30 -The value in seconds of the maximum length of time the Javascript runtime should run until cancelling the query and returning a 408 Timeout. A negative value means no limit. +The maximum length of time the Javascript runtime should run until cancelling the query and returning a 408 Timeout. When gremlin_timeout is an integer is is interpretted as seconds, when it is a string it is [parsed](http://golang.org/pkg/time/#ParseDuration) as a Go time.Duration. A negative duration means no limit. ## Per-Database Options diff --git a/query/gremlin/session.go b/query/gremlin/session.go index 3f8bb0bb4..91c509a2f 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -43,15 +43,15 @@ type Session struct { err error script *otto.Script kill chan struct{} - timeoutSec time.Duration + timeout time.Duration emptyEnv *otto.Otto } -func NewSession(ts graph.TripleStore, timeoutSec int, persist bool) *Session { +func NewSession(ts graph.TripleStore, timeout time.Duration, persist bool) *Session { g := Session{ - ts: ts, - limit: -1, - timeoutSec: time.Duration(timeoutSec), + ts: ts, + limit: -1, + timeout: timeout, } g.env = BuildEnviron(&g) if persist { @@ -125,9 +125,9 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { // Use buffered chan to prevent blocking. s.env.Interrupt = make(chan func(), 1) - if s.timeoutSec >= 0 { + if s.timeout >= 0 { go func() { - time.Sleep(s.timeoutSec * time.Second) + time.Sleep(s.timeout) close(s.kill) s.envLock.Lock() defer s.envLock.Unlock() From ffb52af00b392f422bf22f85a0f89033190a5be1 Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 2 Aug 2014 23:23:43 +0930 Subject: [PATCH 07/24] Rename GremlinTimeout -> Timeout Given that there may be other Turing complete query interfaces (particularly a Go query API), the timeout config should not be specifically tied to gremlin. --- cayley_appengine.cfg | 2 +- cayley_test.go | 10 +++++----- config/config.go | 14 +++++++------- db/repl.go | 2 +- docs/Configuration.md | 4 ++-- http/query.go | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cayley_appengine.cfg b/cayley_appengine.cfg index 2ecd797d5..a762b5f0e 100644 --- a/cayley_appengine.cfg +++ b/cayley_appengine.cfg @@ -3,5 +3,5 @@ "db_path": "30kmoviedata.nq.gz", "read_only": true, "load_size": 10000, -"gremlin_timeout": 10 +"timeout": 10 } diff --git a/cayley_test.go b/cayley_test.go index f0e85b7dd..9bda12a3b 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -293,9 +293,9 @@ var m2_actors = movie2.Save("name","movie2").Follow(filmToActor) var ( once sync.Once cfg = &config.Config{ - DatabasePath: "30kmoviedata.nq.gz", - DatabaseType: "memstore", - GremlinTimeout: 300 * time.Second, + DatabasePath: "30kmoviedata.nq.gz", + DatabaseType: "memstore", + Timeout: 300 * time.Second, } ts graph.TripleStore @@ -317,7 +317,7 @@ func TestQueries(t *testing.T) { if testing.Short() && test.long { continue } - ses := gremlin.NewSession(ts, cfg.GremlinTimeout, true) + ses := gremlin.NewSession(ts, cfg.Timeout, true) _, err := ses.InputParses(test.query) if err != nil { t.Fatalf("Failed to parse benchmark gremlin %s: %v", test.message, err) @@ -358,7 +358,7 @@ func runBench(n int, b *testing.B) { b.Skip() } prepare(b) - ses := gremlin.NewSession(ts, cfg.GremlinTimeout, true) + ses := gremlin.NewSession(ts, cfg.Timeout, true) _, err := ses.InputParses(benchmarkQueries[n].query) if err != nil { b.Fatalf("Failed to parse benchmark gremlin %s: %v", benchmarkQueries[n].message, err) diff --git a/config/config.go b/config/config.go index 04867d81e..d8cb4d88d 100644 --- a/config/config.go +++ b/config/config.go @@ -32,7 +32,7 @@ type Config struct { ListenHost string ListenPort string ReadOnly bool - GremlinTimeout time.Duration + Timeout time.Duration LoadSize int } @@ -43,7 +43,7 @@ type config struct { ListenHost string `json:"listen_host"` ListenPort string `json:"listen_port"` ReadOnly bool `json:"read_only"` - GremlinTimeout duration `json:"gremlin_timeout"` + Timeout duration `json:"timeout"` LoadSize int `json:"load_size"` } @@ -60,7 +60,7 @@ func (c *Config) UnmarshalJSON(data []byte) error { ListenHost: t.ListenHost, ListenPort: t.ListenPort, ReadOnly: t.ReadOnly, - GremlinTimeout: time.Duration(t.GremlinTimeout), + Timeout: time.Duration(t.Timeout), LoadSize: t.LoadSize, } return nil @@ -74,7 +74,7 @@ func (c *Config) MarshalJSON() ([]byte, error) { ListenHost: c.ListenHost, ListenPort: c.ListenPort, ReadOnly: c.ReadOnly, - GremlinTimeout: duration(c.GremlinTimeout), + Timeout: duration(c.Timeout), LoadSize: c.LoadSize, }) } @@ -121,7 +121,7 @@ var ( loadSize = flag.Int("load_size", 10000, "Size of triplesets to load") port = flag.String("port", "64210", "Port to listen on.") readOnly = flag.Bool("read_only", false, "Disable writing via HTTP.") - gremlinTimeout = flag.Duration("gremlin_timeout", 30*time.Second, "Elapsed time until an individual query times out.") + timeout = flag.Duration("timeout", 30*time.Second, "Elapsed time until an individual query times out.") ) func ParseConfigFromFile(filename string) *Config { @@ -183,8 +183,8 @@ func ParseConfigFromFlagsAndFile(fileFlag string) *Config { config.ListenPort = *port } - if config.GremlinTimeout == 0 { - config.GremlinTimeout = *gremlinTimeout + if config.Timeout == 0 { + config.Timeout = *timeout } if config.LoadSize == 0 { diff --git a/db/repl.go b/db/repl.go index 2ef9429f4..c1bb5eaeb 100644 --- a/db/repl.go +++ b/db/repl.go @@ -72,7 +72,7 @@ func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error case "gremlin": fallthrough default: - ses = gremlin.NewSession(ts, cfg.GremlinTimeout, true) + ses = gremlin.NewSession(ts, cfg.Timeout, true) } buf := bufio.NewReader(os.Stdin) var line []byte diff --git a/docs/Configuration.md b/docs/Configuration.md index 37b766906..b464adbeb 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -72,12 +72,12 @@ All command line flags take precedence over the configuration file. ## Language Options -#### **`gremlin_timeout`** +#### **`timeout`** * Type: Integer or String * Default: 30 -The maximum length of time the Javascript runtime should run until cancelling the query and returning a 408 Timeout. When gremlin_timeout is an integer is is interpretted as seconds, when it is a string it is [parsed](http://golang.org/pkg/time/#ParseDuration) as a Go time.Duration. A negative duration means no limit. +The maximum length of time the Javascript runtime should run until cancelling the query and returning a 408 Timeout. When timeout is an integer is is interpretted as seconds, when it is a string it is [parsed](http://golang.org/pkg/time/#ParseDuration) as a Go time.Duration. A negative duration means no limit. ## Per-Database Options diff --git a/http/query.go b/http/query.go index e8b5d725f..f4f5a340b 100644 --- a/http/query.go +++ b/http/query.go @@ -71,7 +71,7 @@ func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params http var ses query.HttpSession switch params.ByName("query_lang") { case "gremlin": - ses = gremlin.NewSession(api.ts, api.config.GremlinTimeout, false) + ses = gremlin.NewSession(api.ts, api.config.Timeout, false) case "mql": ses = mql.NewSession(api.ts) default: @@ -119,7 +119,7 @@ func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params http var ses query.HttpSession switch params.ByName("query_lang") { case "gremlin": - ses = gremlin.NewSession(api.ts, api.config.GremlinTimeout, false) + ses = gremlin.NewSession(api.ts, api.config.Timeout, false) case "mql": ses = mql.NewSession(api.ts) default: From 65136855203bd5865fffbdcc32c25968e18229d5 Mon Sep 17 00:00:00 2001 From: kortschak Date: Mon, 4 Aug 2014 12:32:43 +0930 Subject: [PATCH 08/24] Fix lock contention --- query/gremlin/finals.go | 1 - query/gremlin/session.go | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index 42998fa61..916cb2603 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -255,7 +255,6 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { it, _ = it.Optimize() glog.V(2).Infoln(it.DebugString(0)) for { - // TODO(barakmich): Better halting. select { case <-ses.kill: return diff --git a/query/gremlin/session.go b/query/gremlin/session.go index 91c509a2f..8000c17d5 100644 --- a/query/gremlin/session.go +++ b/query/gremlin/session.go @@ -141,8 +141,9 @@ func (s *Session) runUnsafe(input interface{}) (otto.Value, error) { } s.envLock.Lock() - defer s.envLock.Unlock() - return s.env.Run(input) + env := s.env + s.envLock.Unlock() + return env.Run(input) } func (s *Session) ExecInput(input string, out chan interface{}, limit int) { From 1ae81e6d00aacf0f560a6aadeb06e761e83abb9d Mon Sep 17 00:00:00 2001 From: kortschak Date: Tue, 5 Aug 2014 22:37:08 +0930 Subject: [PATCH 09/24] Fix typos missed --- cayley_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cayley_test.go b/cayley_test.go index 9bda12a3b..b94389a5b 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -392,18 +392,18 @@ func BenchmarkNetAndSpeed(b *testing.B) { runBench(4, b) } -func BenchmarkKeannuAndNet(b *testing.B) { +func BenchmarkKeanuAndNet(b *testing.B) { runBench(5, b) } -func BenchmarkKeannuAndSpeed(b *testing.B) { +func BenchmarkKeanuAndSpeed(b *testing.B) { runBench(6, b) } -func BenchmarkKeannuOther(b *testing.B) { +func BenchmarkKeanuOther(b *testing.B) { runBench(7, b) } -func BenchmarkKeannuBullockOther(b *testing.B) { +func BenchmarkKeanuBullockOther(b *testing.B) { runBench(8, b) } From 6acfdcc5d6b9259e180cd576e0fa3e95a05bfd13 Mon Sep 17 00:00:00 2001 From: kortschak Date: Tue, 5 Aug 2014 22:41:25 +0930 Subject: [PATCH 10/24] Use concrete value for quad.Quad Comparison of -short benchmarks in cayley. $ benchcmp pointer.bench concrete.bench benchmark old ns/op new ns/op delta BenchmarkNamePredicate 1673276 1655093 -1.09% BenchmarkLargeSetsNoIntersection 318985907 261499984 -18.02% BenchmarkNetAndSpeed 104403743 41516981 -60.23% BenchmarkKeanuAndNet 17309258 16857513 -2.61% BenchmarkKeanuAndSpeed 20159161 19282833 -4.35% Comparison of pathological cases are not so happy. benchmark old ns/op new ns/op delta BenchmarkVeryLargeSetsSmallIntersection 55269775527 246084606672 +345.24% BenchmarkHelplessContainsChecker 23436501319 24308906949 +3.72% Profiling the worst case: Pointer: Total: 6121 samples 1973 32.2% 32.2% 1973 32.2% runtime.findfunc 773 12.6% 44.9% 773 12.6% readvarint 510 8.3% 53.2% 511 8.3% step 409 6.7% 59.9% 410 6.7% runtime.gentraceback 390 6.4% 66.2% 391 6.4% pcvalue 215 3.5% 69.8% 215 3.5% runtime.funcdata 181 3.0% 72.7% 181 3.0% checkframecopy 118 1.9% 74.6% 119 1.9% runtime.funcspdelta 96 1.6% 76.2% 96 1.6% runtime.topofstack 76 1.2% 77.5% 76 1.2% scanblock Concrete: Total: 25027 samples 9437 37.7% 37.7% 9437 37.7% runtime.findfunc 3853 15.4% 53.1% 3853 15.4% readvarint 2366 9.5% 62.6% 2366 9.5% step 2186 8.7% 71.3% 2186 8.7% runtime.gentraceback 1816 7.3% 78.5% 1816 7.3% pcvalue 1016 4.1% 82.6% 1016 4.1% runtime.funcdata 859 3.4% 86.0% 859 3.4% checkframecopy 506 2.0% 88.1% 506 2.0% runtime.funcspdelta 410 1.6% 89.7% 410 1.6% runtime.topofstack 303 1.2% 90.9% 303 1.2% runtime.newstack --- db/load.go | 2 +- db/repl.go | 4 +- graph/iterator/mock_ts_test.go | 8 +-- graph/leveldb/leveldb_test.go | 26 ++++---- graph/leveldb/triplestore.go | 26 ++++---- graph/memstore/triplestore.go | 16 ++--- graph/memstore/triplestore_test.go | 6 +- graph/mongo/triplestore.go | 14 ++-- graph/triplestore.go | 8 +-- http/http_test.go | 8 +-- http/write.go | 6 +- quad/cquads/cquads.go | 12 ++-- quad/cquads/cquads_test.go | 102 ++++++++++++++--------------- quad/nquads/nquads.go | 12 ++-- quad/nquads/nquads_test.go | 72 ++++++++++---------- quad/quad.go | 14 ++-- query/gremlin/gremlin_test.go | 6 +- query/mql/mql_test.go | 6 +- query/sexp/parser_test.go | 18 ++--- 19 files changed, 183 insertions(+), 183 deletions(-) diff --git a/db/load.go b/db/load.go index 20953a708..9a3b06962 100644 --- a/db/load.go +++ b/db/load.go @@ -57,7 +57,7 @@ func Load(ts graph.TripleStore, cfg *config.Config, path string) error { return err } - block := make([]*quad.Quad, 0, cfg.LoadSize) + block := make([]quad.Quad, 0, cfg.LoadSize) for { t, err := dec.Unmarshal() if err != nil { diff --git a/db/repl.go b/db/repl.go index c1bb5eaeb..730d59d20 100644 --- a/db/repl.go +++ b/db/repl.go @@ -114,7 +114,7 @@ func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error if bytes.HasPrefix(line, []byte(":a")) { var tripleStmt = line[3:] triple, err := cquads.Parse(string(tripleStmt)) - if triple == nil { + if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } @@ -128,7 +128,7 @@ func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error if bytes.HasPrefix(line, []byte(":d")) { var tripleStmt = line[3:] triple, err := cquads.Parse(string(tripleStmt)) - if triple == nil { + if !triple.IsValid() { if err != nil { fmt.Printf("not a valid triple: %v\n", err) } diff --git a/graph/iterator/mock_ts_test.go b/graph/iterator/mock_ts_test.go index 7332a8fc2..08483c7c2 100644 --- a/graph/iterator/mock_ts_test.go +++ b/graph/iterator/mock_ts_test.go @@ -36,11 +36,11 @@ func (qs *store) ValueOf(s string) graph.Value { return nil } -func (qs *store) AddTriple(*quad.Quad) {} +func (qs *store) AddTriple(quad.Quad) {} -func (qs *store) AddTripleSet([]*quad.Quad) {} +func (qs *store) AddTripleSet([]quad.Quad) {} -func (qs *store) Quad(graph.Value) *quad.Quad { return &quad.Quad{} } +func (qs *store) Quad(graph.Value) quad.Quad { return quad.Quad{} } func (qs *store) TripleIterator(d quad.Direction, i graph.Value) graph.Iterator { return qs.iter @@ -74,4 +74,4 @@ func (qs *store) Close() {} func (qs *store) TripleDirection(graph.Value, quad.Direction) graph.Value { return 0 } -func (qs *store) RemoveTriple(t *quad.Quad) {} +func (qs *store) RemoveTriple(t quad.Quad) {} diff --git a/graph/leveldb/leveldb_test.go b/graph/leveldb/leveldb_test.go index 0e6772a1b..956a5e4b6 100644 --- a/graph/leveldb/leveldb_test.go +++ b/graph/leveldb/leveldb_test.go @@ -26,8 +26,8 @@ import ( "github.com/google/cayley/quad" ) -func makeTripleSet() []*quad.Quad { - tripleSet := []*quad.Quad{ +func makeTripleSet() []quad.Quad { + tripleSet := []quad.Quad{ {"A", "follows", "B", ""}, {"C", "follows", "B", ""}, {"C", "follows", "D", ""}, @@ -43,7 +43,7 @@ func makeTripleSet() []*quad.Quad { return tripleSet } -func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad { +func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []quad.Quad { var res ordered for { val, ok := graph.Next(it) @@ -56,7 +56,7 @@ func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad { return res } -type ordered []*quad.Quad +type ordered []quad.Quad func (o ordered) Len() int { return len(o) } func (o ordered) Less(i, j int) bool { @@ -143,7 +143,7 @@ func TestLoadDatabase(t *testing.T) { t.Error("Failed to create leveldb TripleStore.") } - qs.AddTriple(&quad.Quad{"Something", "points_to", "Something Else", "context"}) + qs.AddTriple(quad.Quad{"Something", "points_to", "Something Else", "context"}) for _, pq := range []string{"Something", "points_to", "Something Else", "context"} { if got := qs.NameOf(qs.ValueOf(pq)); got != pq { t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq) @@ -176,7 +176,7 @@ func TestLoadDatabase(t *testing.T) { t.Errorf("Unexpected triplestore size, got:%d expect:5", s) } - qs.RemoveTriple(&quad.Quad{"A", "follows", "B", ""}) + qs.RemoveTriple(quad.Quad{"A", "follows", "B", ""}) if s := qs.Size(); s != 10 { t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s) } @@ -301,7 +301,7 @@ func TestSetIterator(t *testing.T) { qs.AddTripleSet(makeTripleSet()) - expect := []*quad.Quad{ + expect := []quad.Quad{ {"C", "follows", "B", ""}, {"C", "follows", "D", ""}, } @@ -326,7 +326,7 @@ func TestSetIterator(t *testing.T) { // Object iterator. it = qs.TripleIterator(quad.Object, qs.ValueOf("F")) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "follows", "F", ""}, {"E", "follows", "F", ""}, } @@ -339,7 +339,7 @@ func TestSetIterator(t *testing.T) { and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B"))) and.AddSubIterator(it) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "follows", "F", ""}, } if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) { @@ -349,7 +349,7 @@ func TestSetIterator(t *testing.T) { // Predicate iterator. it = qs.TripleIterator(quad.Predicate, qs.ValueOf("status")) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "status", "cool", "status_graph"}, {"D", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"}, @@ -362,7 +362,7 @@ func TestSetIterator(t *testing.T) { // Label iterator. it = qs.TripleIterator(quad.Label, qs.ValueOf("status_graph")) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "status", "cool", "status_graph"}, {"D", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"}, @@ -378,7 +378,7 @@ func TestSetIterator(t *testing.T) { and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B"))) and.AddSubIterator(it) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "status", "cool", "status_graph"}, } if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) { @@ -391,7 +391,7 @@ func TestSetIterator(t *testing.T) { and.AddSubIterator(it) and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B"))) - expect = []*quad.Quad{ + expect = []quad.Quad{ {"B", "status", "cool", "status_graph"}, } if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) { diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 8092d9685..600b36077 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -116,7 +116,7 @@ func (qs *TripleStore) Size() int64 { return qs.size } -func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte { +func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte { key := make([]byte, 0, 2+(qs.hasher.Size()*3)) // TODO(kortschak) Remove dependence on String() method. key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...) @@ -126,7 +126,7 @@ func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []by return key } -func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte { +func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte { key := make([]byte, 0, 2+(qs.hasher.Size()*4)) // TODO(kortschak) Remove dependence on String() method. key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...) @@ -144,7 +144,7 @@ func (qs *TripleStore) createValueKeyFor(s string) []byte { return key } -func (qs *TripleStore) AddTriple(t *quad.Quad) { +func (qs *TripleStore) AddTriple(t quad.Quad) { batch := &leveldb.Batch{} qs.buildWrite(batch, t) err := qs.db.Write(batch, qs.writeopts) @@ -163,7 +163,7 @@ var ( pso = [3]quad.Direction{quad.Predicate, quad.Subject, quad.Object} ) -func (qs *TripleStore) RemoveTriple(t *quad.Quad) { +func (qs *TripleStore) RemoveTriple(t quad.Quad) { _, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Couldn't access DB to confirm deletion") @@ -192,8 +192,8 @@ func (qs *TripleStore) RemoveTriple(t *quad.Quad) { qs.size-- } -func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) { - bytes, err := json.Marshal(*t) +func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t quad.Quad) { + bytes, err := json.Marshal(t) if err != nil { glog.Errorf("Couldn't write to buffer for triple %s: %s", t, err) return @@ -206,7 +206,7 @@ func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) { } } -func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t *quad.Quad) { +func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t quad.Quad) { qs.buildTripleWrite(batch, t) qs.UpdateValueKeyBy(t.Get(quad.Subject), 1, nil) qs.UpdateValueKeyBy(t.Get(quad.Predicate), 1, nil) @@ -267,7 +267,7 @@ func (qs *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb. } } -func (qs *TripleStore) AddTripleSet(t_s []*quad.Quad) { +func (qs *TripleStore) AddTripleSet(t_s []quad.Quad) { batch := &leveldb.Batch{} newTs := len(t_s) resizeMap := make(map[string]int) @@ -306,23 +306,23 @@ func (qs *TripleStore) Close() { qs.open = false } -func (qs *TripleStore) Quad(k graph.Value) *quad.Quad { +func (qs *TripleStore) Quad(k graph.Value) quad.Quad { var triple quad.Quad b, err := qs.db.Get(k.([]byte), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Error: couldn't get triple from DB.") - return &quad.Quad{} + return quad.Quad{} } if err == leveldb.ErrNotFound { // No harm, no foul. - return &quad.Quad{} + return quad.Quad{} } err = json.Unmarshal(b, &triple) if err != nil { glog.Error("Error: couldn't reconstruct triple.") - return &quad.Quad{} + return quad.Quad{} } - return &triple + return triple } func (qs *TripleStore) convertStringToByteHash(s string) []byte { diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go index 23eb11abe..c364f288f 100644 --- a/graph/memstore/triplestore.go +++ b/graph/memstore/triplestore.go @@ -101,13 +101,13 @@ func newTripleStore() *TripleStore { return &ts } -func (ts *TripleStore) AddTripleSet(triples []*quad.Quad) { +func (ts *TripleStore) AddTripleSet(triples []quad.Quad) { for _, t := range triples { ts.AddTriple(t) } } -func (ts *TripleStore) tripleExists(t *quad.Quad) (bool, int64) { +func (ts *TripleStore) tripleExists(t quad.Quad) (bool, int64) { smallest := -1 var smallest_tree *llrb.LLRB for d := quad.Subject; d <= quad.Label; d++ { @@ -137,19 +137,19 @@ func (ts *TripleStore) tripleExists(t *quad.Quad) (bool, int64) { if !ok { break } - if t.Equals(&ts.triples[val.(int64)]) { + if t.Equals(ts.triples[val.(int64)]) { return true, val.(int64) } } return false, 0 } -func (ts *TripleStore) AddTriple(t *quad.Quad) { +func (ts *TripleStore) AddTriple(t quad.Quad) { if exists, _ := ts.tripleExists(t); exists { return } var tripleID int64 - ts.triples = append(ts.triples, *t) + ts.triples = append(ts.triples, t) tripleID = ts.tripleIdCounter ts.size++ ts.tripleIdCounter++ @@ -178,7 +178,7 @@ func (ts *TripleStore) AddTriple(t *quad.Quad) { // TODO(barakmich): Add VIP indexing } -func (ts *TripleStore) RemoveTriple(t *quad.Quad) { +func (ts *TripleStore) RemoveTriple(t quad.Quad) { var tripleID int64 var exists bool tripleID = 0 @@ -224,8 +224,8 @@ func (ts *TripleStore) RemoveTriple(t *quad.Quad) { } } -func (ts *TripleStore) Quad(index graph.Value) *quad.Quad { - return &ts.triples[index.(int64)] +func (ts *TripleStore) Quad(index graph.Value) quad.Quad { + return ts.triples[index.(int64)] } func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator { diff --git a/graph/memstore/triplestore_test.go b/graph/memstore/triplestore_test.go index 44c43b412..1080b7eb1 100644 --- a/graph/memstore/triplestore_test.go +++ b/graph/memstore/triplestore_test.go @@ -37,7 +37,7 @@ import ( // \-->|#D#|------------->+---+ // +---+ // -var simpleGraph = []*quad.Quad{ +var simpleGraph = []quad.Quad{ {"A", "follows", "B", ""}, {"C", "follows", "B", ""}, {"C", "follows", "D", ""}, @@ -51,7 +51,7 @@ var simpleGraph = []*quad.Quad{ {"G", "status", "cool", "status_graph"}, } -func makeTestStore(data []*quad.Quad) (*TripleStore, []pair) { +func makeTestStore(data []quad.Quad) (*TripleStore, []pair) { seen := make(map[string]struct{}) ts := newTripleStore() var ( @@ -175,7 +175,7 @@ func TestLinksToOptimization(t *testing.T) { func TestRemoveTriple(t *testing.T) { ts, _ := makeTestStore(simpleGraph) - ts.RemoveTriple(&quad.Quad{"E", "follows", "F", ""}) + ts.RemoveTriple(quad.Quad{"E", "follows", "F", ""}) fixed := ts.FixedIterator() fixed.Add(ts.ValueOf("E")) diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go index 2022e6c93..20fea2f93 100644 --- a/graph/mongo/triplestore.go +++ b/graph/mongo/triplestore.go @@ -91,7 +91,7 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro return &qs, nil } -func (qs *TripleStore) getIdForTriple(t *quad.Quad) string { +func (qs *TripleStore) getIdForTriple(t quad.Quad) string { id := qs.ConvertStringToByteHash(t.Subject) id += qs.ConvertStringToByteHash(t.Predicate) id += qs.ConvertStringToByteHash(t.Object) @@ -150,7 +150,7 @@ func (qs *TripleStore) updateNodeBy(node_name string, inc int) { } } -func (qs *TripleStore) writeTriple(t *quad.Quad) bool { +func (qs *TripleStore) writeTriple(t quad.Quad) bool { tripledoc := bson.M{ "_id": qs.getIdForTriple(t), "Subject": t.Subject, @@ -170,7 +170,7 @@ func (qs *TripleStore) writeTriple(t *quad.Quad) bool { return true } -func (qs *TripleStore) AddTriple(t *quad.Quad) { +func (qs *TripleStore) AddTriple(t quad.Quad) { _ = qs.writeTriple(t) qs.updateNodeBy(t.Subject, 1) qs.updateNodeBy(t.Predicate, 1) @@ -180,7 +180,7 @@ func (qs *TripleStore) AddTriple(t *quad.Quad) { } } -func (qs *TripleStore) AddTripleSet(in []*quad.Quad) { +func (qs *TripleStore) AddTripleSet(in []quad.Quad) { qs.session.SetSafe(nil) ids := make(map[string]int) for _, t := range in { @@ -200,7 +200,7 @@ func (qs *TripleStore) AddTripleSet(in []*quad.Quad) { qs.session.SetSafe(&mgo.Safe{}) } -func (qs *TripleStore) RemoveTriple(t *quad.Quad) { +func (qs *TripleStore) RemoveTriple(t quad.Quad) { err := qs.db.C("triples").RemoveId(qs.getIdForTriple(t)) if err == mgo.ErrNotFound { return @@ -216,13 +216,13 @@ func (qs *TripleStore) RemoveTriple(t *quad.Quad) { } } -func (qs *TripleStore) Quad(val graph.Value) *quad.Quad { +func (qs *TripleStore) Quad(val graph.Value) quad.Quad { var bsonDoc bson.M err := qs.db.C("triples").FindId(val.(string)).One(&bsonDoc) if err != nil { glog.Errorf("Error: Couldn't retrieve triple %s %v", val, err) } - return &quad.Quad{ + return quad.Quad{ bsonDoc["Subject"].(string), bsonDoc["Predicate"].(string), bsonDoc["Object"].(string), diff --git a/graph/triplestore.go b/graph/triplestore.go index a722db307..c099d4789 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -40,17 +40,17 @@ type Value interface{} type TripleStore interface { // Add a triple to the store. - AddTriple(*quad.Quad) + AddTriple(quad.Quad) // Add a set of triples to the store, atomically if possible. - AddTripleSet([]*quad.Quad) + AddTripleSet([]quad.Quad) // Removes a triple matching the given one from the database, // if it exists. Does nothing otherwise. - RemoveTriple(*quad.Quad) + RemoveTriple(quad.Quad) // Given an opaque token, returns the triple for that token from the store. - Quad(Value) *quad.Quad + Quad(Value) quad.Quad // Given a direction and a token, creates an iterator of links which have // that node token in that directional field. diff --git a/http/http_test.go b/http/http_test.go index e4d8c30b3..d56bebf12 100644 --- a/http/http_test.go +++ b/http/http_test.go @@ -25,7 +25,7 @@ import ( var parseTests = []struct { message string input string - expect []*quad.Quad + expect []quad.Quad err error }{ { @@ -34,7 +34,7 @@ var parseTests = []struct { {"subject": "foo", "predicate": "bar", "object": "baz"}, {"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"} ]`, - expect: []*quad.Quad{ + expect: []quad.Quad{ {"foo", "bar", "baz", ""}, {"foo", "bar", "baz", "graph"}, }, @@ -45,7 +45,7 @@ var parseTests = []struct { input: `[ {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} ]`, - expect: []*quad.Quad{ + expect: []quad.Quad{ {"foo", "bar", "foo", ""}, }, err: nil, @@ -56,7 +56,7 @@ var parseTests = []struct { {"subject": "foo", "predicate": "bar"} ]`, expect: nil, - err: fmt.Errorf("Invalid triple at index %d. %v", 0, &quad.Quad{"foo", "bar", "", ""}), + err: fmt.Errorf("Invalid triple at index %d. %v", 0, quad.Quad{"foo", "bar", "", ""}), }, } diff --git a/http/write.go b/http/write.go index 7ad205bf4..f81094170 100644 --- a/http/write.go +++ b/http/write.go @@ -29,8 +29,8 @@ import ( "github.com/google/cayley/quad/nquads" ) -func ParseJsonToTripleList(jsonBody []byte) ([]*quad.Quad, error) { - var tripleList []*quad.Quad +func ParseJsonToTripleList(jsonBody []byte) ([]quad.Quad, error) { + var tripleList []quad.Quad err := json.Unmarshal(jsonBody, &tripleList) if err != nil { return nil, err @@ -83,7 +83,7 @@ func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params var ( n int - block = make([]*quad.Quad, 0, blockSize) + block = make([]quad.Quad, 0, blockSize) ) for { t, err := dec.Unmarshal() diff --git a/quad/cquads/cquads.go b/quad/cquads/cquads.go index c3355edef..5045a967d 100644 --- a/quad/cquads/cquads.go +++ b/quad/cquads/cquads.go @@ -34,9 +34,9 @@ import ( // Parse returns a valid quad.Quad or a non-nil error. Parse does // handle comments except where the comment placement does not prevent // a complete valid quad.Quad from being defined. -func Parse(str string) (*quad.Quad, error) { +func Parse(str string) (quad.Quad, error) { q, err := parse([]rune(str)) - return &q, err + return q, err } // Decoder implements simplified N-Quad document parsing. @@ -52,14 +52,14 @@ func NewDecoder(r io.Reader) *Decoder { } // Unmarshal returns the next valid N-Quad as a quad.Quad, or an error. -func (dec *Decoder) Unmarshal() (*quad.Quad, error) { +func (dec *Decoder) Unmarshal() (quad.Quad, error) { dec.line = dec.line[:0] var line []byte for { for { l, pre, err := dec.r.ReadLine() if err != nil { - return nil, err + return quad.Quad{}, err } dec.line = append(dec.line, l...) if !pre { @@ -73,9 +73,9 @@ func (dec *Decoder) Unmarshal() (*quad.Quad, error) { } triple, err := Parse(string(line)) if err != nil { - return nil, fmt.Errorf("failed to parse %q: %v", dec.line, err) + return quad.Quad{}, fmt.Errorf("failed to parse %q: %v", dec.line, err) } - if triple == nil { + if !triple.IsValid() { return dec.Unmarshal() } return triple, nil diff --git a/quad/cquads/cquads_test.go b/quad/cquads/cquads_test.go index 8c4f3ba0f..8d12ca755 100644 --- a/quad/cquads/cquads_test.go +++ b/quad/cquads/cquads_test.go @@ -31,7 +31,7 @@ import ( var testNTriples = []struct { message string input string - expect *quad.Quad + expect quad.Quad err error }{ // Tests from original nquads. @@ -40,7 +40,7 @@ var testNTriples = []struct { { message: "parse simple triples", input: "this is valid .", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "this", Predicate: "is", Object: "valid", @@ -50,7 +50,7 @@ var testNTriples = []struct { { message: "parse quoted triples", input: `this is "valid too" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "this", Predicate: "is", Object: "valid too", @@ -60,7 +60,7 @@ var testNTriples = []struct { { message: "parse escaped quoted triples", input: `he said "\"That's all folks\"" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "he", Predicate: "said", Object: `"That's all folks"`, @@ -70,7 +70,7 @@ var testNTriples = []struct { { message: "parse an example real triple", input: `":/guid/9202a8c04000641f80000000010c843c" "name" "George Morris" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: ":/guid/9202a8c04000641f80000000010c843c", Predicate: "name", Object: "George Morris", @@ -80,7 +80,7 @@ var testNTriples = []struct { { message: "parse a pathologically spaced triple", input: "foo is \"\\tA big tough\\r\\nDeal\\\\\" .", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "foo", Predicate: "is", Object: "\tA big tough\r\nDeal\\", @@ -92,7 +92,7 @@ var testNTriples = []struct { { message: "parse a simple quad", input: "this is valid quad .", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "this", Predicate: "is", Object: "valid", @@ -102,7 +102,7 @@ var testNTriples = []struct { { message: "parse a quoted quad", input: `this is valid "quad thing" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "this", Predicate: "is", Object: "valid", @@ -112,7 +112,7 @@ var testNTriples = []struct { { message: "parse crazy escaped quads", input: `"\"this" "\"is" "\"valid" "\"quad thing".`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: `"this`, Predicate: `"is`, Object: `"valid`, @@ -124,7 +124,7 @@ var testNTriples = []struct { { message: "handle simple case with comments", input: " . # comment", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example/s", Predicate: "http://example/p", Object: "http://example/o", @@ -134,7 +134,7 @@ var testNTriples = []struct { { message: "handle simple case with comments", input: " _:o . # comment", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example/s", Predicate: "http://example/p", Object: "_:o", @@ -144,7 +144,7 @@ var testNTriples = []struct { { message: "handle simple case with comments", input: " \"o\" . # comment", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example/s", Predicate: "http://example/p", Object: "o", @@ -154,7 +154,7 @@ var testNTriples = []struct { { message: "handle simple case with comments", input: " \"o\"^^ . # comment", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example/s", Predicate: "http://example/p", Object: `"o"^^`, @@ -164,7 +164,7 @@ var testNTriples = []struct { { message: "handle simple case with comments", input: " \"o\"@en . # comment", - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example/s", Predicate: "http://example/p", Object: `"o"@en`, @@ -177,7 +177,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: `_:100000 . # example from 30movies`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:100000", Predicate: "/film/performance/actor", Object: "/en/larry_fine_1902", @@ -189,7 +189,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: `_:10011 "Tomás de Torquemada" . # example from 30movies with unicode`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:10011", Predicate: "/film/performance/character", Object: "Tomás de Torquemada", @@ -202,7 +202,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: ` . # comments here`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://one.example/subject1", Predicate: "http://one.example/predicate1", Object: "http://one.example/object1", @@ -213,7 +213,7 @@ var testNTriples = []struct { { message: "parse triple with blank subject node, literal object and no comment (1)", input: `_:subject1 "object1" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject1", Predicate: "http://an.example/predicate1", Object: "object1", @@ -224,7 +224,7 @@ var testNTriples = []struct { { message: "parse triple with blank subject node, literal object and no comment (2)", input: `_:subject2 "object2" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject2", Predicate: "http://an.example/predicate2", Object: "object2", @@ -237,7 +237,7 @@ var testNTriples = []struct { { message: "parse triple with three IRIREFs", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/#spiderman", Predicate: "http://www.perceive.net/schemas/relationship/enemyOf", Object: "http://example.org/#green-goblin", @@ -250,7 +250,7 @@ var testNTriples = []struct { { message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)", input: `_:alice _:bob .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:alice", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "_:bob", @@ -261,7 +261,7 @@ var testNTriples = []struct { { message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)", input: `_:bob _:alice .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:bob", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "_:alice", @@ -274,7 +274,7 @@ var testNTriples = []struct { { message: "parse quad with commment", input: ` . # comments here`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://one.example/subject1", Predicate: "http://one.example/predicate1", Object: "http://one.example/object1", @@ -285,7 +285,7 @@ var testNTriples = []struct { { message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)", input: `_:subject1 "object1" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject1", Predicate: "http://an.example/predicate1", Object: "object1", @@ -296,7 +296,7 @@ var testNTriples = []struct { { message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)", input: `_:subject2 "object2" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject2", Predicate: "http://an.example/predicate2", Object: "object2", @@ -309,7 +309,7 @@ var testNTriples = []struct { { message: "parse quad with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/#spiderman", Predicate: "http://www.perceive.net/schemas/relationship/enemyOf", Object: "http://example.org/#green-goblin", @@ -322,7 +322,7 @@ var testNTriples = []struct { { message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)", input: `_:alice _:bob .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:alice", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "_:bob", @@ -333,7 +333,7 @@ var testNTriples = []struct { { message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)", input: `_:bob _:alice .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:bob", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "_:alice", @@ -346,7 +346,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", Object: "http://xmlns.com/foaf/0.1/Person", @@ -357,7 +357,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "http://example.org/alice#me", @@ -368,7 +368,7 @@ var testNTriples = []struct { { message: "parse triple with IRIREF schema on literal object", input: ` "1990-07-04"^^ .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://schema.org/birthDate", Object: `"1990-07-04"^^`, @@ -379,7 +379,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in triple", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://xmlns.com/foaf/0.1/topic_interest", Object: "http://www.wikidata.org/entity/Q12418", @@ -390,7 +390,7 @@ var testNTriples = []struct { { message: "parse triple with literal subject", input: ` "Mona Lisa" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://www.wikidata.org/entity/Q12418", Predicate: "http://purl.org/dc/terms/title", Object: "Mona Lisa", @@ -401,7 +401,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://www.wikidata.org/entity/Q12418", Predicate: "http://purl.org/dc/terms/creator", Object: "http://dbpedia.org/resource/Leonardo_da_Vinci", @@ -412,7 +412,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619", Predicate: "http://purl.org/dc/terms/subject", Object: "http://www.wikidata.org/entity/Q12418", @@ -425,7 +425,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in quad (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", Object: "http://xmlns.com/foaf/0.1/Person", @@ -436,7 +436,7 @@ var testNTriples = []struct { { message: "parse quad with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://xmlns.com/foaf/0.1/knows", Object: "http://example.org/alice#me", @@ -447,7 +447,7 @@ var testNTriples = []struct { { message: "parse quad with IRIREF schema on literal object", input: ` "1990-07-04"^^ .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://schema.org/birthDate", Object: `"1990-07-04"^^`, @@ -458,7 +458,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in quad (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://xmlns.com/foaf/0.1/topic_interest", Object: "http://www.wikidata.org/entity/Q12418", @@ -469,7 +469,7 @@ var testNTriples = []struct { { message: "parse literal object and colon qualified label in quad", input: ` "Mona Lisa" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://www.wikidata.org/entity/Q12418", Predicate: "http://purl.org/dc/terms/title", Object: "Mona Lisa", @@ -480,7 +480,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts with colon qualified label in quad (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://www.wikidata.org/entity/Q12418", Predicate: "http://purl.org/dc/terms/creator", Object: "http://dbpedia.org/resource/Leonardo_da_Vinci", @@ -491,7 +491,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts with colon qualified label in quad (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619", Predicate: "http://purl.org/dc/terms/subject", Object: "http://www.wikidata.org/entity/Q12418", @@ -502,7 +502,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts (quad section - 1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob", Predicate: "http://purl.org/dc/terms/publisher", Object: "http://example.org", @@ -513,7 +513,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts (quad section - 2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob", Predicate: "http://purl.org/dc/terms/rights", Object: "http://creativecommons.org/licenses/by/3.0/", @@ -526,31 +526,31 @@ var testNTriples = []struct { { message: "parse empty", input: ``, - expect: &quad.Quad{}, + expect: quad.Quad{}, err: quad.ErrIncomplete, }, { message: "parse commented", input: `# is a comment`, - expect: &quad.Quad{}, + expect: quad.Quad{}, err: fmt.Errorf("%v: unexpected rune '#' at 0", quad.ErrInvalid), }, { message: "parse commented internal (1)", input: `is # a comment`, - expect: &quad.Quad{Subject: "is"}, + expect: quad.Quad{Subject: "is"}, err: fmt.Errorf("%v: unexpected rune '#' at 3", quad.ErrInvalid), }, { message: "parse commented internal (2)", input: `is a # comment`, - expect: &quad.Quad{Subject: "is", Predicate: "a"}, + expect: quad.Quad{Subject: "is", Predicate: "a"}, err: fmt.Errorf("%v: unexpected rune '#' at 5", quad.ErrInvalid), }, { message: "parse incomplete quad (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", Object: "", @@ -561,7 +561,7 @@ var testNTriples = []struct { { message: "parse incomplete quad (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "http://example.org/bob#me", Predicate: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", Object: "", @@ -773,7 +773,7 @@ func TestUnescape(t *testing.T) { } } -var result *quad.Quad +var result quad.Quad func BenchmarkParser(b *testing.B) { for n := 0; n < b.N; n++ { diff --git a/quad/nquads/nquads.go b/quad/nquads/nquads.go index 5d54e1da1..7b27d7e2e 100644 --- a/quad/nquads/nquads.go +++ b/quad/nquads/nquads.go @@ -33,9 +33,9 @@ import ( // Parse returns a valid quad.Quad or a non-nil error. Parse does // handle comments except where the comment placement does not prevent // a complete valid quad.Quad from being defined. -func Parse(str string) (*quad.Quad, error) { +func Parse(str string) (quad.Quad, error) { q, err := parse([]rune(str)) - return &q, err + return q, err } // Decoder implements N-Quad document parsing according to the RDF @@ -52,14 +52,14 @@ func NewDecoder(r io.Reader) *Decoder { } // Unmarshal returns the next valid N-Quad as a quad.Quad, or an error. -func (dec *Decoder) Unmarshal() (*quad.Quad, error) { +func (dec *Decoder) Unmarshal() (quad.Quad, error) { dec.line = dec.line[:0] var line []byte for { for { l, pre, err := dec.r.ReadLine() if err != nil { - return nil, err + return quad.Quad{}, err } dec.line = append(dec.line, l...) if !pre { @@ -73,9 +73,9 @@ func (dec *Decoder) Unmarshal() (*quad.Quad, error) { } triple, err := Parse(string(line)) if err != nil { - return nil, fmt.Errorf("failed to parse %q: %v", dec.line, err) + return quad.Quad{}, fmt.Errorf("failed to parse %q: %v", dec.line, err) } - if triple == nil { + if !triple.IsValid() { return dec.Unmarshal() } return triple, nil diff --git a/quad/nquads/nquads_test.go b/quad/nquads/nquads_test.go index fc7b70a73..aaac54942 100644 --- a/quad/nquads/nquads_test.go +++ b/quad/nquads/nquads_test.go @@ -31,7 +31,7 @@ import ( var testNTriples = []struct { message string input string - expect *quad.Quad + expect quad.Quad err error }{ // Tests taken from http://www.w3.org/TR/n-quads/ and http://www.w3.org/TR/n-triples/. @@ -40,7 +40,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: `_:100000 . # example from 30movies`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:100000", Predicate: "", Object: "", @@ -52,7 +52,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: `_:10011 "Tomás de Torquemada" . # example from 30movies with unicode`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:10011", Predicate: "", Object: `"Tomás de Torquemada"`, @@ -65,7 +65,7 @@ var testNTriples = []struct { { message: "parse triple with commment", input: ` . # comments here`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -76,7 +76,7 @@ var testNTriples = []struct { { message: "parse triple with blank subject node, literal object and no comment (1)", input: `_:subject1 "object1" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject1", Predicate: "", Object: `"object1"`, @@ -87,7 +87,7 @@ var testNTriples = []struct { { message: "parse triple with blank subject node, literal object and no comment (2)", input: `_:subject2 "object2" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject2", Predicate: "", Object: `"object2"`, @@ -100,7 +100,7 @@ var testNTriples = []struct { { message: "parse triple with three IRIREFs", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -113,7 +113,7 @@ var testNTriples = []struct { { message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)", input: `_:alice _:bob .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:alice", Predicate: "", Object: "_:bob", @@ -124,7 +124,7 @@ var testNTriples = []struct { { message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)", input: `_:bob _:alice .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:bob", Predicate: "", Object: "_:alice", @@ -137,7 +137,7 @@ var testNTriples = []struct { { message: "parse quad with commment", input: ` . # comments here`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -148,7 +148,7 @@ var testNTriples = []struct { { message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)", input: `_:subject1 "object1" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject1", Predicate: "", Object: `"object1"`, @@ -159,7 +159,7 @@ var testNTriples = []struct { { message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)", input: `_:subject2 "object2" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:subject2", Predicate: "", Object: `"object2"`, @@ -172,7 +172,7 @@ var testNTriples = []struct { { message: "parse quad with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -185,7 +185,7 @@ var testNTriples = []struct { { message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)", input: `_:alice _:bob .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:alice", Predicate: "", Object: "_:bob", @@ -196,7 +196,7 @@ var testNTriples = []struct { { message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)", input: `_:bob _:alice .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "_:bob", Predicate: "", Object: "_:alice", @@ -209,7 +209,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -220,7 +220,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -231,7 +231,7 @@ var testNTriples = []struct { { message: "parse triple with IRIREF schema on literal object", input: ` "1990-07-04"^^ .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: `"1990-07-04"^^`, @@ -242,7 +242,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in triple", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -253,7 +253,7 @@ var testNTriples = []struct { { message: "parse triple with literal subject", input: ` "Mona Lisa" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: `"Mona Lisa"`, @@ -264,7 +264,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -275,7 +275,7 @@ var testNTriples = []struct { { message: "parse triple with all IRIREF parts (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -288,7 +288,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in quad (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -299,7 +299,7 @@ var testNTriples = []struct { { message: "parse quad with all IRIREF parts", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -310,7 +310,7 @@ var testNTriples = []struct { { message: "parse quad with IRIREF schema on literal object", input: ` "1990-07-04"^^ .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: `"1990-07-04"^^`, @@ -321,7 +321,7 @@ var testNTriples = []struct { { message: "parse commented IRIREF in quad (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -332,7 +332,7 @@ var testNTriples = []struct { { message: "parse literal object and colon qualified label in quad", input: ` "Mona Lisa" .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: `"Mona Lisa"`, @@ -343,7 +343,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts with colon qualified label in quad (1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -354,7 +354,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts with colon qualified label in quad (2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -365,7 +365,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts (quad section - 1)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -376,7 +376,7 @@ var testNTriples = []struct { { message: "parse all IRIREF parts (quad section - 2)", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -389,19 +389,19 @@ var testNTriples = []struct { { message: "parse empty", input: ``, - expect: &quad.Quad{}, + expect: quad.Quad{}, err: quad.ErrIncomplete, }, { message: "parse commented", input: `# comment`, - expect: &quad.Quad{}, + expect: quad.Quad{}, err: fmt.Errorf("%v: unexpected rune '#' at 0", quad.ErrInvalid), }, { message: "parse incomplete quad", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -412,7 +412,7 @@ var testNTriples = []struct { { message: "parse incomplete quad", input: ` .`, - expect: &quad.Quad{ + expect: quad.Quad{ Subject: "", Predicate: "", Object: "", @@ -580,7 +580,7 @@ func TestUnescape(t *testing.T) { } } -var result *quad.Quad +var result quad.Quad func BenchmarkParser(b *testing.B) { for n := 0; n < b.N; n++ { diff --git a/quad/quad.go b/quad/quad.go index 3e9943b01..2f172c3cb 100644 --- a/quad/quad.go +++ b/quad/quad.go @@ -104,7 +104,7 @@ func (d Direction) String() string { // instead of the pointer. This needs benchmarking to make the decision. // Per-field accessor for triples -func (q *Quad) Get(d Direction) string { +func (q Quad) Get(d Direction) string { switch d { case Subject: return q.Subject @@ -119,16 +119,16 @@ func (q *Quad) Get(d Direction) string { } } -func (q *Quad) Equals(o *Quad) bool { - return *q == *o +func (q Quad) Equals(o Quad) bool { + return q == o } // Pretty-prints a triple. -func (q *Quad) String() string { +func (q Quad) String() string { return fmt.Sprintf("%s -- %s -> %s", q.Subject, q.Predicate, q.Object) } -func (q *Quad) IsValid() bool { +func (q Quad) IsValid() bool { return q.Subject != "" && q.Predicate != "" && q.Object != "" } @@ -137,7 +137,7 @@ func (q *Quad) IsValid() bool { // from nquads to here to provide UnmarshalText(text []byte) error. // Prints a triple in N-Quad format. -func (q *Quad) NTriple() string { +func (q Quad) NTriple() string { if q.Label == "" { //TODO(barakmich): Proper escaping. return fmt.Sprintf("%s %s %s .", q.Subject, q.Predicate, q.Object) @@ -147,5 +147,5 @@ func (q *Quad) NTriple() string { } type Unmarshaler interface { - Unmarshal() (*Quad, error) + Unmarshal() (Quad, error) } diff --git a/query/gremlin/gremlin_test.go b/query/gremlin/gremlin_test.go index dbd115a10..a638eb70a 100644 --- a/query/gremlin/gremlin_test.go +++ b/query/gremlin/gremlin_test.go @@ -38,7 +38,7 @@ import ( // \-->|#D#|------------->+---+ // +---+ // -var simpleGraph = []*quad.Quad{ +var simpleGraph = []quad.Quad{ {"A", "follows", "B", ""}, {"C", "follows", "B", ""}, {"C", "follows", "D", ""}, @@ -52,7 +52,7 @@ var simpleGraph = []*quad.Quad{ {"G", "status", "cool", "status_graph"}, } -func makeTestSession(data []*quad.Quad) *Session { +func makeTestSession(data []quad.Quad) *Session { ts, _ := graph.NewTripleStore("memstore", "", nil) for _, t := range data { ts.AddTriple(t) @@ -246,7 +246,7 @@ var testQueries = []struct { }, } -func runQueryGetTag(g []*quad.Quad, query string, tag string) []string { +func runQueryGetTag(g []quad.Quad, query string, tag string) []string { js := makeTestSession(g) c := make(chan interface{}, 5) js.ExecInput(query, c, -1) diff --git a/query/mql/mql_test.go b/query/mql/mql_test.go index 6be480e6a..619ae546a 100644 --- a/query/mql/mql_test.go +++ b/query/mql/mql_test.go @@ -37,7 +37,7 @@ import ( // \-->|#D#|------------->+---+ // +---+ // -var simpleGraph = []*quad.Quad{ +var simpleGraph = []quad.Quad{ {"A", "follows", "B", ""}, {"C", "follows", "B", ""}, {"C", "follows", "D", ""}, @@ -51,7 +51,7 @@ var simpleGraph = []*quad.Quad{ {"G", "status", "cool", "status_graph"}, } -func makeTestSession(data []*quad.Quad) *Session { +func makeTestSession(data []quad.Quad) *Session { ts, _ := graph.NewTripleStore("memstore", "", nil) for _, t := range data { ts.AddTriple(t) @@ -165,7 +165,7 @@ var testQueries = []struct { }, } -func runQuery(g []*quad.Quad, query string) interface{} { +func runQuery(g []quad.Quad, query string) interface{} { s := makeTestSession(g) c := make(chan interface{}, 5) go s.ExecInput(query, c, -1) diff --git a/query/sexp/parser_test.go b/query/sexp/parser_test.go index e7e66bf65..ed4776a5a 100644 --- a/query/sexp/parser_test.go +++ b/query/sexp/parser_test.go @@ -32,21 +32,21 @@ func TestBadParse(t *testing.T) { var testQueries = []struct { message string - add *quad.Quad + add quad.Quad query string typ graph.Type expect string }{ { message: "get a single triple linkage", - add: &quad.Quad{"i", "can", "win", ""}, + add: quad.Quad{"i", "can", "win", ""}, query: "($a (:can \"win\"))", typ: graph.And, expect: "i", }, { message: "get a single triple linkage", - add: &quad.Quad{"i", "can", "win", ""}, + add: quad.Quad{"i", "can", "win", ""}, query: "(\"i\" (:can $a))", typ: graph.And, expect: "i", @@ -60,7 +60,7 @@ func TestMemstoreBackedSexp(t *testing.T) { t.Errorf(`Incorrect type for empty query, got:%q expect: "null"`, it.Type()) } for _, test := range testQueries { - if test.add != nil { + if test.add.IsValid() { ts.AddTriple(test.add) } it := BuildIteratorTreeForQuery(ts, test.query) @@ -79,8 +79,8 @@ func TestMemstoreBackedSexp(t *testing.T) { func TestTreeConstraintParse(t *testing.T) { ts, _ := graph.NewTripleStore("memstore", "", nil) - ts.AddTriple(&quad.Quad{"i", "like", "food", ""}) - ts.AddTriple(&quad.Quad{"food", "is", "good", ""}) + ts.AddTriple(quad.Quad{"i", "like", "food", ""}) + ts.AddTriple(quad.Quad{"food", "is", "good", ""}) query := "(\"i\"\n" + "(:like\n" + "($a (:is :good))))" @@ -99,8 +99,8 @@ func TestTreeConstraintParse(t *testing.T) { func TestTreeConstraintTagParse(t *testing.T) { ts, _ := graph.NewTripleStore("memstore", "", nil) - ts.AddTriple(&quad.Quad{"i", "like", "food", ""}) - ts.AddTriple(&quad.Quad{"food", "is", "good", ""}) + ts.AddTriple(quad.Quad{"i", "like", "food", ""}) + ts.AddTriple(quad.Quad{"food", "is", "good", ""}) query := "(\"i\"\n" + "(:like\n" + "($a (:is :good))))" @@ -119,7 +119,7 @@ func TestTreeConstraintTagParse(t *testing.T) { func TestMultipleConstraintParse(t *testing.T) { ts, _ := graph.NewTripleStore("memstore", "", nil) - for _, tv := range []*quad.Quad{ + for _, tv := range []quad.Quad{ {"i", "like", "food", ""}, {"i", "like", "beer", ""}, {"you", "like", "beer", ""}, From 2c6f0f7345c43d5aa3188a1f536fa087f7f66945 Mon Sep 17 00:00:00 2001 From: kortschak Date: Tue, 5 Aug 2014 23:27:13 +0930 Subject: [PATCH 11/24] Clean up residual changes Remove done TODOs and unnecessary helpers. --- graph/memstore/triplestore.go | 2 +- quad/cquads/cquads.go | 8 -- quad/cquads/parse.go | 145 ++++++++++++++++++---------------- quad/cquads/parse.rl | 7 +- quad/nquads/nquads.go | 8 -- quad/nquads/parse.go | 77 +++++++++--------- quad/nquads/parse.rl | 7 +- quad/quad.go | 11 --- 8 files changed, 129 insertions(+), 136 deletions(-) diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go index c364f288f..15a6ecde1 100644 --- a/graph/memstore/triplestore.go +++ b/graph/memstore/triplestore.go @@ -137,7 +137,7 @@ func (ts *TripleStore) tripleExists(t quad.Quad) (bool, int64) { if !ok { break } - if t.Equals(ts.triples[val.(int64)]) { + if t == ts.triples[val.(int64)] { return true, val.(int64) } } diff --git a/quad/cquads/cquads.go b/quad/cquads/cquads.go index 5045a967d..ea399f9eb 100644 --- a/quad/cquads/cquads.go +++ b/quad/cquads/cquads.go @@ -31,14 +31,6 @@ import ( "github.com/google/cayley/quad" ) -// Parse returns a valid quad.Quad or a non-nil error. Parse does -// handle comments except where the comment placement does not prevent -// a complete valid quad.Quad from being defined. -func Parse(str string) (quad.Quad, error) { - q, err := parse([]rune(str)) - return q, err -} - // Decoder implements simplified N-Quad document parsing. type Decoder struct { r *bufio.Reader diff --git a/quad/cquads/parse.go b/quad/cquads/parse.go index 6500fcfa2..5fa2f55a0 100644 --- a/quad/cquads/parse.go +++ b/quad/cquads/parse.go @@ -37,7 +37,12 @@ const quads_en_statement int = 1 // line 34 "parse.rl" -func parse(data []rune) (quad.Quad, error) { +// Parse returns a valid quad.Quad or a non-nil error. Parse does +// handle comments except where the comment placement does not prevent +// a complete valid quad.Quad from being defined. +func Parse(statement string) (quad.Quad, error) { + data := []rune(statement) + var ( cs, p int pe = len(data) @@ -55,15 +60,15 @@ func parse(data []rune) (quad.Quad, error) { ) -// line 59 "parse.go" +// line 64 "parse.go" { cs = quads_start } -// line 54 "parse.rl" +// line 59 "parse.rl" -// line 67 "parse.go" +// line 72 "parse.go" { if p == pe { goto _test_eof @@ -599,7 +604,7 @@ tr0: return q, quad.ErrIncomplete goto st0 -// line 603 "parse.go" +// line 608 "parse.go" st_case_0: st0: cs = 0 @@ -623,7 +628,7 @@ tr203: goto _test_eof2 } st_case_2: -// line 627 "parse.go" +// line 632 "parse.go" switch data[p] { case 9: goto tr7 @@ -696,7 +701,7 @@ tr215: goto _test_eof3 } st_case_3: -// line 700 "parse.go" +// line 705 "parse.go" switch data[p] { case 9: goto st3 @@ -741,7 +746,7 @@ tr153: goto _test_eof4 } st_case_4: -// line 745 "parse.go" +// line 750 "parse.go" switch data[p] { case 9: goto tr17 @@ -814,7 +819,7 @@ tr165: goto _test_eof5 } st_case_5: -// line 818 "parse.go" +// line 823 "parse.go" switch data[p] { case 9: goto st5 @@ -859,7 +864,7 @@ tr101: goto _test_eof6 } st_case_6: -// line 863 "parse.go" +// line 868 "parse.go" switch data[p] { case 9: goto tr27 @@ -932,7 +937,7 @@ tr113: goto _test_eof7 } st_case_7: -// line 936 "parse.go" +// line 941 "parse.go" switch data[p] { case 9: goto st7 @@ -977,7 +982,7 @@ tr48: goto _test_eof8 } st_case_8: -// line 981 "parse.go" +// line 986 "parse.go" switch data[p] { case 9: goto tr37 @@ -1050,7 +1055,7 @@ tr60: goto _test_eof9 } st_case_9: -// line 1054 "parse.go" +// line 1059 "parse.go" switch data[p] { case 9: goto st9 @@ -1123,7 +1128,7 @@ tr114: goto _test_eof178 } st_case_178: -// line 1127 "parse.go" +// line 1132 "parse.go" switch data[p] { case 9: goto st178 @@ -1144,7 +1149,7 @@ tr250: goto _test_eof179 } st_case_179: -// line 1148 "parse.go" +// line 1153 "parse.go" goto st179 tr34: // line 38 "actions.rl" @@ -1187,7 +1192,7 @@ tr49: goto _test_eof180 } st_case_180: -// line 1191 "parse.go" +// line 1196 "parse.go" switch data[p] { case 9: goto st178 @@ -1236,7 +1241,7 @@ tr252: goto _test_eof181 } st_case_181: -// line 1240 "parse.go" +// line 1245 "parse.go" switch data[p] { case 9: goto tr253 @@ -1309,7 +1314,7 @@ tr287: goto _test_eof182 } st_case_182: -// line 1313 "parse.go" +// line 1318 "parse.go" switch data[p] { case 9: goto st182 @@ -1353,7 +1358,7 @@ tr288: goto _test_eof183 } st_case_183: -// line 1357 "parse.go" +// line 1362 "parse.go" switch data[p] { case 9: goto st183 @@ -1404,7 +1409,7 @@ tr265: goto _test_eof184 } st_case_184: -// line 1408 "parse.go" +// line 1413 "parse.go" switch data[p] { case 9: goto st183 @@ -1469,7 +1474,7 @@ tr266: goto _test_eof186 } st_case_186: -// line 1473 "parse.go" +// line 1478 "parse.go" switch data[p] { case 34: goto st187 @@ -1704,7 +1709,7 @@ tr50: goto _test_eof11 } st_case_11: -// line 1708 "parse.go" +// line 1713 "parse.go" switch data[p] { case 34: goto st12 @@ -1917,7 +1922,7 @@ tr87: goto _test_eof21 } st_case_21: -// line 1921 "parse.go" +// line 1926 "parse.go" switch data[p] { case 34: goto st22 @@ -1949,7 +1954,7 @@ tr88: goto _test_eof22 } st_case_22: -// line 1953 "parse.go" +// line 1958 "parse.go" switch data[p] { case 9: goto tr60 @@ -2077,7 +2082,7 @@ tr81: goto _test_eof29 } st_case_29: -// line 2081 "parse.go" +// line 2086 "parse.go" switch data[p] { case 62: goto st30 @@ -2118,7 +2123,7 @@ tr82: goto _test_eof30 } st_case_30: -// line 2122 "parse.go" +// line 2127 "parse.go" switch data[p] { case 9: goto tr37 @@ -2140,7 +2145,7 @@ tr83: goto _test_eof31 } st_case_31: -// line 2144 "parse.go" +// line 2149 "parse.go" switch data[p] { case 85: goto st32 @@ -2337,7 +2342,7 @@ tr89: goto _test_eof41 } st_case_41: -// line 2341 "parse.go" +// line 2346 "parse.go" switch data[p] { case 34: goto st42 @@ -2541,7 +2546,7 @@ tr36: goto _test_eof51 } st_case_51: -// line 2545 "parse.go" +// line 2550 "parse.go" switch data[p] { case 9: goto tr37 @@ -2599,7 +2604,7 @@ tr102: goto _test_eof196 } st_case_196: -// line 2603 "parse.go" +// line 2608 "parse.go" switch data[p] { case 9: goto st178 @@ -2641,7 +2646,7 @@ tr273: goto _test_eof197 } st_case_197: -// line 2645 "parse.go" +// line 2650 "parse.go" switch data[p] { case 9: goto tr274 @@ -2697,7 +2702,7 @@ tr327: goto _test_eof198 } st_case_198: -// line 2701 "parse.go" +// line 2706 "parse.go" switch data[p] { case 9: goto st198 @@ -2742,7 +2747,7 @@ tr314: goto _test_eof199 } st_case_199: -// line 2746 "parse.go" +// line 2751 "parse.go" switch data[p] { case 34: goto st200 @@ -2774,7 +2779,7 @@ tr315: goto _test_eof200 } st_case_200: -// line 2778 "parse.go" +// line 2783 "parse.go" switch data[p] { case 9: goto tr287 @@ -2902,7 +2907,7 @@ tr308: goto _test_eof207 } st_case_207: -// line 2906 "parse.go" +// line 2911 "parse.go" switch data[p] { case 62: goto st208 @@ -2943,7 +2948,7 @@ tr309: goto _test_eof208 } st_case_208: -// line 2947 "parse.go" +// line 2952 "parse.go" switch data[p] { case 9: goto tr253 @@ -2965,7 +2970,7 @@ tr310: goto _test_eof209 } st_case_209: -// line 2969 "parse.go" +// line 2974 "parse.go" switch data[p] { case 85: goto st210 @@ -3162,7 +3167,7 @@ tr316: goto _test_eof219 } st_case_219: -// line 3166 "parse.go" +// line 3171 "parse.go" switch data[p] { case 34: goto st220 @@ -3366,7 +3371,7 @@ tr283: goto _test_eof229 } st_case_229: -// line 3370 "parse.go" +// line 3375 "parse.go" switch data[p] { case 9: goto tr253 @@ -3424,7 +3429,7 @@ tr329: goto _test_eof230 } st_case_230: -// line 3428 "parse.go" +// line 3433 "parse.go" switch data[p] { case 9: goto st183 @@ -3482,7 +3487,7 @@ tr330: goto _test_eof232 } st_case_232: -// line 3486 "parse.go" +// line 3491 "parse.go" switch data[p] { case 34: goto st233 @@ -3688,7 +3693,7 @@ tr24: goto _test_eof52 } st_case_52: -// line 3692 "parse.go" +// line 3697 "parse.go" switch data[p] { case 33: goto st6 @@ -3725,7 +3730,7 @@ tr103: goto _test_eof53 } st_case_53: -// line 3729 "parse.go" +// line 3734 "parse.go" switch data[p] { case 34: goto st54 @@ -3938,7 +3943,7 @@ tr140: goto _test_eof63 } st_case_63: -// line 3942 "parse.go" +// line 3947 "parse.go" switch data[p] { case 34: goto st64 @@ -3970,7 +3975,7 @@ tr141: goto _test_eof64 } st_case_64: -// line 3974 "parse.go" +// line 3979 "parse.go" switch data[p] { case 9: goto tr113 @@ -4098,7 +4103,7 @@ tr134: goto _test_eof71 } st_case_71: -// line 4102 "parse.go" +// line 4107 "parse.go" switch data[p] { case 62: goto st72 @@ -4139,7 +4144,7 @@ tr135: goto _test_eof72 } st_case_72: -// line 4143 "parse.go" +// line 4148 "parse.go" switch data[p] { case 9: goto tr27 @@ -4161,7 +4166,7 @@ tr136: goto _test_eof73 } st_case_73: -// line 4165 "parse.go" +// line 4170 "parse.go" switch data[p] { case 85: goto st74 @@ -4358,7 +4363,7 @@ tr142: goto _test_eof83 } st_case_83: -// line 4362 "parse.go" +// line 4367 "parse.go" switch data[p] { case 34: goto st84 @@ -4562,7 +4567,7 @@ tr26: goto _test_eof93 } st_case_93: -// line 4566 "parse.go" +// line 4571 "parse.go" switch data[p] { case 9: goto tr27 @@ -4605,7 +4610,7 @@ tr154: goto _test_eof94 } st_case_94: -// line 4609 "parse.go" +// line 4614 "parse.go" switch data[p] { case 33: goto st4 @@ -4642,7 +4647,7 @@ tr155: goto _test_eof95 } st_case_95: -// line 4646 "parse.go" +// line 4651 "parse.go" switch data[p] { case 34: goto st96 @@ -4855,7 +4860,7 @@ tr190: goto _test_eof105 } st_case_105: -// line 4859 "parse.go" +// line 4864 "parse.go" switch data[p] { case 34: goto st106 @@ -4887,7 +4892,7 @@ tr191: goto _test_eof106 } st_case_106: -// line 4891 "parse.go" +// line 4896 "parse.go" switch data[p] { case 9: goto tr165 @@ -5009,7 +5014,7 @@ tr184: goto _test_eof113 } st_case_113: -// line 5013 "parse.go" +// line 5018 "parse.go" switch data[p] { case 62: goto st114 @@ -5050,7 +5055,7 @@ tr185: goto _test_eof114 } st_case_114: -// line 5054 "parse.go" +// line 5059 "parse.go" switch data[p] { case 9: goto tr17 @@ -5070,7 +5075,7 @@ tr186: goto _test_eof115 } st_case_115: -// line 5074 "parse.go" +// line 5079 "parse.go" switch data[p] { case 85: goto st116 @@ -5267,7 +5272,7 @@ tr192: goto _test_eof125 } st_case_125: -// line 5271 "parse.go" +// line 5276 "parse.go" switch data[p] { case 34: goto st126 @@ -5471,7 +5476,7 @@ tr16: goto _test_eof135 } st_case_135: -// line 5475 "parse.go" +// line 5480 "parse.go" switch data[p] { case 9: goto tr17 @@ -5514,7 +5519,7 @@ tr204: goto _test_eof136 } st_case_136: -// line 5518 "parse.go" +// line 5523 "parse.go" switch data[p] { case 33: goto st2 @@ -5551,7 +5556,7 @@ tr205: goto _test_eof137 } st_case_137: -// line 5555 "parse.go" +// line 5560 "parse.go" switch data[p] { case 34: goto st138 @@ -5764,7 +5769,7 @@ tr240: goto _test_eof147 } st_case_147: -// line 5768 "parse.go" +// line 5773 "parse.go" switch data[p] { case 34: goto st148 @@ -5796,7 +5801,7 @@ tr241: goto _test_eof148 } st_case_148: -// line 5800 "parse.go" +// line 5805 "parse.go" switch data[p] { case 9: goto tr215 @@ -5918,7 +5923,7 @@ tr234: goto _test_eof155 } st_case_155: -// line 5922 "parse.go" +// line 5927 "parse.go" switch data[p] { case 62: goto st156 @@ -5959,7 +5964,7 @@ tr235: goto _test_eof156 } st_case_156: -// line 5963 "parse.go" +// line 5968 "parse.go" switch data[p] { case 9: goto tr7 @@ -5979,7 +5984,7 @@ tr236: goto _test_eof157 } st_case_157: -// line 5983 "parse.go" +// line 5988 "parse.go" switch data[p] { case 85: goto st158 @@ -6176,7 +6181,7 @@ tr242: goto _test_eof167 } st_case_167: -// line 6180 "parse.go" +// line 6185 "parse.go" switch data[p] { case 34: goto st168 @@ -6380,7 +6385,7 @@ tr6: goto _test_eof177 } st_case_177: -// line 6384 "parse.go" +// line 6389 "parse.go" switch data[p] { case 9: goto tr7 @@ -6679,14 +6684,14 @@ tr6: return q, nil -// line 6683 "parse.go" +// line 6688 "parse.go" } } _out: {} } -// line 56 "parse.rl" +// line 61 "parse.rl" return quad.Quad{}, quad.ErrInvalid } diff --git a/quad/cquads/parse.rl b/quad/cquads/parse.rl index 05d2c3a18..b0329d6d7 100644 --- a/quad/cquads/parse.rl +++ b/quad/cquads/parse.rl @@ -33,7 +33,12 @@ import ( write data; }%% -func parse(data []rune) (quad.Quad, error) { +// Parse returns a valid quad.Quad or a non-nil error. Parse does +// handle comments except where the comment placement does not prevent +// a complete valid quad.Quad from being defined. +func Parse(statement string) (quad.Quad, error) { + data := []rune(statement) + var ( cs, p int pe = len(data) diff --git a/quad/nquads/nquads.go b/quad/nquads/nquads.go index 7b27d7e2e..9652ebc0b 100644 --- a/quad/nquads/nquads.go +++ b/quad/nquads/nquads.go @@ -30,14 +30,6 @@ import ( "github.com/google/cayley/quad" ) -// Parse returns a valid quad.Quad or a non-nil error. Parse does -// handle comments except where the comment placement does not prevent -// a complete valid quad.Quad from being defined. -func Parse(str string) (quad.Quad, error) { - q, err := parse([]rune(str)) - return q, err -} - // Decoder implements N-Quad document parsing according to the RDF // 1.1 N-Quads specification. type Decoder struct { diff --git a/quad/nquads/parse.go b/quad/nquads/parse.go index 9ec8554ee..8ea0c2c3f 100644 --- a/quad/nquads/parse.go +++ b/quad/nquads/parse.go @@ -37,7 +37,12 @@ const quads_en_statement int = 1 // line 34 "parse.rl" -func parse(data []rune) (quad.Quad, error) { +// Parse returns a valid quad.Quad or a non-nil error. Parse does +// handle comments except where the comment placement does not prevent +// a complete valid quad.Quad from being defined. +func Parse(statement string) (quad.Quad, error) { + data := []rune(statement) + var ( cs, p int pe = len(data) @@ -54,15 +59,15 @@ func parse(data []rune) (quad.Quad, error) { ) -// line 58 "parse.go" +// line 63 "parse.go" { cs = quads_start } -// line 53 "parse.rl" +// line 58 "parse.rl" -// line 66 "parse.go" +// line 71 "parse.go" { if p == pe { goto _test_eof @@ -286,7 +291,7 @@ tr0: return q, quad.ErrIncomplete goto st0 -// line 290 "parse.go" +// line 295 "parse.go" st_case_0: st0: cs = 0 @@ -310,7 +315,7 @@ tr120: goto _test_eof2 } st_case_2: -// line 314 "parse.go" +// line 319 "parse.go" switch data[p] { case 62: goto st3 @@ -351,7 +356,7 @@ tr121: goto _test_eof3 } st_case_3: -// line 355 "parse.go" +// line 360 "parse.go" switch data[p] { case 9: goto tr7 @@ -377,7 +382,7 @@ tr7: goto _test_eof4 } st_case_4: -// line 381 "parse.go" +// line 386 "parse.go" switch data[p] { case 9: goto st4 @@ -422,7 +427,7 @@ tr108: goto _test_eof5 } st_case_5: -// line 426 "parse.go" +// line 431 "parse.go" switch data[p] { case 62: goto st6 @@ -463,7 +468,7 @@ tr109: goto _test_eof6 } st_case_6: -// line 467 "parse.go" +// line 472 "parse.go" switch data[p] { case 9: goto tr14 @@ -493,7 +498,7 @@ tr14: goto _test_eof7 } st_case_7: -// line 497 "parse.go" +// line 502 "parse.go" switch data[p] { case 9: goto st7 @@ -542,7 +547,7 @@ tr79: goto _test_eof8 } st_case_8: -// line 546 "parse.go" +// line 551 "parse.go" switch data[p] { case 34: goto st9 @@ -574,7 +579,7 @@ tr80: goto _test_eof9 } st_case_9: -// line 578 "parse.go" +// line 583 "parse.go" switch data[p] { case 9: goto tr25 @@ -628,7 +633,7 @@ tr96: goto _test_eof10 } st_case_10: -// line 632 "parse.go" +// line 637 "parse.go" switch data[p] { case 9: goto st10 @@ -669,7 +674,7 @@ tr39: goto _test_eof88 } st_case_88: -// line 673 "parse.go" +// line 678 "parse.go" switch data[p] { case 9: goto st88 @@ -690,7 +695,7 @@ tr127: goto _test_eof89 } st_case_89: -// line 694 "parse.go" +// line 699 "parse.go" goto st89 tr27: // line 54 "actions.rl" @@ -727,7 +732,7 @@ tr50: goto _test_eof11 } st_case_11: -// line 731 "parse.go" +// line 736 "parse.go" switch data[p] { case 62: goto st12 @@ -768,7 +773,7 @@ tr51: goto _test_eof12 } st_case_12: -// line 772 "parse.go" +// line 777 "parse.go" switch data[p] { case 9: goto tr38 @@ -794,7 +799,7 @@ tr38: goto _test_eof13 } st_case_13: -// line 798 "parse.go" +// line 803 "parse.go" switch data[p] { case 9: goto st13 @@ -816,7 +821,7 @@ tr52: goto _test_eof14 } st_case_14: -// line 820 "parse.go" +// line 825 "parse.go" switch data[p] { case 85: goto st15 @@ -1029,7 +1034,7 @@ tr34: goto _test_eof24 } st_case_24: -// line 1033 "parse.go" +// line 1038 "parse.go" if data[p] == 58 { goto st25 } @@ -1211,7 +1216,7 @@ tr55: goto _test_eof90 } st_case_90: -// line 1215 "parse.go" +// line 1220 "parse.go" switch data[p] { case 9: goto st88 @@ -1522,7 +1527,7 @@ tr73: goto _test_eof34 } st_case_34: -// line 1526 "parse.go" +// line 1531 "parse.go" switch data[p] { case 62: goto st35 @@ -1563,7 +1568,7 @@ tr74: goto _test_eof35 } st_case_35: -// line 1567 "parse.go" +// line 1572 "parse.go" switch data[p] { case 9: goto tr25 @@ -1589,7 +1594,7 @@ tr75: goto _test_eof36 } st_case_36: -// line 1593 "parse.go" +// line 1598 "parse.go" switch data[p] { case 85: goto st37 @@ -1786,7 +1791,7 @@ tr81: goto _test_eof46 } st_case_46: -// line 1790 "parse.go" +// line 1795 "parse.go" switch data[p] { case 34: goto st47 @@ -2006,7 +2011,7 @@ tr21: goto _test_eof56 } st_case_56: -// line 2010 "parse.go" +// line 2015 "parse.go" if data[p] == 58 { goto st57 } @@ -2190,7 +2195,7 @@ tr90: goto _test_eof91 } st_case_91: -// line 2194 "parse.go" +// line 2199 "parse.go" switch data[p] { case 9: goto st88 @@ -2377,7 +2382,7 @@ tr91: goto _test_eof60 } st_case_60: -// line 2381 "parse.go" +// line 2386 "parse.go" switch data[p] { case 9: goto tr25 @@ -2582,7 +2587,7 @@ tr95: goto _test_eof62 } st_case_62: -// line 2586 "parse.go" +// line 2591 "parse.go" switch data[p] { case 9: goto tr96 @@ -2691,7 +2696,7 @@ tr97: goto _test_eof92 } st_case_92: -// line 2695 "parse.go" +// line 2700 "parse.go" switch data[p] { case 9: goto st88 @@ -2869,7 +2874,7 @@ tr110: goto _test_eof64 } st_case_64: -// line 2873 "parse.go" +// line 2878 "parse.go" switch data[p] { case 85: goto st65 @@ -3066,7 +3071,7 @@ tr122: goto _test_eof74 } st_case_74: -// line 3070 "parse.go" +// line 3075 "parse.go" switch data[p] { case 85: goto st75 @@ -3263,7 +3268,7 @@ tr3: goto _test_eof84 } st_case_84: -// line 3267 "parse.go" +// line 3272 "parse.go" if data[p] == 58 { goto st85 } @@ -3639,14 +3644,14 @@ tr3: return q, nil -// line 3643 "parse.go" +// line 3648 "parse.go" } } _out: {} } -// line 55 "parse.rl" +// line 60 "parse.rl" return quad.Quad{}, quad.ErrInvalid } diff --git a/quad/nquads/parse.rl b/quad/nquads/parse.rl index f1c9cfe8d..ce5e6c920 100644 --- a/quad/nquads/parse.rl +++ b/quad/nquads/parse.rl @@ -33,7 +33,12 @@ import ( write data; }%% -func parse(data []rune) (quad.Quad, error) { +// Parse returns a valid quad.Quad or a non-nil error. Parse does +// handle comments except where the comment placement does not prevent +// a complete valid quad.Quad from being defined. +func Parse(statement string) (quad.Quad, error) { + data := []rune(statement) + var ( cs, p int pe = len(data) diff --git a/quad/quad.go b/quad/quad.go index 2f172c3cb..eaf7d9856 100644 --- a/quad/quad.go +++ b/quad/quad.go @@ -100,9 +100,6 @@ func (d Direction) String() string { } } -// TODO(kortschak) Consider writing methods onto the concrete type -// instead of the pointer. This needs benchmarking to make the decision. - // Per-field accessor for triples func (q Quad) Get(d Direction) string { switch d { @@ -119,10 +116,6 @@ func (q Quad) Get(d Direction) string { } } -func (q Quad) Equals(o Quad) bool { - return q == o -} - // Pretty-prints a triple. func (q Quad) String() string { return fmt.Sprintf("%s -- %s -> %s", q.Subject, q.Predicate, q.Object) @@ -132,10 +125,6 @@ func (q Quad) IsValid() bool { return q.Subject != "" && q.Predicate != "" && q.Object != "" } -// TODO(kortschak) NTriple looks like a good candidate for conversion -// to MarshalText() (text []byte, err error) and then move parsing code -// from nquads to here to provide UnmarshalText(text []byte) error. - // Prints a triple in N-Quad format. func (q Quad) NTriple() string { if q.Label == "" { From 6d676094ee7230407f5ef3e2d059aaf04a7ffb31 Mon Sep 17 00:00:00 2001 From: Iain Collins Date: Tue, 5 Aug 2014 17:08:09 +0100 Subject: [PATCH 12/24] Fixed typo in quick setup command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 628fc7369..0e616a835 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you prefer to build from source, see the documentation on the wiki at [How to `cd` to the directory and give it a quick test with: ``` -./cayley repl --dbpath=testdata.nt +./cayley repl --dbpath=testdata.nq ``` You should see a `cayley>` REPL prompt. Go ahead and give it a try: From 09244ddd38bbfad7bebd3ca35db7b54975a9a84c Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 02:55:36 -0400 Subject: [PATCH 13/24] materialize implementation and and optimization --- graph/iterator.go | 15 ++ graph/iterator/and_iterator_optimize.go | 17 ++ graph/iterator/materialize_iterator.go | 235 ++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 graph/iterator/materialize_iterator.go diff --git a/graph/iterator.go b/graph/iterator.go index 972f3347e..e7c2ad859 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -153,6 +153,19 @@ func Next(it Iterator) (Value, bool) { return nil, false } +// Height is a convienence function to measure the height of an iterator tree. +func Height(it Iterator) int { + subs := it.SubIterators() + maxDepth := 0 + for _, sub := range subs { + h := Height(sub) + if h > maxDepth { + maxDepth = h + } + } + return maxDepth + 1 +} + // FixedIterator wraps iterators that are modifiable by addition of fixed value sets. type FixedIterator interface { Iterator @@ -180,6 +193,7 @@ const ( Fixed Not Optional + Materialize ) var ( @@ -200,6 +214,7 @@ var ( "fixed", "not", "optional", + "materialize", } ) diff --git a/graph/iterator/and_iterator_optimize.go b/graph/iterator/and_iterator_optimize.go index 92b6b4162..f0adfad82 100644 --- a/graph/iterator/and_iterator_optimize.go +++ b/graph/iterator/and_iterator_optimize.go @@ -70,6 +70,8 @@ func (it *And) Optimize() (graph.Iterator, bool) { // now a permutation of itself, but the contents are unchanged. its = optimizeOrder(its) + its = materializeIts(its) + // Okay! At this point we have an optimized order. // The easiest thing to do at this point is merely to create a new And iterator @@ -293,6 +295,21 @@ func hasOneUsefulIterator(its []graph.Iterator) graph.Iterator { return nil } +func materializeIts(its []graph.Iterator) []graph.Iterator { + var out []graph.Iterator + for _, it := range its { + stats := it.Stats() + if stats.Size*stats.NextCost < stats.ContainsCost { + if graph.Height(it) > 10 { + out = append(out, NewMaterialize(it)) + continue + } + } + out = append(out, it) + } + return out +} + // and.Stats() lives here in and-iterator-optimize.go because it may // in the future return different statistics based on how it is optimized. // For now, however, it's pretty static. diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go new file mode 100644 index 000000000..4180a0bc2 --- /dev/null +++ b/graph/iterator/materialize_iterator.go @@ -0,0 +1,235 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iterator + +// A simple iterator that, when first called Contains() or Next() upon, materializes the whole subiterator, stores it locally, and responds. Essentially a cache. + +import ( + "fmt" + "strings" + + "github.com/google/cayley/graph" +) + +var abortMaterializeAt = 1000 + +type result struct { + id graph.Value + tags map[string]graph.Value +} + +type Materialize struct { + uid uint64 + tags graph.Tagger + containsMap map[graph.Value]int + values []result + lastIndex int + subIt graph.Iterator + hasRun bool + aborted bool +} + +func NewMaterialize(sub graph.Iterator) *Materialize { + return &Materialize{ + uid: NextUID(), + containsMap: make(map[graph.Value]int), + subIt: sub, + } +} + +func (it *Materialize) UID() uint64 { + return it.uid +} + +func (it *Materialize) Reset() { + it.subIt.Reset() + it.lastIndex = 0 +} + +func (it *Materialize) Close() { + it.subIt.Close() + it.containsMap = nil + it.values = nil + it.hasRun = false +} + +func (it *Materialize) Tagger() *graph.Tagger { + return &it.tags +} + +func (it *Materialize) TagResults(dst map[string]graph.Value) { + if !it.hasRun { + return + } + for _, tag := range it.tags.Tags() { + dst[tag] = it.Result() + } + + for tag, value := range it.values[it.lastIndex].tags { + dst[tag] = value + } +} + +func (it *Materialize) Clone() graph.Iterator { + out := NewMaterialize(it.subIt.Clone()) + out.tags.CopyFrom(it) + return out +} + +// Print some information about the iterator. +func (it *Materialize) DebugString(indent int) string { + return fmt.Sprintf("%s(%s tags: %s Size: %d\n%s)", + strings.Repeat(" ", indent), + it.Type(), + it.tags.Tags(), + len(it.values), + it.subIt.DebugString(indent+4), + ) +} + +// Register this iterator as a Materialize iterator. +func (it *Materialize) Type() graph.Type { return graph.Materialize } + +// DEPRECATED +func (it *Materialize) ResultTree() *graph.ResultTree { + tree := graph.NewResultTree(it.Result()) + tree.AddSubtree(it.subIt.ResultTree()) + return tree +} + +func (it *Materialize) Result() graph.Value { + if it.lastIndex+1 > len(it.values) { + return nil + } + return it.values[it.lastIndex].id +} + +func (it *Materialize) SubIterators() []graph.Iterator { + return []graph.Iterator{it.subIt} +} + +func (it *Materialize) Optimize() (graph.Iterator, bool) { + newSub, changed := it.subIt.Optimize() + if changed { + it.subIt = newSub + if it.subIt.Type() == graph.Null { + return it.subIt, true + } + } + return it, false +} + +// Size is the number of values stored, if we've got them all. +// Otherwise, guess based on the size of the subiterator. +func (it *Materialize) Size() (int64, bool) { + if it.hasRun { + return int64(len(it.values)), true + } + return it.subIt.Size() +} + +// The entire point of Materialize is to amortize the cost by +// putting it all up front. +func (it *Materialize) Stats() graph.IteratorStats { + overhead := int64(2) + size, _ := it.Size() + subitStats := it.subIt.Stats() + return graph.IteratorStats{ + ContainsCost: overhead * subitStats.NextCost, + NextCost: overhead * subitStats.NextCost, + Size: size, + } +} + +func (it *Materialize) Next() (graph.Value, bool) { + if !it.hasRun { + it.materializeSet() + } + if it.aborted { + return graph.Next(it.subIt) + } + + lastVal := it.Result() + for it.lastIndex < len(it.values) { + it.lastIndex++ + if it.Result() != lastVal { + return it.Result(), true + } + } + return nil, false +} + +func (it *Materialize) Contains(v graph.Value) bool { + if !it.hasRun { + it.materializeSet() + } + if it.aborted { + return it.subIt.Contains(v) + } + if i, ok := it.containsMap[v]; ok { + it.lastIndex = i + return true + } + return false +} + +func (it *Materialize) NextResult() bool { + if !it.hasRun { + it.materializeSet() + } + if it.aborted { + return it.subIt.NextResult() + } + + i := it.lastIndex + 1 + if i == len(it.values) { + return false + } + if it.Result() == it.values[i].id { + it.lastIndex = i + return true + } + return false +} + +func (it *Materialize) materializeSet() { + i := 0 + for { + val, ok := graph.Next(it.subIt) + if !ok { + break + } + i += 1 + if i > abortMaterializeAt { + it.aborted = true + break + } + tags := make(map[string]graph.Value) + it.subIt.TagResults(tags) + it.containsMap[val] = len(it.values) + it.values = append(it.values, result{id: val, tags: tags}) + for it.subIt.NextResult() == true { + tags := make(map[string]graph.Value) + it.subIt.TagResults(tags) + it.values = append(it.values, result{id: val, tags: tags}) + } + } + if it.aborted { + it.values = nil + it.containsMap = nil + it.subIt.Reset() + } + it.hasRun = true +} From 24f57df85932f0a57186030418ac7df45bcbc9a0 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 03:20:33 -0400 Subject: [PATCH 14/24] fix overshoot and optimize better --- cayley_test.go | 2 +- graph/iterator.go | 7 +++++-- graph/iterator/and_iterator_optimize.go | 2 +- graph/iterator/materialize_iterator.go | 20 ++++++++++++++------ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cayley_test.go b/cayley_test.go index b94389a5b..e9410719d 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -348,7 +348,7 @@ func TestQueries(t *testing.T) { // TODO(kortschak) Be more rigorous in this result validation. if len(got) != len(test.expect) { - t.Errorf("Unexpected number of results, got:%d expect:%d.", len(got), len(test.expect)) + t.Errorf("Unexpected number of results, got:%d expect:%d on %s.", len(got), len(test.expect), test.message) } } } diff --git a/graph/iterator.go b/graph/iterator.go index e7c2ad859..3880b80b0 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -154,11 +154,14 @@ func Next(it Iterator) (Value, bool) { } // Height is a convienence function to measure the height of an iterator tree. -func Height(it Iterator) int { +func Height(it Iterator, until Type) int { + if it.Type() == until { + return 1 + } subs := it.SubIterators() maxDepth := 0 for _, sub := range subs { - h := Height(sub) + h := Height(sub, until) if h > maxDepth { maxDepth = h } diff --git a/graph/iterator/and_iterator_optimize.go b/graph/iterator/and_iterator_optimize.go index f0adfad82..774904d59 100644 --- a/graph/iterator/and_iterator_optimize.go +++ b/graph/iterator/and_iterator_optimize.go @@ -300,7 +300,7 @@ func materializeIts(its []graph.Iterator) []graph.Iterator { for _, it := range its { stats := it.Stats() if stats.Size*stats.NextCost < stats.ContainsCost { - if graph.Height(it) > 10 { + if graph.Height(it, graph.Materialize) > 10 { out = append(out, NewMaterialize(it)) continue } diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 4180a0bc2..03360800f 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -73,10 +73,16 @@ func (it *Materialize) TagResults(dst map[string]graph.Value) { if !it.hasRun { return } + if it.aborted { + it.subIt.TagResults(dst) + return + } + if it.lastIndex > len(it.values) { + return + } for _, tag := range it.tags.Tags() { dst[tag] = it.Result() } - for tag, value := range it.values[it.lastIndex].tags { dst[tag] = value } @@ -154,6 +160,7 @@ func (it *Materialize) Stats() graph.IteratorStats { } func (it *Materialize) Next() (graph.Value, bool) { + graph.NextLogIn(it) if !it.hasRun { it.materializeSet() } @@ -164,14 +171,15 @@ func (it *Materialize) Next() (graph.Value, bool) { lastVal := it.Result() for it.lastIndex < len(it.values) { it.lastIndex++ - if it.Result() != lastVal { - return it.Result(), true + if it.Result() != lastVal && it.Result() != nil { + return graph.NextLogOut(it, it.Result(), true) } } - return nil, false + return graph.NextLogOut(it, nil, false) } func (it *Materialize) Contains(v graph.Value) bool { + graph.ContainsLogIn(it, v) if !it.hasRun { it.materializeSet() } @@ -180,9 +188,9 @@ func (it *Materialize) Contains(v graph.Value) bool { } if i, ok := it.containsMap[v]; ok { it.lastIndex = i - return true + return graph.ContainsLogOut(it, v, true) } - return false + return graph.ContainsLogOut(it, v, false) } func (it *Materialize) NextResult() bool { From d10239483672aae49f345483139d02f27f9d887d Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 03:44:01 -0400 Subject: [PATCH 15/24] bounds errors --- graph/iterator/materialize_iterator.go | 32 +++++++++++++++++--------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 03360800f..8434bb53f 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -20,6 +20,8 @@ import ( "fmt" "strings" + "github.com/barakmich/glog" + "github.com/google/cayley/graph" ) @@ -35,7 +37,7 @@ type Materialize struct { tags graph.Tagger containsMap map[graph.Value]int values []result - lastIndex int + index int subIt graph.Iterator hasRun bool aborted bool @@ -46,6 +48,7 @@ func NewMaterialize(sub graph.Iterator) *Materialize { uid: NextUID(), containsMap: make(map[graph.Value]int), subIt: sub, + index: -1, } } @@ -55,7 +58,7 @@ func (it *Materialize) UID() uint64 { func (it *Materialize) Reset() { it.subIt.Reset() - it.lastIndex = 0 + it.index = -1 } func (it *Materialize) Close() { @@ -77,13 +80,13 @@ func (it *Materialize) TagResults(dst map[string]graph.Value) { it.subIt.TagResults(dst) return } - if it.lastIndex > len(it.values) { + if it.Result() == nil { return } for _, tag := range it.tags.Tags() { dst[tag] = it.Result() } - for tag, value := range it.values[it.lastIndex].tags { + for tag, value := range it.values[it.index].tags { dst[tag] = value } } @@ -116,10 +119,16 @@ func (it *Materialize) ResultTree() *graph.ResultTree { } func (it *Materialize) Result() graph.Value { - if it.lastIndex+1 > len(it.values) { + if len(it.values) == 0 { + return nil + } + if it.index == -1 { + return nil + } + if it.index >= len(it.values) { return nil } - return it.values[it.lastIndex].id + return it.values[it.index].id } func (it *Materialize) SubIterators() []graph.Iterator { @@ -169,8 +178,8 @@ func (it *Materialize) Next() (graph.Value, bool) { } lastVal := it.Result() - for it.lastIndex < len(it.values) { - it.lastIndex++ + for it.index < len(it.values) { + it.index++ if it.Result() != lastVal && it.Result() != nil { return graph.NextLogOut(it, it.Result(), true) } @@ -187,7 +196,7 @@ func (it *Materialize) Contains(v graph.Value) bool { return it.subIt.Contains(v) } if i, ok := it.containsMap[v]; ok { - it.lastIndex = i + it.index = i return graph.ContainsLogOut(it, v, true) } return graph.ContainsLogOut(it, v, false) @@ -201,12 +210,12 @@ func (it *Materialize) NextResult() bool { return it.subIt.NextResult() } - i := it.lastIndex + 1 + i := it.index + 1 if i == len(it.values) { return false } if it.Result() == it.values[i].id { - it.lastIndex = i + it.index = i return true } return false @@ -239,5 +248,6 @@ func (it *Materialize) materializeSet() { it.containsMap = nil it.subIt.Reset() } + glog.Infof("Materialization List %d: %#v", it.values) it.hasRun = true } From 76efc2fcb7cb5d1936e0a89b102e01d43dbab95c Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 04:07:30 -0400 Subject: [PATCH 16/24] redo data structure for sensibility --- graph/iterator/materialize_iterator.go | 42 ++++++++++++++------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 8434bb53f..2e41969f1 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -36,8 +36,9 @@ type Materialize struct { uid uint64 tags graph.Tagger containsMap map[graph.Value]int - values []result + values [][]result index int + subindex int subIt graph.Iterator hasRun bool aborted bool @@ -86,7 +87,7 @@ func (it *Materialize) TagResults(dst map[string]graph.Value) { for _, tag := range it.tags.Tags() { dst[tag] = it.Result() } - for tag, value := range it.values[it.index].tags { + for tag, value := range it.values[it.index][it.subindex].tags { dst[tag] = value } } @@ -128,7 +129,7 @@ func (it *Materialize) Result() graph.Value { if it.index >= len(it.values) { return nil } - return it.values[it.index].id + return it.values[it.index][it.subindex].id } func (it *Materialize) SubIterators() []graph.Iterator { @@ -177,14 +178,12 @@ func (it *Materialize) Next() (graph.Value, bool) { return graph.Next(it.subIt) } - lastVal := it.Result() - for it.index < len(it.values) { - it.index++ - if it.Result() != lastVal && it.Result() != nil { - return graph.NextLogOut(it, it.Result(), true) - } + it.index++ + it.subindex = 0 + if it.index >= len(it.values) { + return graph.NextLogOut(it, nil, false) } - return graph.NextLogOut(it, nil, false) + return graph.NextLogOut(it, it.Result(), true) } func (it *Materialize) Contains(v graph.Value) bool { @@ -197,6 +196,7 @@ func (it *Materialize) Contains(v graph.Value) bool { } if i, ok := it.containsMap[v]; ok { it.index = i + it.subindex = 0 return graph.ContainsLogOut(it, v, true) } return graph.ContainsLogOut(it, v, false) @@ -210,15 +210,13 @@ func (it *Materialize) NextResult() bool { return it.subIt.NextResult() } - i := it.index + 1 - if i == len(it.values) { + it.subindex++ + if it.subindex >= len(it.values[it.index]) { + // Don't go off the end of the world + it.subindex-- return false } - if it.Result() == it.values[i].id { - it.index = i - return true - } - return false + return true } func (it *Materialize) materializeSet() { @@ -233,14 +231,18 @@ func (it *Materialize) materializeSet() { it.aborted = true break } + if _, ok := it.containsMap[val]; !ok { + it.containsMap[val] = len(it.values) + it.values = append(it.values, nil) + } + index := it.containsMap[val] tags := make(map[string]graph.Value) it.subIt.TagResults(tags) - it.containsMap[val] = len(it.values) - it.values = append(it.values, result{id: val, tags: tags}) + it.values[index] = append(it.values[index], result{id: val, tags: tags}) for it.subIt.NextResult() == true { tags := make(map[string]graph.Value) it.subIt.TagResults(tags) - it.values = append(it.values, result{id: val, tags: tags}) + it.values[index] = append(it.values[index], result{id: val, tags: tags}) } } if it.aborted { From 0c3e0381f3ef8d70ce07db85ece5299d3898b6c1 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 05:03:20 -0400 Subject: [PATCH 17/24] Fix the integration test from crashing Sessions are expected to only have one running query (perhaps this is a bug). So we need to make a new session for each of the benchmark runs, timing only the running part. --- cayley_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cayley_test.go b/cayley_test.go index b94389a5b..15428a867 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -363,12 +363,18 @@ func runBench(n int, b *testing.B) { if err != nil { b.Fatalf("Failed to parse benchmark gremlin %s: %v", benchmarkQueries[n].message, err) } + b.StopTimer() b.ResetTimer() for i := 0; i < b.N; i++ { c := make(chan interface{}, 5) + ses := gremlin.NewSession(ts, cfg.Timeout, true) + // Do the parsing we know works. + ses.InputParses(benchmarkQueries[n].query) + b.StartTimer() go ses.ExecInput(benchmarkQueries[n].query, c, 100) for _ = range c { } + b.StopTimer() } } From 11891f02a978b50d0b3aa03a7fab9dec380e0019 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 05:19:46 -0400 Subject: [PATCH 18/24] remove lead-in parsing test --- cayley_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cayley_test.go b/cayley_test.go index 15428a867..96461e78c 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -358,11 +358,6 @@ func runBench(n int, b *testing.B) { b.Skip() } prepare(b) - ses := gremlin.NewSession(ts, cfg.Timeout, true) - _, err := ses.InputParses(benchmarkQueries[n].query) - if err != nil { - b.Fatalf("Failed to parse benchmark gremlin %s: %v", benchmarkQueries[n].message, err) - } b.StopTimer() b.ResetTimer() for i := 0; i < b.N; i++ { From 2bec255b52cbf113269a1f71849f673296261b5b Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 14:37:37 -0400 Subject: [PATCH 19/24] Copy refs and comment on Value --- graph/iterator/materialize_iterator.go | 6 ++++++ graph/triplestore.go | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 2e41969f1..1b448b3e2 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -95,6 +95,12 @@ func (it *Materialize) TagResults(dst map[string]graph.Value) { func (it *Materialize) Clone() graph.Iterator { out := NewMaterialize(it.subIt.Clone()) out.tags.CopyFrom(it) + if it.hasRun { + out.hasRun = true + out.aborted = it.aborted + out.values = it.values + out.containsMap = it.containsMap + } return out } diff --git a/graph/triplestore.go b/graph/triplestore.go index c099d4789..bfdf62f8e 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -28,14 +28,17 @@ import ( "github.com/google/cayley/quad" ) -// Defines an opaque "triple store value" type. However the backend wishes to -// implement it, a Value is merely a token to a triple or a node that the backing -// store itself understands, and the base iterators pass around. +// Value defines an opaque "triple store value" type. However the backend wishes +// to implement it, a Value is merely a token to a triple or a node that the +// backing store itself understands, and the base iterators pass around. // // For example, in a very traditional, graphd-style graph, these are int64s // (guids of the primitives). In a very direct sort of graph, these could be // pointers to structs, or merely triples, or whatever works best for the // backing store. +// +// These must be comparable, ie, not arrays or maps, as they may be used as keys +// for maps. type Value interface{} type TripleStore interface { From 6f1e46c2d88ce50bbfaf78089310c4ba2e15f0fd Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 15:34:30 -0400 Subject: [PATCH 20/24] Fix hashability for graph.Value --- graph/iterator/materialize_iterator.go | 10 +++++++++- graph/leveldb/all_iterator.go | 4 ++-- graph/leveldb/iterator.go | 24 ++++++++++++++++-------- graph/leveldb/triplestore.go | 22 ++++++++++++++-------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 1b448b3e2..8bd2a5355 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -32,6 +32,10 @@ type result struct { tags map[string]graph.Value } +type hashable interface { + Hashable() interface{} +} + type Materialize struct { uid uint64 tags graph.Tagger @@ -228,7 +232,7 @@ func (it *Materialize) NextResult() bool { func (it *Materialize) materializeSet() { i := 0 for { - val, ok := graph.Next(it.subIt) + id, ok := graph.Next(it.subIt) if !ok { break } @@ -237,6 +241,10 @@ func (it *Materialize) materializeSet() { it.aborted = true break } + val := id + if h, ok := id.(hashable); ok { + val = h.Hashable() + } if _, ok := it.containsMap[val]; !ok { it.containsMap[val] = len(it.values) it.values = append(it.values, nil) diff --git a/graph/leveldb/all_iterator.go b/graph/leveldb/all_iterator.go index 5346f97da..a810792d9 100644 --- a/graph/leveldb/all_iterator.go +++ b/graph/leveldb/all_iterator.go @@ -117,8 +117,8 @@ func (it *AllIterator) Next() (graph.Value, bool) { it.Close() return nil, false } - it.result = out - return out, true + it.result = Token(out) + return it.result, true } func (it *AllIterator) ResultTree() *graph.ResultTree { diff --git a/graph/leveldb/iterator.go b/graph/leveldb/iterator.go index b434dfd92..720b664e8 100644 --- a/graph/leveldb/iterator.go +++ b/graph/leveldb/iterator.go @@ -42,7 +42,7 @@ type Iterator struct { } func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator { - vb := value.([]byte) + vb := value.(Token) p := make([]byte, 0, 2+qs.hasher.Size()) p = append(p, []byte(prefix)...) p = append(p, []byte(vb[1:])...) @@ -105,7 +105,7 @@ func (it *Iterator) TagResults(dst map[string]graph.Value) { } func (it *Iterator) Clone() graph.Iterator { - out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs) + out := NewIterator(it.originalPrefix, it.dir, Token(it.checkId), it.qs) out.tags.CopyFrom(it) return out } @@ -134,12 +134,12 @@ func (it *Iterator) Next() (graph.Value, bool) { if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) { out := make([]byte, len(it.iter.Key())) copy(out, it.iter.Key()) - it.result = out + it.result = Token(out) ok := it.iter.Next() if !ok { it.Close() } - return out, true + return Token(out), true } it.Close() it.result = nil @@ -216,7 +216,7 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int { } func (it *Iterator) Contains(v graph.Value) bool { - val := v.([]byte) + val := v.(Token) if val[0] == 'z' { return false } @@ -227,7 +227,7 @@ func (it *Iterator) Contains(v graph.Value) bool { } } else { nameForDir := it.qs.Quad(v).Get(it.dir) - hashForDir := it.qs.ValueOf(nameForDir).([]byte) + hashForDir := it.qs.ValueOf(nameForDir).(Token) if bytes.Equal(hashForDir, it.checkId) { return true } @@ -236,12 +236,20 @@ func (it *Iterator) Contains(v graph.Value) bool { } func (it *Iterator) Size() (int64, bool) { - return it.qs.SizeOf(it.checkId), true + return it.qs.SizeOf(Token(it.checkId)), true } func (it *Iterator) DebugString(indent int) string { size, _ := it.Size() - return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.tags.Tags(), it.dir, size, it.qs.NameOf(it.checkId)) + return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", + strings.Repeat(" ", indent), + it.Type(), + it.UID(), + it.tags.Tags(), + it.dir, + size, + it.qs.NameOf(Token(it.checkId)), + ) } var levelDBType graph.Type diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 600b36077..8ba7bde95 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -42,6 +42,12 @@ const ( DefaultWriteBufferSize = 20 ) +type Token []byte + +func (t Token) Hashable() interface{} { + return string(t) +} + type TripleStore struct { dbOpts *opt.Options db *leveldb.DB @@ -308,7 +314,7 @@ func (qs *TripleStore) Close() { func (qs *TripleStore) Quad(k graph.Value) quad.Quad { var triple quad.Quad - b, err := qs.db.Get(k.([]byte), qs.readopts) + b, err := qs.db.Get(k.(Token), qs.readopts) if err != nil && err != leveldb.ErrNotFound { glog.Error("Error: couldn't get triple from DB.") return quad.Quad{} @@ -334,7 +340,7 @@ func (qs *TripleStore) convertStringToByteHash(s string) []byte { } func (qs *TripleStore) ValueOf(s string) graph.Value { - return qs.createValueKeyFor(s) + return Token(qs.createValueKeyFor(s)) } func (qs *TripleStore) valueData(value_key []byte) ValueData { @@ -362,14 +368,14 @@ func (qs *TripleStore) NameOf(k graph.Value) string { glog.V(2).Info("k was nil") return "" } - return qs.valueData(k.([]byte)).Name + return qs.valueData(k.(Token)).Name } func (qs *TripleStore) SizeOf(k graph.Value) int64 { if k == nil { return 0 } - return int64(qs.valueData(k.([]byte)).Size) + return int64(qs.valueData(k.(Token)).Size) } func (qs *TripleStore) getSize() { @@ -432,17 +438,17 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator { } func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value { - v := val.([]uint8) + v := val.(Token) offset := PositionOf(v[0:2], d, qs) if offset != -1 { - return append([]byte("z"), v[offset:offset+qs.hasher.Size()]...) + return Token(append([]byte("z"), v[offset:offset+qs.hasher.Size()]...)) } else { - return qs.Quad(val).Get(d) + return Token(qs.Quad(val).Get(d)) } } func compareBytes(a, b graph.Value) bool { - return bytes.Equal(a.([]uint8), b.([]uint8)) + return bytes.Equal(a.(Token), b.(Token)) } func (qs *TripleStore) FixedIterator() graph.FixedIterator { From 840c3412742d3af76e52b1b397bff47817f919d8 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 15:37:42 -0400 Subject: [PATCH 21/24] Fix ID Lookup --- graph/iterator/materialize_iterator.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 8bd2a5355..e488a31e9 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -204,7 +204,11 @@ func (it *Materialize) Contains(v graph.Value) bool { if it.aborted { return it.subIt.Contains(v) } - if i, ok := it.containsMap[v]; ok { + key := v + if h, ok := v.(hashable); ok { + key = h.Hashable() + } + if i, ok := it.containsMap[key]; ok { it.index = i it.subindex = 0 return graph.ContainsLogOut(it, v, true) @@ -252,11 +256,11 @@ func (it *Materialize) materializeSet() { index := it.containsMap[val] tags := make(map[string]graph.Value) it.subIt.TagResults(tags) - it.values[index] = append(it.values[index], result{id: val, tags: tags}) + it.values[index] = append(it.values[index], result{id: id, tags: tags}) for it.subIt.NextResult() == true { tags := make(map[string]graph.Value) it.subIt.TagResults(tags) - it.values[index] = append(it.values[index], result{id: val, tags: tags}) + it.values[index] = append(it.values[index], result{id: id, tags: tags}) } } if it.aborted { From 4d9dd42dd8e704343102d7e7b394991f5e31c5cb Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 15:51:31 -0400 Subject: [PATCH 22/24] comment --- graph/triplestore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graph/triplestore.go b/graph/triplestore.go index bfdf62f8e..195b6875e 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -37,8 +37,8 @@ import ( // pointers to structs, or merely triples, or whatever works best for the // backing store. // -// These must be comparable, ie, not arrays or maps, as they may be used as keys -// for maps. +// These must be comparable, or implement a `Hashable() interface{}` function +// so that they may be stored in maps. type Value interface{} type TripleStore interface { From 2b431851174d9aee9f2086a84d967d1a5aa78705 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 15:57:00 -0400 Subject: [PATCH 23/24] rename Hashable/Hasher --- graph/iterator/materialize_iterator.go | 12 ++++++------ graph/leveldb/triplestore.go | 2 +- graph/triplestore.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index e488a31e9..46baf0c6b 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -32,8 +32,8 @@ type result struct { tags map[string]graph.Value } -type hashable interface { - Hashable() interface{} +type hasher interface { + Hasher() interface{} } type Materialize struct { @@ -205,8 +205,8 @@ func (it *Materialize) Contains(v graph.Value) bool { return it.subIt.Contains(v) } key := v - if h, ok := v.(hashable); ok { - key = h.Hashable() + if h, ok := v.(hasher); ok { + key = h.Hasher() } if i, ok := it.containsMap[key]; ok { it.index = i @@ -246,8 +246,8 @@ func (it *Materialize) materializeSet() { break } val := id - if h, ok := id.(hashable); ok { - val = h.Hashable() + if h, ok := id.(hasher); ok { + val = h.Hasher() } if _, ok := it.containsMap[val]; !ok { it.containsMap[val] = len(it.values) diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 8ba7bde95..2bd127918 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -44,7 +44,7 @@ const ( type Token []byte -func (t Token) Hashable() interface{} { +func (t Token) Hasher() interface{} { return string(t) } diff --git a/graph/triplestore.go b/graph/triplestore.go index 195b6875e..5a6884164 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -37,7 +37,7 @@ import ( // pointers to structs, or merely triples, or whatever works best for the // backing store. // -// These must be comparable, or implement a `Hashable() interface{}` function +// These must be comparable, or implement a `Hasher() interface{}` function // so that they may be stored in maps. type Value interface{} From b74cb142f02bc7a0079cf080de1bd4c112b2acab Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 6 Aug 2014 16:59:46 -0400 Subject: [PATCH 24/24] Key/Keyer --- graph/iterator/materialize_iterator.go | 12 ++++++------ graph/leveldb/triplestore.go | 2 +- graph/triplestore.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 46baf0c6b..f74059412 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -32,8 +32,8 @@ type result struct { tags map[string]graph.Value } -type hasher interface { - Hasher() interface{} +type Keyer interface { + Key() interface{} } type Materialize struct { @@ -205,8 +205,8 @@ func (it *Materialize) Contains(v graph.Value) bool { return it.subIt.Contains(v) } key := v - if h, ok := v.(hasher); ok { - key = h.Hasher() + if h, ok := v.(Keyer); ok { + key = h.Key() } if i, ok := it.containsMap[key]; ok { it.index = i @@ -246,8 +246,8 @@ func (it *Materialize) materializeSet() { break } val := id - if h, ok := id.(hasher); ok { - val = h.Hasher() + if h, ok := id.(Keyer); ok { + val = h.Key() } if _, ok := it.containsMap[val]; !ok { it.containsMap[val] = len(it.values) diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 2bd127918..1a305d2bd 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -44,7 +44,7 @@ const ( type Token []byte -func (t Token) Hasher() interface{} { +func (t Token) Key() interface{} { return string(t) } diff --git a/graph/triplestore.go b/graph/triplestore.go index 5a6884164..b25ca7479 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -37,7 +37,7 @@ import ( // pointers to structs, or merely triples, or whatever works best for the // backing store. // -// These must be comparable, or implement a `Hasher() interface{}` function +// These must be comparable, or implement a `Key() interface{}` function // so that they may be stored in maps. type Value interface{}