Skip to content

Commit 1827787

Browse files
author
Javier Alvarado
committed
Add --new_uids option to live loader.
1 parent 3db2c82 commit 1827787

7 files changed

Lines changed: 333 additions & 15 deletions

File tree

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
[
2+
{
3+
"uid":"0x2001",
4+
"name":"Homer",
5+
"age":"38",
6+
"role":"father",
7+
"parent_to": [
8+
{ "uid":"0x3001" },
9+
{ "uid":"0x3002" },
10+
{ "uid":"0x3003" }
11+
]
12+
},
13+
{
14+
"uid":"0x2101",
15+
"name":"Marge",
16+
"age":"34",
17+
"role":"mother",
18+
"aka":"Midge",
19+
"parent_to": [
20+
{ "uid":"0x3001" },
21+
{ "uid":"0x3002" },
22+
{ "uid":"0x3003" }
23+
]
24+
},
25+
{
26+
"uid":"0x3001",
27+
"name":"Bart",
28+
"age":"10",
29+
"role":"son",
30+
"aka":"El Barto",
31+
"carries":"slingshot",
32+
"sibling_of": [
33+
{ "uid":"0x3002" },
34+
{ "uid":"0x3003" }
35+
]
36+
},
37+
{
38+
"uid":"0x3002",
39+
"name":"Lisa",
40+
"age":"8",
41+
"role":"daughter",
42+
"carries":"saxomophone",
43+
"sibling_of": [
44+
{ "uid":"0x3001" },
45+
{ "uid":"0x3003" }
46+
]
47+
},
48+
{
49+
"uid":"0x3003",
50+
"name":"Maggie",
51+
"age":"1",
52+
"role":"daughter",
53+
"carries":"pacifier",
54+
"sibling_of": [
55+
{ "uid":"0x3001" },
56+
{ "uid":"0x3002" }
57+
]
58+
},
59+
{
60+
"uid":"0x1001",
61+
"name":"Abraham",
62+
"age":"83",
63+
"role":"father",
64+
"aka":"Grampa",
65+
"parent_to": [
66+
{ "uid":"0x2001" }
67+
]
68+
}
69+
]
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<0x1001> <age> "83"^^<xs:int> .
2+
<0x1001> <aka> "Grampa"^^<xs:string> .
3+
<0x1001> <name> "Abraham"^^<xs:string> .
4+
<0x1001> <parent_to> <0x2001> .
5+
<0x1001> <role> "father"^^<xs:string> .
6+
#
7+
<0x2001> <age> "38"^^<xs:int> .
8+
<0x2001> <name> "Homer"^^<xs:string> .
9+
<0x2001> <parent_to> <0x3001> .
10+
<0x2001> <parent_to> <0x3002> .
11+
<0x2001> <parent_to> <0x3003> .
12+
<0x2001> <role> "father"^^<xs:string> .
13+
#
14+
<0x2101> <age> "34"^^<xs:int> .
15+
<0x2101> <aka> "Midge"^^<xs:string> .
16+
<0x2101> <name> "Marge"^^<xs:string> .
17+
<0x2101> <parent_to> <0x3001> .
18+
<0x2101> <parent_to> <0x3002> .
19+
<0x2101> <parent_to> <0x3003> .
20+
<0x2101> <role> "mother"^^<xs:string> .
21+
#
22+
<0x3001> <age> "10"^^<xs:int> .
23+
<0x3001> <aka> "El Barto"^^<xs:string> .
24+
<0x3001> <carries> "slingshot"^^<xs:string> .
25+
<0x3001> <name> "Bart"^^<xs:string> .
26+
<0x3001> <role> "son"^^<xs:string> .
27+
<0x3001> <sibling_of> <0x3002> .
28+
<0x3001> <sibling_of> <0x3003> .
29+
#
30+
<0x3002> <age> "8"^^<xs:int> .
31+
<0x3002> <carries> "saxomophone"^^<xs:string> .
32+
<0x3002> <name> "Lisa"^^<xs:string> .
33+
<0x3002> <role> "daughter"^^<xs:string> .
34+
<0x3002> <sibling_of> <0x3001> .
35+
<0x3002> <sibling_of> <0x3003> .
36+
#
37+
<0x3003> <age> "1"^^<xs:int> .
38+
<0x3003> <carries> "pacifier"^^<xs:string> .
39+
<0x3003> <name> "Maggie"^^<xs:string> .
40+
<0x3003> <role> "daughter"^^<xs:string> .
41+
<0x3003> <sibling_of> <0x3001> .
42+
<0x3003> <sibling_of> <0x3002> .
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name:string @index(term) .
2+
age: int .
3+
role:string @index(term) .
4+
aka:string @index(term) .
5+
carries:string @index(term) .
6+
parent_to: [uid] @reverse .
7+
sibling_of: [uid] @reverse .
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/*
2+
* Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"context"
21+
"io/ioutil"
22+
"os"
23+
"path"
24+
"runtime"
25+
"testing"
26+
27+
"github.com/dgraph-io/dgo"
28+
"github.com/stretchr/testify/require"
29+
30+
"github.com/dgraph-io/dgraph/x"
31+
"github.com/dgraph-io/dgraph/z"
32+
)
33+
34+
const alphaService = ":9180"
35+
36+
var (
37+
testDataDir string
38+
dg *dgo.Dgraph
39+
tmpDir string
40+
)
41+
42+
func checkDifferentUid(t *testing.T, wantMap, gotMap map[string]interface{}) {
43+
require.NotEqual(t, gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
44+
wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
45+
"new uid was assigned")
46+
47+
gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
48+
wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
49+
z.CompareJSONMaps(t, wantMap, gotMap)
50+
}
51+
52+
func checkLoadedData(t *testing.T, newUids bool) {
53+
resp, err := dg.NewTxn().Query(context.Background(), `
54+
{
55+
q(func: anyofterms(name, "Homer")) {
56+
uid
57+
name
58+
age
59+
role
60+
}
61+
}
62+
`)
63+
require.NoError(t, err)
64+
65+
gotMap := z.UnmarshalJSON(t, string(resp.GetJson()))
66+
wantMap := z.UnmarshalJSON(t, `
67+
{
68+
"q": [
69+
{
70+
"uid": "0x2001",
71+
"name": "Homer",
72+
"age": 38,
73+
"role": "father"
74+
}
75+
]
76+
}
77+
`)
78+
if newUids {
79+
checkDifferentUid(t, wantMap, gotMap)
80+
} else {
81+
z.CompareJSONMaps(t, wantMap, gotMap)
82+
}
83+
84+
resp, err = dg.NewTxn().Query(context.Background(), `
85+
{
86+
q(func: anyofterms(name, "Maggie")) {
87+
uid
88+
name
89+
role
90+
carries
91+
}
92+
}
93+
`)
94+
require.NoError(t, err)
95+
96+
gotMap = z.UnmarshalJSON(t, string(resp.GetJson()))
97+
wantMap = z.UnmarshalJSON(t, `
98+
{
99+
"q": [
100+
{
101+
"uid": "0x3003",
102+
"name": "Maggie",
103+
"role": "daughter",
104+
"carries": "pacifier"
105+
}
106+
]
107+
}
108+
`)
109+
if newUids {
110+
checkDifferentUid(t, wantMap, gotMap)
111+
} else {
112+
z.CompareJSONMaps(t, wantMap, gotMap)
113+
}
114+
}
115+
116+
func TestLiveLoadJsonUidKeep(t *testing.T) {
117+
z.DropAll(t, dg)
118+
119+
pipeline := [][]string{
120+
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
121+
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
122+
"--dgraph", alphaService},
123+
}
124+
err := z.Pipeline(pipeline)
125+
require.NoError(t, err, "live loading JSON file ran successfully")
126+
127+
checkLoadedData(t, false)
128+
}
129+
130+
func TestLiveLoadJsonUidDiscard(t *testing.T) {
131+
z.DropAll(t, dg)
132+
133+
pipeline := [][]string{
134+
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
135+
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
136+
"--dgraph", alphaService},
137+
}
138+
err := z.Pipeline(pipeline)
139+
require.NoError(t, err, "live loading JSON file ran successfully")
140+
141+
checkLoadedData(t, true)
142+
}
143+
144+
func TestLiveLoadRdfUidKeep(t *testing.T) {
145+
z.DropAll(t, dg)
146+
147+
pipeline := [][]string{
148+
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
149+
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
150+
"--dgraph", alphaService},
151+
}
152+
err := z.Pipeline(pipeline)
153+
require.NoError(t, err, "live loading JSON file ran successfully")
154+
155+
checkLoadedData(t, false)
156+
}
157+
158+
func TestLiveLoadRdfUidDiscard(t *testing.T) {
159+
z.DropAll(t, dg)
160+
161+
pipeline := [][]string{
162+
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
163+
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
164+
"--dgraph", alphaService},
165+
}
166+
err := z.Pipeline(pipeline)
167+
require.NoError(t, err, "live loading JSON file ran successfully")
168+
169+
checkLoadedData(t, true)
170+
}
171+
172+
func TestMain(m *testing.M) {
173+
_, thisFile, _, _ := runtime.Caller(0)
174+
testDataDir = path.Dir(thisFile)
175+
176+
dg = z.DgraphClient(alphaService)
177+
178+
// Try to create any files in a dedicated temp directory that gets cleaned up
179+
// instead of all over /tmp or the working directory.
180+
tmpDir, err := ioutil.TempDir("", "test.tmp-")
181+
x.Check(err)
182+
os.Chdir(tmpDir)
183+
defer os.RemoveAll(tmpDir)
184+
185+
os.Exit(m.Run())
186+
}

dgraph/cmd/live/run.go

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ type options struct {
6060
ignoreIndexConflict bool
6161
authToken string
6262
useCompression bool
63+
newUids bool
6364
}
6465

6566
var (
@@ -98,6 +99,8 @@ func init() {
9899
"The auth token passed to the server for Alter operation of the schema file")
99100
flag.BoolP("use_compression", "C", false,
100101
"Enable compression on connection to alpha server")
102+
flag.Bool("new_uids", false, ""+
103+
"Ignore UIDs in load files and assign new ones.")
101104

102105
// TLS configuration
103106
x.RegisterTLSFlags(flag)
@@ -134,7 +137,7 @@ func processSchemaFile(ctx context.Context, file string, dgraphClient *dgo.Dgrap
134137
}
135138

136139
f, err := os.Open(file)
137-
x.Check(err)
140+
x.CheckfNoTrace(err)
138141
defer f.Close()
139142

140143
var reader io.Reader
@@ -161,9 +164,11 @@ func (l *loader) uid(val string) string {
161164
// to be an existing node in the graph. There is limited protection against
162165
// a user selecting an unassigned UID in this way - it may be assigned
163166
// later to another node. It is up to the user to avoid this.
164-
if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
165-
l.alloc.BumpTo(uid)
166-
return fmt.Sprintf("%#x", uid)
167+
if !opt.newUids {
168+
if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
169+
l.alloc.BumpTo(uid)
170+
return fmt.Sprintf("%#x", uid)
171+
}
167172
}
168173

169174
uid := l.alloc.AssignUid(val)
@@ -292,6 +297,7 @@ func run() error {
292297
ignoreIndexConflict: Live.Conf.GetBool("ignore_index_conflict"),
293298
authToken: Live.Conf.GetString("auth_token"),
294299
useCompression: Live.Conf.GetBool("use_compression"),
300+
newUids: Live.Conf.GetBool("new_uids"),
295301
}
296302
x.LoadTLSConfig(&tlsConf, Live.Conf, x.TlsClientCert, x.TlsClientKey)
297303
tlsConf.ServerName = Live.Conf.GetString("tls_server_name")

z/exec.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@ import (
2626
)
2727

2828
// for debugging the tests
29-
const showOutput = false
30-
const showCommands = false
29+
30+
var (
31+
showOutput bool = os.Getenv("DEBUG_SHOW_OUTPUT") != ""
32+
showCommand bool = os.Getenv("DEBUG_SHOW_COMMAND") != ""
33+
)
3134

3235
// Pipeline runs several commands such that the output of one command becomes the input of the next.
3336
// The first argument should be an two-dimensional array containing the commands.
@@ -41,7 +44,7 @@ func Pipeline(cmds [][]string) error {
4144
// Run all commands in parallel, connecting stdin of each to the stdout of the previous.
4245
for i, c := range cmds {
4346
lastCmd := i == numCmds-1
44-
if showCommands {
47+
if showCommand {
4548
fmt.Fprintf(os.Stderr, "%+v", c)
4649
}
4750

@@ -58,7 +61,7 @@ func Pipeline(cmds [][]string) error {
5861
}
5962
}
6063

61-
if showCommands {
64+
if showCommand {
6265
if lastCmd {
6366
fmt.Fprintf(os.Stderr, "\n")
6467
} else {

0 commit comments

Comments
 (0)