Skip to content
Permalink
Browse files

Add --new_uids option to live loader.

  • Loading branch information...
Javier Alvarado
Javier Alvarado committed Feb 15, 2019
1 parent 3db2c82 commit 18277872f4bac4f8c91f2cbac94db34fd56715c2
@@ -0,0 +1,69 @@
[
{
"uid":"0x2001",
"name":"Homer",
"age":"38",
"role":"father",
"parent_to": [
{ "uid":"0x3001" },
{ "uid":"0x3002" },
{ "uid":"0x3003" }
]
},
{
"uid":"0x2101",
"name":"Marge",
"age":"34",
"role":"mother",
"aka":"Midge",
"parent_to": [
{ "uid":"0x3001" },
{ "uid":"0x3002" },
{ "uid":"0x3003" }
]
},
{
"uid":"0x3001",
"name":"Bart",
"age":"10",
"role":"son",
"aka":"El Barto",
"carries":"slingshot",
"sibling_of": [
{ "uid":"0x3002" },
{ "uid":"0x3003" }
]
},
{
"uid":"0x3002",
"name":"Lisa",
"age":"8",
"role":"daughter",
"carries":"saxomophone",
"sibling_of": [
{ "uid":"0x3001" },
{ "uid":"0x3003" }
]
},
{
"uid":"0x3003",
"name":"Maggie",
"age":"1",
"role":"daughter",
"carries":"pacifier",
"sibling_of": [
{ "uid":"0x3001" },
{ "uid":"0x3002" }
]
},
{
"uid":"0x1001",
"name":"Abraham",
"age":"83",
"role":"father",
"aka":"Grampa",
"parent_to": [
{ "uid":"0x2001" }
]
}
]
@@ -0,0 +1,42 @@
<0x1001> <age> "83"^^<xs:int> .
<0x1001> <aka> "Grampa"^^<xs:string> .
<0x1001> <name> "Abraham"^^<xs:string> .
<0x1001> <parent_to> <0x2001> .
<0x1001> <role> "father"^^<xs:string> .
#
<0x2001> <age> "38"^^<xs:int> .
<0x2001> <name> "Homer"^^<xs:string> .
<0x2001> <parent_to> <0x3001> .
<0x2001> <parent_to> <0x3002> .
<0x2001> <parent_to> <0x3003> .
<0x2001> <role> "father"^^<xs:string> .
#
<0x2101> <age> "34"^^<xs:int> .
<0x2101> <aka> "Midge"^^<xs:string> .
<0x2101> <name> "Marge"^^<xs:string> .
<0x2101> <parent_to> <0x3001> .
<0x2101> <parent_to> <0x3002> .
<0x2101> <parent_to> <0x3003> .
<0x2101> <role> "mother"^^<xs:string> .
#
<0x3001> <age> "10"^^<xs:int> .
<0x3001> <aka> "El Barto"^^<xs:string> .
<0x3001> <carries> "slingshot"^^<xs:string> .
<0x3001> <name> "Bart"^^<xs:string> .
<0x3001> <role> "son"^^<xs:string> .
<0x3001> <sibling_of> <0x3002> .
<0x3001> <sibling_of> <0x3003> .
#
<0x3002> <age> "8"^^<xs:int> .
<0x3002> <carries> "saxomophone"^^<xs:string> .
<0x3002> <name> "Lisa"^^<xs:string> .
<0x3002> <role> "daughter"^^<xs:string> .
<0x3002> <sibling_of> <0x3001> .
<0x3002> <sibling_of> <0x3003> .
#
<0x3003> <age> "1"^^<xs:int> .
<0x3003> <carries> "pacifier"^^<xs:string> .
<0x3003> <name> "Maggie"^^<xs:string> .
<0x3003> <role> "daughter"^^<xs:string> .
<0x3003> <sibling_of> <0x3001> .
<0x3003> <sibling_of> <0x3002> .
@@ -0,0 +1,7 @@
name:string @index(term) .
age: int .
role:string @index(term) .
aka:string @index(term) .
carries:string @index(term) .
parent_to: [uid] @reverse .
sibling_of: [uid] @reverse .
@@ -0,0 +1,186 @@
/*
* Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package main

import (
"context"
"io/ioutil"
"os"
"path"
"runtime"
"testing"

"github.com/dgraph-io/dgo"
"github.com/stretchr/testify/require"

"github.com/dgraph-io/dgraph/x"
"github.com/dgraph-io/dgraph/z"
)

const alphaService = ":9180"

var (
testDataDir string
dg *dgo.Dgraph
tmpDir string
)

func checkDifferentUid(t *testing.T, wantMap, gotMap map[string]interface{}) {
require.NotEqual(t, gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
"new uid was assigned")

gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
z.CompareJSONMaps(t, wantMap, gotMap)
}

func checkLoadedData(t *testing.T, newUids bool) {
resp, err := dg.NewTxn().Query(context.Background(), `
{
q(func: anyofterms(name, "Homer")) {
uid
name
age
role
}
}
`)
require.NoError(t, err)

gotMap := z.UnmarshalJSON(t, string(resp.GetJson()))
wantMap := z.UnmarshalJSON(t, `
{
"q": [
{
"uid": "0x2001",
"name": "Homer",
"age": 38,
"role": "father"
}
]
}
`)
if newUids {
checkDifferentUid(t, wantMap, gotMap)
} else {
z.CompareJSONMaps(t, wantMap, gotMap)
}

resp, err = dg.NewTxn().Query(context.Background(), `
{
q(func: anyofterms(name, "Maggie")) {
uid
name
role
carries
}
}
`)
require.NoError(t, err)

gotMap = z.UnmarshalJSON(t, string(resp.GetJson()))
wantMap = z.UnmarshalJSON(t, `
{
"q": [
{
"uid": "0x3003",
"name": "Maggie",
"role": "daughter",
"carries": "pacifier"
}
]
}
`)
if newUids {
checkDifferentUid(t, wantMap, gotMap)
} else {
z.CompareJSONMaps(t, wantMap, gotMap)
}
}

func TestLiveLoadJsonUidKeep(t *testing.T) {
z.DropAll(t, dg)

pipeline := [][]string{
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
"--dgraph", alphaService},
}
err := z.Pipeline(pipeline)
require.NoError(t, err, "live loading JSON file ran successfully")

checkLoadedData(t, false)
}

func TestLiveLoadJsonUidDiscard(t *testing.T) {
z.DropAll(t, dg)

pipeline := [][]string{
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
"--dgraph", alphaService},
}
err := z.Pipeline(pipeline)
require.NoError(t, err, "live loading JSON file ran successfully")

checkLoadedData(t, true)
}

func TestLiveLoadRdfUidKeep(t *testing.T) {
z.DropAll(t, dg)

pipeline := [][]string{
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
"--dgraph", alphaService},
}
err := z.Pipeline(pipeline)
require.NoError(t, err, "live loading JSON file ran successfully")

checkLoadedData(t, false)
}

func TestLiveLoadRdfUidDiscard(t *testing.T) {
z.DropAll(t, dg)

pipeline := [][]string{
{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
"--dgraph", alphaService},
}
err := z.Pipeline(pipeline)
require.NoError(t, err, "live loading JSON file ran successfully")

checkLoadedData(t, true)
}

func TestMain(m *testing.M) {
_, thisFile, _, _ := runtime.Caller(0)
testDataDir = path.Dir(thisFile)

dg = z.DgraphClient(alphaService)

// Try to create any files in a dedicated temp directory that gets cleaned up
// instead of all over /tmp or the working directory.
tmpDir, err := ioutil.TempDir("", "test.tmp-")
x.Check(err)
os.Chdir(tmpDir)
defer os.RemoveAll(tmpDir)

os.Exit(m.Run())
}
@@ -60,6 +60,7 @@ type options struct {
ignoreIndexConflict bool
authToken string
useCompression bool
newUids bool
}

var (
@@ -98,6 +99,8 @@ func init() {
"The auth token passed to the server for Alter operation of the schema file")
flag.BoolP("use_compression", "C", false,
"Enable compression on connection to alpha server")
flag.Bool("new_uids", false, ""+
"Ignore UIDs in load files and assign new ones.")

// TLS configuration
x.RegisterTLSFlags(flag)
@@ -134,7 +137,7 @@ func processSchemaFile(ctx context.Context, file string, dgraphClient *dgo.Dgrap
}

f, err := os.Open(file)
x.Check(err)
x.CheckfNoTrace(err)
defer f.Close()

var reader io.Reader
@@ -161,9 +164,11 @@ func (l *loader) uid(val string) string {
// to be an existing node in the graph. There is limited protection against
// a user selecting an unassigned UID in this way - it may be assigned
// later to another node. It is up to the user to avoid this.
if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
l.alloc.BumpTo(uid)
return fmt.Sprintf("%#x", uid)
if !opt.newUids {
if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
l.alloc.BumpTo(uid)
return fmt.Sprintf("%#x", uid)
}
}

uid := l.alloc.AssignUid(val)
@@ -292,6 +297,7 @@ func run() error {
ignoreIndexConflict: Live.Conf.GetBool("ignore_index_conflict"),
authToken: Live.Conf.GetString("auth_token"),
useCompression: Live.Conf.GetBool("use_compression"),
newUids: Live.Conf.GetBool("new_uids"),
}
x.LoadTLSConfig(&tlsConf, Live.Conf, x.TlsClientCert, x.TlsClientKey)
tlsConf.ServerName = Live.Conf.GetString("tls_server_name")
@@ -26,8 +26,11 @@ import (
)

// for debugging the tests
const showOutput = false
const showCommands = false

var (
showOutput bool = os.Getenv("DEBUG_SHOW_OUTPUT") != ""
showCommand bool = os.Getenv("DEBUG_SHOW_COMMAND") != ""
)

// Pipeline runs several commands such that the output of one command becomes the input of the next.
// The first argument should be an two-dimensional array containing the commands.
@@ -41,7 +44,7 @@ func Pipeline(cmds [][]string) error {
// Run all commands in parallel, connecting stdin of each to the stdout of the previous.
for i, c := range cmds {
lastCmd := i == numCmds-1
if showCommands {
if showCommand {
fmt.Fprintf(os.Stderr, "%+v", c)
}

@@ -58,7 +61,7 @@ func Pipeline(cmds [][]string) error {
}
}

if showCommands {
if showCommand {
if lastCmd {
fmt.Fprintf(os.Stderr, "\n")
} else {

0 comments on commit 1827787

Please sign in to comment.
You can’t perform that action at this time.