Update

tok/embedflag.go

@@ -2,11 +2,43 @@
 
 package tok
 
-// #cgo CPPFLAGS: -I./icu/common -DU_DISABLE_RENAMING=1
+// #cgo CPPFLAGS: -I${SRCDIR}/icu/icuembed -DU_DISABLE_RENAMING=1
 // #cgo darwin LDFLAGS: -Wl,-undefined -Wl,dynamic_lookup
 // #cgo !darwin LDFLAGS: -Wl,-unresolved-symbols=ignore-all -lrt
+// #include <unicode/utypes.h>
+// #include <unicode/udata.h>
 import "C"
 
 import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"time"
+	"unsafe"
+
 	_ "github.com/dgraph-io/dgraph/tok/icu/icuembed"
+	"github.com/dgraph-io/dgraph/x"
+)
+
+var (
+	icuDataFile = flag.String("icu", "",
+		"Location of ICU data file such as icudt57l.dat.")
+	icuData []byte // Hold a reference.
 )
+
+func init() {
+	x.AddInit(func() {
+		x.Assertf(len(*icuDataFile) > 0, "ICU data file empty")
+
+		start := time.Now()
+		icuData, err := ioutil.ReadFile(*icuDataFile)
+		x.Check(err)
+
+		var icuErr C.UErrorCode
+		C.udata_setCommonData(unsafe.Pointer(byteToChar(icuData)), &icuErr)
+		x.Assertf(int(icuErr) >= 0, "Error occurred with udata_setCommonData: %d",
+			int(icuErr))
+
+		log.Printf("Loaded ICU data from [%s] in %s", *icuDataFile, time.Since(start))
+	})
+}

tok/icu/gen.sh

@@ -23,8 +23,10 @@ cp -Rf icu/source/common $TARGET
 cp -f icu/source/stubdata/stubdata.c $TARGET/common
 mv $TARGET/common $TARGET/icuembed
 
-cp $DIR/$DATA $TARGET
+cp -f $DIR/$DATA $TARGET
 
-# Add in some Go files.
+# Add in some Go file(s).
 cd $TARGET
 cp -f icuembed.go.tmpl icuembed/icuembed.go
+
+echo -e "Please run dgraph with the flag:\n-icu $TARGET/$DATA"

tok/tok_test.go

@@ -17,8 +17,11 @@
 package tok
 
 import (
+	"os"
 	"strings"
 	"testing"
+
+	"github.com/dgraph-io/dgraph/x"
 )
 
 func TestTokenizeBasic(t *testing.T) {
@@ -72,3 +75,8 @@ func TestTokenizeBasic(t *testing.T) {
 		}(d.in, d.expected)
 	}
 }
+
+func TestMain(m *testing.M) {
+	x.Init()
+	os.Exit(m.Run())
+}