Track a few anonymous metrics to measure Dgraph adoption (#2554)

- Assign a unique Cluster ID to each Dgraph cluster.
- Track the size of cluster, i.e. number of Zero, Alpha nodes, groups, tablets, total size of tablets, OS, architecture and Dgraph version.

None of this has any uniquely identifying information, just generic anonymous stats which can be aggregated to get a sense of Dgraph's adoption curve.

Author: manishrjain

dgraph/cmd/zero/raft.go

@@ -22,6 +22,8 @@ import (
 	"github.com/dgraph-io/dgraph/conn"
 	"github.com/dgraph-io/dgraph/protos/intern"
 	"github.com/dgraph-io/dgraph/x"
+	"github.com/golang/glog"
+	"github.com/google/uuid"
 	"golang.org/x/net/context"
 	"golang.org/x/net/trace"
 )
@@ -178,6 +180,12 @@ func (n *node) applyProposal(e raftpb.Entry) (string, error) {
 
 	state := n.server.state
 	state.Counter = e.Index
+	if len(p.Cid) > 0 {
+		if len(state.Cid) > 0 {
+			return p.Key, errInvalidProposal
+		}
+		state.Cid = p.Cid
+	}
 	if p.MaxRaftId > 0 {
 		if p.MaxRaftId <= state.MaxRaftId {
 			return p.Key, errInvalidProposal
@@ -411,6 +419,23 @@ func (n *node) initAndStartNode() error {
 		x.Check(err)
 		peers := []raft.Peer{{ID: n.Id, Context: data}}
 		n.SetRaft(raft.StartNode(n.Cfg, peers))
+
+		go func() {
+			// This is a new cluster. So, propose a new ID for the cluster.
+			for {
+				id := uuid.New().String()
+				err := n.proposeAndWait(context.Background(), &intern.ZeroProposal{Cid: id})
+				if err == nil {
+					glog.Infof("CID set for cluster: %v", id)
+					return
+				}
+				if err == errInvalidProposal {
+					return
+				}
+				glog.Errorf("While proposing CID: %v. Retrying...", err)
+				time.Sleep(3 * time.Second)
+			}
+		}()
 	}
 
 	go n.Run()

dgraph/cmd/zero/run.go

@@ -72,6 +72,7 @@ instances to achieve high-availability.
 	flag.String("peer", "", "Address of another dgraphzero server.")
 	flag.StringP("wal", "w", "zw", "Directory storing WAL.")
 	flag.Duration("rebalance_interval", 8*time.Minute, "Interval for trying a predicate move.")
+	flag.Bool("telemetry", true, "Send anonymous telemetry data to Dgraph devs.")
 }
 
 func setupListener(addr string, port int, kind string) (listener net.Listener, err error) {
@@ -196,6 +197,10 @@ func run() {
 	// This must be here. It does not work if placed before Grpc init.
 	x.Check(st.node.initAndStartNode())
 
+	if Zero.Conf.GetBool("telemetry") {
+		go st.zero.periodicallyPostTelemetry()
+	}
+
 	sdCh := make(chan os.Signal, 1)
 	signal.Notify(sdCh, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
 

dgraph/cmd/zero/telemetry.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2018 Dgraph Labs, Inc.
+ *
+ * This file is available under the Apache License, Version 2.0,
+ * with the Commons Clause restriction.
+ */
+
+package zero
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+
+	"github.com/dgraph-io/dgraph/protos/intern"
+	"github.com/dgraph-io/dgraph/x"
+	"github.com/golang/glog"
+)
+
+type Telemetry struct {
+	Arch        string
+	Cid         string
+	ClusterSize int
+	DiskUsageMB int64
+	NumAlphas   int
+	NumGroups   int
+	NumTablets  int
+	NumZeros    int
+	OS          string
+	SinceHours  int
+	Version     string
+}
+
+var keenUrl = "https://api.keen.io/3.0/projects/5b809dfac9e77c0001783ad0/events"
+
+func newTelemetry(ms *intern.MembershipState) *Telemetry {
+	if len(ms.Cid) == 0 {
+		glog.V(2).Infoln("No CID found yet")
+		return nil
+	}
+	t := &Telemetry{
+		Cid:       ms.Cid,
+		NumGroups: len(ms.GetGroups()),
+		NumZeros:  len(ms.GetZeros()),
+		Version:   x.Version(),
+		OS:        runtime.GOOS,
+		Arch:      runtime.GOARCH,
+	}
+	for _, g := range ms.GetGroups() {
+		t.NumAlphas += len(g.GetMembers())
+		for _, tablet := range g.GetTablets() {
+			t.NumTablets++
+			t.DiskUsageMB += tablet.GetSpace()
+		}
+	}
+	t.DiskUsageMB /= (1 << 20)
+	t.ClusterSize = t.NumAlphas + t.NumZeros
+	glog.V(2).Infof("Posting Telemetry data: %+v", t)
+	return t
+}
+
+func (t *Telemetry) post() error {
+	data, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+	url := keenUrl + "/dev"
+	if len(t.Version) > 0 {
+		url = keenUrl + "/pings"
+	}
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "D0398E8C83BB30F67C519FDA6175975F680921890C35B36C34BE109544597497CA758881BD7D56CC2355A2F36B4560102CBC3279AC7B27E5391372C36A31167EB0D06BF3764894AD20A0554BAFF14C292A40BC252BB9FF008736A0FD1D44E085")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	glog.V(2).Infof("Telemetry response status: %v", resp.Status)
+	glog.V(2).Infof("Telemetry response body: %s", body)
+	return nil
+}

dgraph/cmd/zero/zero.go

@@ -19,6 +19,7 @@ import (
 	"github.com/dgraph-io/dgraph/protos/intern"
 	"github.com/dgraph-io/dgraph/x"
 	"github.com/gogo/protobuf/proto"
+	"github.com/golang/glog"
 )
 
 var (
@@ -74,6 +75,37 @@ func (s *Server) Init() {
 	go s.purgeOracle()
 }
 
+func (s *Server) periodicallyPostTelemetry() {
+	glog.V(2).Infof("Starting telemetry data collection...")
+	start := time.Now()
+
+	ticker := time.NewTicker(time.Minute)
+	defer ticker.Stop()
+
+	var lastPostedAt time.Time
+	for range ticker.C {
+		if !s.Node.AmLeader() {
+			continue
+		}
+		if time.Since(lastPostedAt) < time.Hour {
+			continue
+		}
+		ms := s.membershipState()
+		t := newTelemetry(ms)
+		if t == nil {
+			continue
+		}
+		t.SinceHours = int(time.Since(start).Hours())
+		glog.V(2).Infof("Posting Telemetry data: %+v", t)
+
+		err := t.post()
+		glog.V(2).Infof("Telemetry data posted with error: %v", err)
+		if err == nil {
+			lastPostedAt = time.Now()
+		}
+	}
+}
+
 func (s *Server) triggerLeaderChange() {
 	s.Lock()
 	defer s.Unlock()

dgraph/docker-compose.yml

@@ -5,13 +5,13 @@
 version: "3.5"
 services:
   zero1:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.1
     working_dir: /data/dg0.1
     ports:
       - 5080:5080
       - 6080:6080
-    command: /gobin/dgraph zero --my=zero1:5080 --replicas 3 --idx 1 --bindall --expose_trace --profile_mode block --block_rate 10
+    command: /gobin/dgraph zero --my=zero1:5080 --replicas 3 --idx 1 --bindall --expose_trace --profile_mode block --block_rate 10 --logtostderr -v=2
     volumes:
       - type: bind
         source: $GOPATH/bin
@@ -22,7 +22,7 @@ services:
       #   target: /data
 
   zero2:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.2
     command: /gobin/dgraph zero -o 2 --my=zero2:5082 --replicas 3 --peer=zero1:5080 --idx 2
     ports:
@@ -35,7 +35,7 @@ services:
         read_only: true
 
   zero3:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.3
     command: /gobin/dgraph zero -o 3 --my=zero3:5083 --replicas 3 --peer=zero1:5080 --idx 3
     ports:
@@ -49,6 +49,7 @@ services:
 
   # zero4:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg0.4
   #   command: /gobin/dgraph zero --my=zero4:5080 --replicas 3 --peer=zero1:5080 --idx 4
   #   volumes:
@@ -59,6 +60,7 @@ services:
 
   # zero5:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg0.5
   #   command: /gobin/dgraph zero --my=zero5:5080 --replicas 3 --peer=zero1:5080 --idx 5
   #   volumes:
@@ -68,7 +70,7 @@ services:
   #       read_only: true
 
   dg1:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg1
     working_dir: /data/dg1
     volumes:
@@ -85,7 +87,7 @@ services:
     command: /gobin/dgraph server --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 -o 100 --expose_trace  --trace 1.0 --profile_mode block --block_rate 10
 
   dg2:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg2
     working_dir: /data/dg2
     volumes:
@@ -102,7 +104,7 @@ services:
     command: /gobin/dgraph server --my=dg2:7182 --lru_mb=1024 --zero=zero1:5080 -o 102 --expose_trace  --trace 1.0 --profile_mode block --block_rate 10
 
   dg3:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg3
     working_dir: /data/dg3
     volumes:
@@ -120,6 +122,7 @@ services:
 
   # dg4:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg4
   #   working_dir: /data/dg4
   #   volumes:
@@ -137,6 +140,7 @@ services:
 
   # dg5:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg5
   #   working_dir: /data/dg5
   #   volumes: