Make the elasticsearch utility useful.

It is now a full command-line utility that accepts a list of packages
on the command line, as well as information on how to connect to
ElasticSearch and wether to include tests in the analyzed graph.

elasticsearch/elasticsearch.go

@@ -16,35 +16,44 @@ const (
 	maxErrorsReported = 20
 )
 
+// PackageExists returns whether the provided loadpath + version tuple
+// exists in this index.
+func PackageExists(loadpath string, version int64, client *elastic.Client) bool {
+	ctx := context.Background()
+	docID := fmt.Sprintf("v1@%d@%s", version, loadpath)
+	pkgDoc, _ := client.Get().
+		Index("goref").
+		Type("package").
+		Id(docID).
+		Do(ctx)
+	// TODO: handle errors better. Right now we assume that any
+	// error is a 404 and can be ignored safely.
+	return pkgDoc != nil
+}
+
+// LoadGraphToElastic loads all Packages and Refs from a PackageGraph
+// to the provided ES index.
 func LoadGraphToElastic(pg goref.PackageGraph, client *elastic.Client) ([]*goref.Ref, error) {
 	ctx := context.Background()
 	missedRefs := make([]*goref.Ref, 0)
 	errs := make([]error, 0)
 
 	for _, p := range pg.Packages {
-		docID := p.DocumentID()
-		pkgDoc, err := client.Get().
-			Index("goref").
-			Type("package").
-			Id(docID).
-			Do(ctx)
-
-		if err == nil {
-			return nil, err
-		}
+		log.Infof("Processing package %s", p.Path)
 
-		if pkgDoc != nil {
-			log.Infof("Package %s already exists in this index.", docID)
+		if PackageExists(p.Path, p.Version, client) {
+			log.Infof("Package %s already exists in this index.", p)
 			continue
 		}
 
-		log.Infof("Creating Package document %s in the index", docID)
+		log.Infof("Creating Package %s in the index", p)
 		if _, err := client.Index().
 			Index("goref").
 			Type("package").
-			Id(docID).
+			Id(p.DocumentID()).
 			BodyJson(p).
 			Do(ctx); err != nil {
+			log.Infof("2 %s", err)
 			return nil, err
 		}
 

elasticsearch/main/main.go

@@ -1,27 +1,79 @@
 package main
 
 import (
-	"log"
+	"flag"
 
 	"github.com/korfuri/goref"
 	"github.com/korfuri/goref/elasticsearch"
+	log "github.com/sirupsen/logrus"
 	elastic "gopkg.in/olivere/elastic.v5"
 )
 
+const (
+	Usage = `elastic_goref -version 42 -include_tests <true|false> \\
+  -elastic_url http://localhost:9200/ -elastic_user elastic -elastic_password changeme \\
+  github.com/korfuri/goref github.com/korfuri/goref/elastic/main`
+)
+
+var (
+	version = flag.Int64("version", -1,
+		"Version of the code being examined. Should increase monotonically when the code is updated.")
+	includeTests = flag.Bool("include_tests", true,
+		"Whether XTest packages should be included in the index.")
+	elasticUrl = flag.String("elastic_url", "http://localhost:9200",
+		"URL of the ElasticSearch cluster.")
+	elasticUsername = flag.String("elastic_user", "elastic",
+		"Username to authenticate with ElasticSearch.")
+	elasticPassword = flag.String("elastic_password", "changeme",
+		"Password to authenticate with ElasticSearch.")
+)
+
+func usage() {
+	log.Fatal(Usage)
+}
+
 func main() {
+	flag.Parse()
+	args := flag.Args()
+
+	if *version == -1 || len(args) == 0 {
+		usage()
+	}
+
 	// Create a client
 	client, err := elastic.NewClient(
-		elastic.SetURL("http://localhost:9200"),
-		elastic.SetBasicAuth("elastic", "changeme"))
+		elastic.SetURL(*elasticUrl),
+		elastic.SetBasicAuth(*elasticUsername, *elasticPassword))
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	// Filter out packages that already exist at this version in
+	// the index.
+	packages := make([]string, 0)
+	for _, a := range args {
+		if !elasticsearch.PackageExists(a, *version, client) {
+			packages = append(packages, a)
+		}
+	}
+
+	// Index the requested packages
+	log.Infof("Indexing packages: %v", packages)
+	if *includeTests {
+		log.Info("This index will include XTests.")
+	}
 	pg := goref.NewPackageGraph(0)
-	pg.LoadPrograms([]string{"github.com/korfuri/goref/main/main"}, true)
+	pg.LoadPrograms(packages, *includeTests)
+	log.Info("Computing the interface-implementation matrix.")
 	pg.ComputeInterfaceImplementationMatrix()
 
+	log.Infof("%d packages in the graph.", len(pg.Packages))
+	log.Infof("%d files in the graph.", len(pg.Files))
+
+	// Load the indexed references into ElasticSearch
+	log.Info("Inserting references into ElasticSearch.")
 	if missed, err := elasticsearch.LoadGraphToElastic(*pg, client); err != nil {
 		log.Fatalf("Couldn't load %d references. Error: %s", len(missed), err)
 	}
+	log.Info("Done, bye.")
 }