diff --git a/elasticsearch/elasticsearch.go b/elasticsearch/elasticsearch.go index d7f180a..8e34b24 100644 --- a/elasticsearch/elasticsearch.go +++ b/elasticsearch/elasticsearch.go @@ -16,35 +16,44 @@ const ( maxErrorsReported = 20 ) +// PackageExists returns whether the provided loadpath + version tuple +// exists in this index. +func PackageExists(loadpath string, version int64, client *elastic.Client) bool { + ctx := context.Background() + docID := fmt.Sprintf("v1@%d@%s", version, loadpath) + pkgDoc, _ := client.Get(). + Index("goref"). + Type("package"). + Id(docID). + Do(ctx) + // TODO: handle errors better. Right now we assume that any + // error is a 404 and can be ignored safely. + return pkgDoc != nil +} + +// LoadGraphToElastic loads all Packages and Refs from a PackageGraph +// to the provided ES index. func LoadGraphToElastic(pg goref.PackageGraph, client *elastic.Client) ([]*goref.Ref, error) { ctx := context.Background() missedRefs := make([]*goref.Ref, 0) errs := make([]error, 0) for _, p := range pg.Packages { - docID := p.DocumentID() - pkgDoc, err := client.Get(). - Index("goref"). - Type("package"). - Id(docID). - Do(ctx) - - if err == nil { - return nil, err - } + log.Infof("Processing package %s", p.Path) - if pkgDoc != nil { - log.Infof("Package %s already exists in this index.", docID) + if PackageExists(p.Path, p.Version, client) { + log.Infof("Package %s already exists in this index.", p) continue } - log.Infof("Creating Package document %s in the index", docID) + log.Infof("Creating Package %s in the index", p) if _, err := client.Index(). Index("goref"). Type("package"). - Id(docID). + Id(p.DocumentID()). BodyJson(p). Do(ctx); err != nil { + log.Infof("2 %s", err) return nil, err } diff --git a/elasticsearch/main/main.go b/elasticsearch/main/main.go index ed6de69..11d2923 100644 --- a/elasticsearch/main/main.go +++ b/elasticsearch/main/main.go @@ -1,27 +1,79 @@ package main import ( - "log" + "flag" "github.com/korfuri/goref" "github.com/korfuri/goref/elasticsearch" + log "github.com/sirupsen/logrus" elastic "gopkg.in/olivere/elastic.v5" ) +const ( + Usage = `elastic_goref -version 42 -include_tests \\ + -elastic_url http://localhost:9200/ -elastic_user elastic -elastic_password changeme \\ + github.com/korfuri/goref github.com/korfuri/goref/elastic/main` +) + +var ( + version = flag.Int64("version", -1, + "Version of the code being examined. Should increase monotonically when the code is updated.") + includeTests = flag.Bool("include_tests", true, + "Whether XTest packages should be included in the index.") + elasticUrl = flag.String("elastic_url", "http://localhost:9200", + "URL of the ElasticSearch cluster.") + elasticUsername = flag.String("elastic_user", "elastic", + "Username to authenticate with ElasticSearch.") + elasticPassword = flag.String("elastic_password", "changeme", + "Password to authenticate with ElasticSearch.") +) + +func usage() { + log.Fatal(Usage) +} + func main() { + flag.Parse() + args := flag.Args() + + if *version == -1 || len(args) == 0 { + usage() + } + // Create a client client, err := elastic.NewClient( - elastic.SetURL("http://localhost:9200"), - elastic.SetBasicAuth("elastic", "changeme")) + elastic.SetURL(*elasticUrl), + elastic.SetBasicAuth(*elasticUsername, *elasticPassword)) if err != nil { log.Fatal(err) } + // Filter out packages that already exist at this version in + // the index. + packages := make([]string, 0) + for _, a := range args { + if !elasticsearch.PackageExists(a, *version, client) { + packages = append(packages, a) + } + } + + // Index the requested packages + log.Infof("Indexing packages: %v", packages) + if *includeTests { + log.Info("This index will include XTests.") + } pg := goref.NewPackageGraph(0) - pg.LoadPrograms([]string{"github.com/korfuri/goref/main/main"}, true) + pg.LoadPrograms(packages, *includeTests) + log.Info("Computing the interface-implementation matrix.") pg.ComputeInterfaceImplementationMatrix() + log.Infof("%d packages in the graph.", len(pg.Packages)) + log.Infof("%d files in the graph.", len(pg.Files)) + + // Load the indexed references into ElasticSearch + log.Info("Inserting references into ElasticSearch.") if missed, err := elasticsearch.LoadGraphToElastic(*pg, client); err != nil { log.Fatalf("Couldn't load %d references. Error: %s", len(missed), err) } + log.Info("Done, bye.") }