Determine package version on a per-package basis.

This has two drawbacks currently:

- The Elastic importer can't filter packages before adding them to the
  graph. To determine their version they would need to be imported,
  which would be somewhat costly.
- Right now versions are int64s, representing the mtime in nanoseconds
  from the Go Epoch. But ElasticSearch uses the JS convention of
  storing large numbers in float64s, which can't handle recent MTimes
  without very significant loss of precision.

dotimports_test.go

@@ -13,7 +13,7 @@ func TestDotImports(t *testing.T) {
 		pkgpath = "github.com/korfuri/goref/testprograms/dotimports"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgpath}, true)
 	assert.Contains(t, pg.Packages, pkgpath)
 	assert.Contains(t, pg.Packages, pkgpath+"/lib")

elasticsearch/main/main.go

@@ -10,14 +10,12 @@ import (
 )
 
 const (
-	Usage = `elastic_goref -version 42 -include_tests <true|false> \\
+	Usage = `elastic_goref -include_tests <true|false> \\
   -elastic_url http://localhost:9200/ -elastic_user elastic -elastic_password changeme \\
   github.com/korfuri/goref github.com/korfuri/goref/elastic/main`
 )
 
 var (
-	version = flag.Int64("version", -1,
-		"Version of the code being examined. Should increase monotonically when the code is updated.")
 	includeTests = flag.Bool("include_tests", true,
 		"Whether XTest packages should be included in the index.")
 	elasticUrl = flag.String("elastic_url", "http://localhost:9200",
@@ -38,7 +36,7 @@ func main() {
 	flag.Parse()
 	args := flag.Args()
 
-	if *version == -1 || len(args) == 0 {
+	if len(args) == 0 {
 		usage()
 	}
 
@@ -50,21 +48,26 @@ func main() {
 		log.Fatal(err)
 	}
 
-	// Filter out packages that already exist at this version in
-	// the index.
-	packages := make([]string, 0)
-	for _, a := range args {
-		if !elasticsearch.PackageExists(a, *version, client, *elasticIndex) {
-			packages = append(packages, a)
-		}
-	}
-
+	// // TODO(korfuri): This is broken now that versions are //
+	// // per-package and generated after processing a package's
+	// // files.
+	// //
+	// // Filter out packages that already exist at this version in
+	// // the index.
+	// packages := make([]string, 0)
+	// for _, a := range args {
+	// 	if !elasticsearch.PackageExists(a, *version, client, *elasticIndex) {
+	// 		packages = append(packages, a)
+	// 	}
+	// }
+	packages := args
+
 	// Index the requested packages
 	log.Infof("Indexing packages: %v", packages)
 	if *includeTests {
 		log.Info("This index will include XTests.")
 	}
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.FileMTimeVersion)
 	pg.LoadPrograms(packages, *includeTests)
 	log.Info("Computing the interface-implementation matrix.")
 	pg.ComputeInterfaceImplementationMatrix()

empty_test.go

@@ -12,7 +12,7 @@ func TestCanImportEmptyPackage(t *testing.T) {
 		emptypkgpath = "github.com/korfuri/goref/testprograms/empty"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{emptypkgpath}, false)
 	assert.Len(t, pg.Packages, 1)
 	assert.Len(t, pg.Files, 1)

interfaces_test.go

@@ -13,7 +13,7 @@ func TestInterfaceImplMatrix(t *testing.T) {
 		pkgpath = "github.com/korfuri/goref/testprograms/interfaces"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgpath}, false)
 	assert.Contains(t, pg.Packages, pkgpath)
 	pg.ComputeInterfaceImplementationMatrix()

main/main.go

@@ -27,8 +27,8 @@ func main() {
 
 	start := time.Now()
 
-	m := goref.NewPackageGraph(0)
-	m.LoadPrograms([]string{"github.com/korfuri/goref/main/main"}, true)
+	m := goref.NewPackageGraph(goref.FileMTimeVersion)
+	m.LoadPrograms([]string{"github.com/korfuri/goref/main"}, true)
 
 	log.Printf("Loading took %s\n", time.Since(start))
 	reportMemory()

multiple_mains_test.go

@@ -13,7 +13,7 @@ func TestMultipleMainsFromMultipleCalls(t *testing.T) {
 		filebase = "testprograms/multiple_mains/"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgbase + "1"}, false)
 	assert.Contains(t, pg.Packages, pkgbase+"1")
 	assert.Contains(t, pg.Packages, pkgbase+"common")
@@ -33,7 +33,7 @@ func TestMultipleMainsFromSameCalls(t *testing.T) {
 		filebase = "testprograms/multiple_mains/"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgbase + "1", pkgbase + "2"}, false)
 	assert.Contains(t, pg.Packages, pkgbase+"1")
 	assert.Contains(t, pg.Packages, pkgbase+"common")

packagegraph.go

@@ -20,10 +20,15 @@ type PackageGraph struct {
 	// Map of file path to File objects.
 	Files map[string]*File
 
-	// version is passed to all packages loaded in this
-	// graph. This assumes that all packages we'll load are loaded
-	// from the same snapshot of the Go universe.
-	version int64
+	// versionF is a function that returns the version of the
+	// provided Go package as an int64.
+	//
+	// A simple way to implement it, for a graph that will not
+	// outlive its packages' versions, is to provide a function
+	// that always returns 0. A more correct way to implement it
+	// is to look at the package's files and check their mtime.
+	// Both are provided in packagegraph_utils.go.
+	versionF func(loader.Program, loader.PackageInfo) (int64, error)
 }
 
 // CleanImportSpec takes an ast.ImportSpec and cleans the Path
@@ -74,14 +79,55 @@ func CandidatePaths(loadpath, parent string) []string {
 	return paths
 }
 
+// specialPackage returns hardcoded packages for packages wihtout a Go
+// implementation. Currently this is only "unsafe".
+func specialPackage(loadpath string) *Package {
+	if loadpath == "unsafe" {
+		return &Package{
+			Name:       "unsafe",
+			Path:       "unsafe",
+			InRefs:     make([]*Ref, 0),
+
+			// The Files map must be empty as no files define that package.
+			//
+			// There exists a src/unsafe/unsafe.go but it
+			// is not loaded by go/loader, and so
+			// referencing it would cause more issues than
+			// not (as no Fileset would contain its
+			// contents)
+			Files:      make(map[string]*File),
+			Fset:       nil,
+
+			// We assume Unsafe has no interfaces, types or outrefs.
+			OutRefs:    make([]*Ref, 0),
+			Interfaces: make([]*types.Named, 0),
+			Impls:      make([]*types.Named, 0),
+
+			// We assume Unsafe only has one version and
+			// that version is immutable. If Unsafe's API
+			// changes that's OK, since anything can
+			// reference it at this version already.
+			Version:    1,
+		}
+	}
+	return nil
+}
+
 // loadPackage recursively loads a Go package into the Package
 // Graph. If the package was already loaded, it returns early. It
 // always returns the Package object for the loaded package.
 func (pg *PackageGraph) loadPackage(prog *loader.Program, loadpath string, pi *loader.PackageInfo) *Package {
+	if pkg := specialPackage(loadpath); pkg != nil {
+		return pkg
+	}
 	if pkg, in := pg.Packages[loadpath]; in {
 		return pkg
 	}
-	pkg := newPackage(pi, prog.Fset, pg.version)
+	version, err := pg.versionF(*prog, *pi)
+	if err != nil {
+		return nil
+	}
+	pkg := newPackage(pi, prog.Fset, version)
 	pg.Packages[loadpath] = pkg
 
 	// Iterate over all files in that package.
@@ -114,7 +160,14 @@ func (pg *PackageGraph) loadPackage(prog *loader.Program, loadpath string, pi *l
 				continue
 			}
 			importedPkg := pg.loadPackage(prog, ipath, i)
-
+			if importedPkg == nil {
+				// This happens if versionF fails to
+				// determine the pacakge's version.
+				// Often this would mean that the
+				// package is a C package.
+				continue
+			}
+
 			// Set up the edges on the package dependency graph
 			var importAs string
 			// If the import is unqualified
@@ -269,10 +322,10 @@ func (pg *PackageGraph) ComputeInterfaceImplementationMatrix() {
 }
 
 // NewPackageGraph returns a new, empty PackageGraph.
-func NewPackageGraph(version int64) *PackageGraph {
+func NewPackageGraph(versionF func(loader.Program, loader.PackageInfo) (int64, error)) *PackageGraph {
 	return &PackageGraph{
 		Packages: make(map[string]*Package),
 		Files:    make(map[string]*File),
-		version:  version,
+		versionF: versionF,
 	}
 }

packagegraph_utils.go

@@ -0,0 +1,43 @@
+package goref
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"golang.org/x/tools/go/loader"
+)
+
+// ConstantVersion returns a versionF function that always replies
+// with a constant version. Useful for experimenting, or for graphs
+// who load from an immutable snapshot of the Go universe.
+func ConstantVersion(v int64) (func(loader.Program, loader.PackageInfo) (int64, error)) {
+	return func(prog loader.Program, pi loader.PackageInfo) (int64, error) {
+		return v, nil
+	}
+}
+
+// FileMTimeVersion is a versionF function that processes all files in
+// the provided PackageInfo and returns the newest mtime's second as a
+// time.Time-compatible int64.
+func FileMTimeVersion(prog loader.Program, pi loader.PackageInfo) (int64, error) {
+	newestMTime := time.Time{}
+	for _, f := range pi.Files {
+		filepath := prog.Fset.File(f.Package).Name()
+		fi, err := os.Stat(filepath)
+		if err != nil {
+			return -1, err
+		}
+		fmt.Printf("%s mtime: %v\n", filepath, fi.ModTime())
+		if fi.ModTime().After(newestMTime) {
+			fmt.Printf("win!\n")
+			newestMTime = fi.ModTime()
+		}
+	}
+	if newestMTime == (time.Time{}) {
+		return -1, fmt.Errorf("Unable to determine the version of package %s", pi.Pkg.Path())
+	}
+	// newestMTime - epoch gives us a duration which is an int64
+	// of nanoseconds since the Go epoch (1/1/1 00:00:00 UTC).
+	return int64(newestMTime.UTC().Sub(time.Time{})), nil
+}

simple_test.go

@@ -14,7 +14,7 @@ func TestSimplePackage(t *testing.T) {
 		filepath = "testprograms/simple/main.go"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgpath}, false)
 	assert.Contains(t, pg.Packages, pkgpath)
 	assert.Contains(t, pg.Packages, "fmt")

vendored_test.go

@@ -13,7 +13,7 @@ func TestVendoredPackage(t *testing.T) {
 		filepath = "testprograms/vendored/main.go"
 	)
 
-	pg := goref.NewPackageGraph(0)
+	pg := goref.NewPackageGraph(goref.ConstantVersion(0))
 	pg.LoadPrograms([]string{pkgpath}, false)
 	assert.Contains(t, pg.Packages, pkgpath)
 	assert.Contains(t, pg.Packages, "github.com/korfuri/goref/testprograms/vendored/vendor/github.com/korfuri/somedep")