From 7c26f69ec59e8fd1f23928904f4d9b897ba65c17 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 21 Jun 2019 13:59:07 +0200 Subject: [PATCH] gitbase: use only one cache for all reposiories Also fix a bug getting the cache size from cli. Signed-off-by: Javi Fontan --- cmd/gitbase/command/server.go | 6 ++-- cmd/gitbase/command/server_test.go | 49 ++++++++++++++++++++++++++ common_test.go | 6 ++-- database_test.go | 4 +-- integration_test.go | 6 ++-- internal/function/commit_stats_test.go | 2 +- internal/function/uast_test.go | 7 +--- internal/rule/squashjoins_test.go | 8 ++--- repositories_test.go | 2 +- repository_pool.go | 6 ++-- repository_pool_test.go | 8 ++--- 11 files changed, 74 insertions(+), 30 deletions(-) diff --git a/cmd/gitbase/command/server.go b/cmd/gitbase/command/server.go index 1d7ea546a..8b4165e13 100644 --- a/cmd/gitbase/command/server.go +++ b/cmd/gitbase/command/server.go @@ -220,10 +220,10 @@ func (c *Server) buildDatabase() error { ) } - c.rootLibrary = libraries.New(libraries.Options{}) - c.pool = gitbase.NewRepositoryPool(c.CacheSize*cache.MiByte, c.rootLibrary) + c.sharedCache = cache.NewObjectLRU(c.CacheSize * cache.MiByte) - c.sharedCache = cache.NewObjectLRU(512 * cache.MiByte) + c.rootLibrary = libraries.New(libraries.Options{}) + c.pool = gitbase.NewRepositoryPool(c.sharedCache, c.rootLibrary) if err := c.addDirectories(); err != nil { return err diff --git a/cmd/gitbase/command/server_test.go b/cmd/gitbase/command/server_test.go index 5adf3a375..8eb038521 100644 --- a/cmd/gitbase/command/server_test.go +++ b/cmd/gitbase/command/server_test.go @@ -3,7 +3,11 @@ package command import ( "testing" + "io/ioutil" + "os" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing" ) func TestDirectories(t *testing.T) { @@ -147,3 +151,48 @@ func TestDirectories(t *testing.T) { }) } } + +func TestCache(t *testing.T) { + require := require.New(t) + + tmpDir, err := ioutil.TempDir("", "gitbase") + require.NoError(err) + func() { + require.NoError(os.RemoveAll(tmpDir)) + }() + + server := &Server{ + CacheSize: 512, + Format: "siva", + Bucket: 0, + LogLevel: "debug", + Directories: []string{"../../../_testdata"}, + IndexDir: tmpDir, + } + + err = server.buildDatabase() + require.NoError(err) + + cache := server.sharedCache + pool := server.pool + hash := plumbing.NewHash("dbfab055c70379219cbcf422f05316fdf4e1aed3") + + _, ok := cache.Get(hash) + require.False(ok) + + repo, err := pool.GetRepo("015da2f4-6d89-7ec8-5ac9-a38329ea875b") + require.NoError(err) + + _, ok = repo.Cache().Get(hash) + require.False(ok) + require.Equal(cache, repo.Cache()) + + _, err = repo.CommitObject(hash) + require.NoError(err) + + _, ok = cache.Get(hash) + require.True(ok) + + _, ok = repo.Cache().Get(hash) + require.True(ok) +} diff --git a/common_test.go b/common_test.go index e2ba9d737..e2b13a887 100644 --- a/common_test.go +++ b/common_test.go @@ -49,7 +49,7 @@ func buildSession( lib, err := newMultiLibrary() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) for _, fixture := range repos { path := fixture.Worktree().Root() ok, err := IsGitRepo(path) @@ -173,7 +173,7 @@ func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedLibrary require.NoError(err) closedLib := &closedLibrary{multiLibrary: lib} - pool := NewRepositoryPool(cache.DefaultMaxSize, closedLib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), closedLib) cwd, err := os.Getwd() require.NoError(err) @@ -298,7 +298,7 @@ func newMultiPool() (*multiLibrary, *RepositoryPool, error) { return nil, nil, err } - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) return lib, pool, err } diff --git a/database_test.go b/database_test.go index 3a7f4111d..61681a3f1 100644 --- a/database_test.go +++ b/database_test.go @@ -15,7 +15,7 @@ func TestDatabase_Tables(t *testing.T) { require := require.New(t) lib := libraries.New(libraries.Options{}) - db := getDB(t, testDBName, NewRepositoryPool(0, lib)) + db := getDB(t, testDBName, NewRepositoryPool(nil, lib)) tables := db.Tables() var tableNames []string @@ -46,7 +46,7 @@ func TestDatabase_Name(t *testing.T) { require := require.New(t) lib := libraries.New(libraries.Options{}) - db := getDB(t, testDBName, NewRepositoryPool(0, lib)) + db := getDB(t, testDBName, NewRepositoryPool(nil, lib)) require.Equal(testDBName, db.Name()) } diff --git a/integration_test.go b/integration_test.go index 16f6671c8..65355dd4d 100644 --- a/integration_test.go +++ b/integration_test.go @@ -51,7 +51,7 @@ func TestIntegration(t *testing.T) { require.NoError(t, err) lib.AddLocation(loc) - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) engine := newBaseEngine(pool) testCases := []struct { @@ -639,7 +639,7 @@ func TestMissingHeadRefs(t *testing.T) { }) require.NoError(err) - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) engine := newBaseEngine(pool) session := gitbase.NewSession(pool) @@ -1013,7 +1013,7 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) { require.NoError(t, err) lib.AddLocation(loc) - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) cleanup := func() { require.NoError(t, fixtures.Clean()) diff --git a/internal/function/commit_stats_test.go b/internal/function/commit_stats_test.go index 63e9f269c..e7cbdb17c 100644 --- a/internal/function/commit_stats_test.go +++ b/internal/function/commit_stats_test.go @@ -108,7 +108,7 @@ func setupPool(t *testing.T) (*gitbase.RepositoryPool, func()) { require.NoError(t, err) lib.AddLocation(loc) - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) return pool, cleanup } diff --git a/internal/function/uast_test.go b/internal/function/uast_test.go index 77c68690c..d3fc6b3b6 100644 --- a/internal/function/uast_test.go +++ b/internal/function/uast_test.go @@ -423,11 +423,6 @@ func bblfshFixtures( func setup(t *testing.T) (*sql.Context, func()) { t.Helper() - // pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) - // for _, f := range fixtures.ByTag("worktree") { - // pool.AddGit(f.Worktree().Root()) - // } - // create library directory and move repo inside path := fixtures.ByTag("worktree").One().Worktree().Root() @@ -445,7 +440,7 @@ func setup(t *testing.T) (*sql.Context, func()) { require.NoError(t, err) lib.AddLocation(loc) - pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib) session := gitbase.NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) diff --git a/internal/rule/squashjoins_test.go b/internal/rule/squashjoins_test.go index c5f768ab8..7856b3ef0 100644 --- a/internal/rule/squashjoins_test.go +++ b/internal/rule/squashjoins_test.go @@ -22,7 +22,7 @@ func TestAnalyzeSquashJoinsExchange(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase( - gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)), + gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)), ) a := analyzer.NewBuilder(catalog). WithParallelism(2). @@ -61,7 +61,7 @@ func TestAnalyzeSquashNaturalJoins(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase( - gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)), + gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)), ) a := analyzer.NewBuilder(catalog). WithParallelism(2). @@ -2289,7 +2289,7 @@ func TestIsJoinLeafSquashable(t *testing.T) { func TestOrderedTableNames(t *testing.T) { lib := libraries.New(libraries.Options{}) - tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)).Tables() + tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)).Tables() input := []sql.Table{ tables[gitbase.BlobsTableName], @@ -2831,5 +2831,5 @@ func (l dummyLookup) Indexes() []string { func gitbaseTables() map[string]sql.Table { lib := libraries.New(libraries.Options{}) - return gitbase.NewDatabase("", gitbase.NewRepositoryPool(0, lib)).Tables() + return gitbase.NewDatabase("", gitbase.NewRepositoryPool(nil, lib)).Tables() } diff --git a/repositories_test.go b/repositories_test.go index e8854f92e..230dcf2ff 100644 --- a/repositories_test.go +++ b/repositories_test.go @@ -22,7 +22,7 @@ func TestRepositoriesTable(t *testing.T) { lib, err := newMultiLibrary() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() for _, id := range repoIDs { diff --git a/repository_pool.go b/repository_pool.go index 1e495ac9a..22294fe79 100644 --- a/repository_pool.go +++ b/repository_pool.go @@ -74,13 +74,13 @@ type RepositoryPool struct { library borges.Library } -// NewRepositoryPool initializes a new RepositoryPool with LRU cache. +// NewRepositoryPool holds a repository library and a shared object cache. func NewRepositoryPool( - maxCacheSize cache.FileSize, + c cache.Object, lib borges.Library, ) *RepositoryPool { return &RepositoryPool{ - cache: cache.NewObjectLRU(maxCacheSize), + cache: c, library: lib, } } diff --git a/repository_pool_test.go b/repository_pool_test.go index 5d51e4b62..8ad30467a 100644 --- a/repository_pool_test.go +++ b/repository_pool_test.go @@ -20,7 +20,7 @@ func TestRepositoryPoolBasic(t *testing.T) { lib, err := newMultiLibrary() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) iter, err := pool.RepoIter() require.NoError(err) @@ -66,7 +66,7 @@ func TestRepositoryPoolGit(t *testing.T) { lib, err := newMultiLibrary() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) require.NoError(lib.AddPlain(path, path, nil)) @@ -107,7 +107,7 @@ func TestRepositoryPoolIterator(t *testing.T) { lib, err := newMultiLibrary() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) err = lib.AddPlain("0", path, nil) require.NoError(err) err = lib.AddPlain("1", path, nil) @@ -145,7 +145,7 @@ func TestRepositoryPoolSiva(t *testing.T) { cwd, err := os.Getwd() require.NoError(err) - pool := NewRepositoryPool(cache.DefaultMaxSize, lib) + pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib) path := filepath.Join(cwd, "_testdata") require.NoError(