Factor out store and zipcache from searcher to store pkg

cmd/searcher/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/pkg/env"
 	"github.com/sourcegraph/sourcegraph/pkg/gitserver"
 	"github.com/sourcegraph/sourcegraph/pkg/search/rpc"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 	"github.com/sourcegraph/sourcegraph/pkg/tracer"
 	"github.com/sourcegraph/sourcegraph/pkg/vcs/git"
 )
@@ -49,7 +50,7 @@ func main() {
 	}
 
 	service := &search.Service{
-		Store: &search.Store{
+		Store: &store.Store{
 			FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) {
 				return git.Archive(ctx, repo, git.ArchiveOptions{Treeish: string(commit), Format: "tar"})
 			},

cmd/searcher/search/eval.go

@@ -31,6 +31,7 @@ import (
 	api "github.com/sourcegraph/sourcegraph/pkg/search"
 	"github.com/sourcegraph/sourcegraph/pkg/search/matchtree"
 	"github.com/sourcegraph/sourcegraph/pkg/search/query"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 	"golang.org/x/net/trace"
 )
 
@@ -54,7 +55,7 @@ const source = api.Source("textjit")
 // StoreSearcher provides a pkg/search.Searcher which searches over a search
 // store.
 type StoreSearcher struct {
-	Store *Store
+	Store *store.Store
 }
 
 // Search implements pkg/search.Search
@@ -125,7 +126,7 @@ func (s *StoreSearcher) Search(ctx context.Context, q query.Q, opts *api.Options
 		prepareCtx, cancel = context.WithTimeout(ctx, opts.FetchTimeout)
 		defer cancel()
 	}
-	path, err := s.Store.prepareZip(prepareCtx, gitserver.Repo{Name: repo.Name}, repo.Commit)
+	path, err := s.Store.PrepareZip(prepareCtx, gitserver.Repo{Name: repo.Name}, repo.Commit)
 	if err != nil {
 		if errcode.IsTimeout(err) {
 			return emptyResultWithStatus(api.RepositoryStatusTimedOut), nil
@@ -134,7 +135,7 @@ func (s *StoreSearcher) Search(ctx context.Context, q query.Q, opts *api.Options
 		}
 		return nil, err
 	}
-	zf, err := s.Store.zipCache.get(path)
+	zf, err := s.Store.ZipCache.Get(path)
 	if err != nil {
 		return nil, err
 	}
@@ -406,8 +407,8 @@ type candidateMatch struct {
 }
 
 type contentProvider struct {
-	zf   *zipFile
-	file *srcFile
+	zf   *store.ZipFile
+	file *store.SrcFile
 
 	// Cache
 	fileName []byte

cmd/searcher/search/matcher.go

@@ -17,17 +17,14 @@ import (
 
 	"github.com/sourcegraph/sourcegraph/cmd/searcher/protocol"
 	"github.com/sourcegraph/sourcegraph/pkg/pathmatch"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	otlog "github.com/opentracing/opentracing-go/log"
 )
 
 const (
-	// maxFileSize is the limit on file size in bytes. Only files smaller
-	// than this are searched.
-	maxFileSize = 1 << 20 // 1MB; match https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/sourcegraph/zoekt%24+%22-file_limit%22
-
 	// maxLineSize is the maximum length of a line in bytes.
 	// Lines larger than this are not scanned for results.
 	// (e.g. minified javascript files that are all on one line).
@@ -184,7 +181,7 @@ func (rg *readerGrep) matchString(s string) bool {
 // Find returns a LineMatch for each line that matches rg in reader.
 // LimitHit is true if some matches may not have been included in the result.
 // NOTE: This is not safe to use concurrently.
-func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatch, limitHit bool, err error) {
+func (rg *readerGrep) Find(zf *store.ZipFile, f *store.SrcFile) (matches []protocol.LineMatch, limitHit bool, err error) {
 	if rg.ignoreCase && rg.transformBuf == nil {
 		rg.transformBuf = make([]byte, zf.MaxLen)
 	}
@@ -262,7 +259,7 @@ func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatc
 			}
 			matches = append(matches, protocol.LineMatch{
 				// making a copy of lineBuf is intentional.
-				// we are not allowed to use the fileBuf data after the zipFile has been Closed,
+				// we are not allowed to use the fileBuf data after the ZipFile has been Closed,
 				// which currently occurs before Preview has been serialized.
 				// TODO: consider moving the call to Close until after we are
 				// done with Preview, and stop making a copy here.
@@ -279,7 +276,7 @@ func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatc
 }
 
 // FindZip is a convenience function to run Find on f.
-func (rg *readerGrep) FindZip(zf *zipFile, f *srcFile) (protocol.FileMatch, error) {
+func (rg *readerGrep) FindZip(zf *store.ZipFile, f *store.SrcFile) (protocol.FileMatch, error) {
 	lm, limitHit, err := rg.Find(zf, f)
 	return protocol.FileMatch{
 		Path:        f.Name,
@@ -289,7 +286,7 @@ func (rg *readerGrep) FindZip(zf *zipFile, f *srcFile) (protocol.FileMatch, erro
 }
 
 // concurrentFind searches files in zr looking for matches using rg.
-func concurrentFind(ctx context.Context, rg *readerGrep, zf *zipFile, fileMatchLimit int, patternMatchesContent, patternMatchesPaths bool) (fm []protocol.FileMatch, limitHit bool, err error) {
+func concurrentFind(ctx context.Context, rg *readerGrep, zf *store.ZipFile, fileMatchLimit int, patternMatchesContent, patternMatchesPaths bool) (fm []protocol.FileMatch, limitHit bool, err error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "ConcurrentFind")
 	ext.Component.Set(span, "matcher")
 	if rg.re != nil {

cmd/searcher/search/matcher_test.go

@@ -15,6 +15,7 @@ import (
 	"testing/quick"
 
 	"github.com/sourcegraph/sourcegraph/cmd/searcher/protocol"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 	"github.com/sourcegraph/sourcegraph/pkg/testutil"
 )
 
@@ -248,13 +249,13 @@ func benchConcurrentFind(b *testing.B, p *protocol.Request) {
 	}
 
 	ctx := context.Background()
-	path, err := githubStore.prepareZip(ctx, p.GitserverRepo(), p.Commit)
+	path, err := githubStore.PrepareZip(ctx, p.GitserverRepo(), p.Commit)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	var zc zipCache
-	zf, err := zc.get(path)
+	var zc store.ZipCache
+	zf, err := zc.Get(path)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -445,11 +446,11 @@ func TestLineLimit(t *testing.T) {
 
 	for i, test := range tests {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			fakeZipFile := zipFile{
+			fakeZipFile := store.ZipFile{
 				MaxLen: maxBuf,
 				Data:   bytes.Repeat([]byte("A"), test.size),
 			}
-			fakeSrcFile := srcFile{Len: int32(test.size)}
+			fakeSrcFile := store.SrcFile{Len: int32(test.size)}
 			matches, limitHit, err := rg.Find(&fakeZipFile, &fakeSrcFile)
 			if err != nil {
 				t.Fatal(err)
@@ -491,7 +492,7 @@ func TestMaxMatches(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	zf, err := mockZipFile(buf.Bytes())
+	zf, err := store.MockZipFile(buf.Bytes())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -547,7 +548,7 @@ func TestPathMatches(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	zf, err := mockZipFile(zipData)
+	zf, err := store.MockZipFile(zipData)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -599,7 +600,7 @@ func createZip(files map[string]string) ([]byte, error) {
 }
 
 // githubStore fetches from github and caches across test runs.
-var githubStore = &Store{
+var githubStore = &store.Store{
 	FetchTar: testutil.FetchTarFromGithub,
 	Path:     "/tmp/search_test/store",
 }

cmd/searcher/search/retry.go

@@ -3,11 +3,13 @@ package search
 import (
 	"os"
 	"strings"
+
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 )
 
 // getZipFileWithRetry retries getting a zip file if the zip is for some reason
 // invalid.
-func getZipFileWithRetry(get func() (string, *zipFile, error)) (zf *zipFile, err error) {
+func getZipFileWithRetry(get func() (string, *store.ZipFile, error)) (zf *store.ZipFile, err error) {
 	var path string
 	tries := 0
 	for zf == nil {

cmd/searcher/search/retry_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/pkg/errors"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 )
 
 func TestGetZipFileWithRetry(t *testing.T) {
@@ -48,17 +49,17 @@ func TestGetZipFileWithRetry(t *testing.T) {
 			}()
 
 			tries := 0
-			get := func() (string, *zipFile, error) {
+			get := func() (string, *store.ZipFile, error) {
 				var err error
 				tmp, err = ioutil.TempFile("", "")
 				if err != nil {
 					t.Fatalf("TempFile(%v)", err)
 				}
 
 				err = test.errs[tries]
-				var zf *zipFile
+				var zf *store.ZipFile
 				if err == nil {
-					zf = &zipFile{}
+					zf = &store.ZipFile{}
 				}
 				tries++
 				return tmp.Name(), zf, err

cmd/searcher/search/search.go

@@ -24,6 +24,7 @@ import (
 	log15 "gopkg.in/inconshreveable/log15.v2"
 
 	"github.com/sourcegraph/sourcegraph/cmd/searcher/protocol"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 
 	"github.com/pkg/errors"
 
@@ -36,7 +37,7 @@ import (
 
 // Service is the search service. It is an http.Handler.
 type Service struct {
-	Store *Store
+	Store *store.Store
 	Log   log15.Logger
 }
 
@@ -186,12 +187,12 @@ func (s *Service) search(ctx context.Context, p *protocol.Request) (matches []pr
 	prepareCtx, cancel := context.WithTimeout(ctx, fetchTimeout)
 	defer cancel()
 
-	getZf := func() (string, *zipFile, error) {
-		path, err := s.Store.prepareZip(prepareCtx, p.GitserverRepo(), p.Commit)
+	getZf := func() (string, *store.ZipFile, error) {
+		path, err := s.Store.PrepareZip(prepareCtx, p.GitserverRepo(), p.Commit)
 		if err != nil {
 			return "", nil, err
 		}
-		zf, err := s.Store.zipCache.get(path)
+		zf, err := s.Store.ZipCache.Get(path)
 		return path, zf, err
 	}
 

cmd/searcher/search/search_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/pkg/gitserver"
 	searchapi "github.com/sourcegraph/sourcegraph/pkg/search"
 	"github.com/sourcegraph/sourcegraph/pkg/search/query"
+	"github.com/sourcegraph/sourcegraph/pkg/store"
 )
 
 func TestSearch(t *testing.T) {
@@ -384,7 +385,7 @@ func doSearch(u string, p *protocol.Request) ([]protocol.FileMatch, error) {
 	return r.Matches, err
 }
 
-func newStore(files map[string]string) (*search.Store, func(), error) {
+func newStore(files map[string]string) (*store.Store, func(), error) {
 	buf := new(bytes.Buffer)
 	w := tar.NewWriter(buf)
 	for name, body := range files {
@@ -415,7 +416,7 @@ func newStore(files map[string]string) (*search.Store, func(), error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	return &search.Store{
+	return &store.Store{
 		FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) {
 			return ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
 		},

cmd/searcher/search/store.go → pkg/store/store.go

@@ -1,4 +1,4 @@
-package search
+package store
 
 import (
 	"archive/tar"
@@ -27,6 +27,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const (
+	// maxFileSize is the limit on file size in bytes. Only files smaller
+	// than this are searched.
+	maxFileSize = 1 << 20 // 1MB; match https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/sourcegraph/zoekt%24+%22-file_limit%22
+)
+
 // Store manages the fetching and storing of git archives. Its main purpose is
 // keeping a local disk cache of the fetched archives to help speed up future
 // requests for the same archive. As a performance optimization, it is also
@@ -67,8 +73,8 @@ type Store struct {
 	// fetchLimiter limits concurrent calls to FetchTar.
 	fetchLimiter *mutablelimiter.Limiter
 
-	// zipCache provides efficient access to repo zip files.
-	zipCache zipCache
+	// ZipCache provides efficient access to repo zip files.
+	ZipCache ZipCache
 }
 
 // SetMaxConcurrentFetchTar sets the maximum number of concurrent calls allowed
@@ -96,15 +102,15 @@ func (s *Store) Start() {
 			Dir:               s.Path,
 			Component:         "store",
 			BackgroundTimeout: 2 * time.Minute,
-			BeforeEvict:       s.zipCache.delete,
+			BeforeEvict:       s.ZipCache.delete,
 		}
 		go s.watchAndEvict()
 	})
 }
 
-// prepareZip returns the path to a local zip archive of repo at commit.
+// PrepareZip returns the path to a local zip archive of repo at commit.
 // It will first consult the local cache, otherwise will fetch from the network.
-func (s *Store) prepareZip(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (path string, err error) {
+func (s *Store) PrepareZip(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (path string, err error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "Store.prepareZip")
 	ext.Component.Set(span, "store")
 	defer func() {

cmd/searcher/search/store_test.go → pkg/store/store_test.go

@@ -1,4 +1,4 @@
-package search
+package store
 
 import (
 	"archive/tar"
@@ -41,7 +41,7 @@ func TestPrepareZip(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		go func() {
 			<-startPrepareZip
-			_, err := s.prepareZip(context.Background(), wantRepo, wantCommit)
+			_, err := s.PrepareZip(context.Background(), wantRepo, wantCommit)
 			prepareZipErr <- err
 		}()
 	}
@@ -50,7 +50,7 @@ func TestPrepareZip(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		err := <-prepareZipErr
 		if err != nil {
-			t.Fatal("expected prepareZip to succeed:", err)
+			t.Fatal("expected PrepareZip to succeed:", err)
 		}
 	}
 
@@ -75,9 +75,9 @@ func TestPrepareZip(t *testing.T) {
 	if !onDisk {
 		t.Fatal("timed out waiting for items to appear in cache at", s.Path)
 	}
-	_, err := s.prepareZip(context.Background(), wantRepo, wantCommit)
+	_, err := s.PrepareZip(context.Background(), wantRepo, wantCommit)
 	if err != nil {
-		t.Fatal("expected prepareZip to succeed:", err)
+		t.Fatal("expected PrepareZip to succeed:", err)
 	}
 }
 
@@ -88,9 +88,9 @@ func TestPrepareZip_fetchTarFail(t *testing.T) {
 	s.FetchTar = func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) {
 		return nil, fetchErr
 	}
-	_, err := s.prepareZip(context.Background(), gitserver.Repo{Name: "foo"}, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	_, err := s.PrepareZip(context.Background(), gitserver.Repo{Name: "foo"}, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
 	if errors.Cause(err) != fetchErr {
-		t.Fatalf("expected prepareZip to fail with %v, failed with %v", fetchErr, err)
+		t.Fatalf("expected PrepareZip to fail with %v, failed with %v", fetchErr, err)
 	}
 }
 
@@ -126,7 +126,7 @@ func TestIngoreSizeMax(t *testing.T) {
 }
 
 func tmpStore(t *testing.T) (*Store, func()) {
-	d, err := ioutil.TempDir("", "search_test")
+	d, err := ioutil.TempDir("", "store_test")
 	if err != nil {
 		t.Fatal(err)
 	}