From e0e7a5a00f8ed8d7710b934c56fe6704e952ceb4 Mon Sep 17 00:00:00 2001 From: Rijnard van Tonder Date: Mon, 22 Apr 2019 21:31:16 -0400 Subject: [PATCH 1/2] Factors out store and zipcache from searcher to store pkg. --- cmd/searcher/main.go | 3 +- cmd/searcher/search/eval.go | 11 ++-- cmd/searcher/search/matcher.go | 13 ++-- cmd/searcher/search/matcher_test.go | 17 +++--- cmd/searcher/search/search.go | 7 ++- cmd/searcher/search/search_test.go | 5 +- {cmd/searcher/search => pkg/store}/store.go | 18 ++++-- .../search => pkg/store}/store_test.go | 16 ++--- .../searcher/search => pkg/store}/zipcache.go | 60 +++++++++---------- .../search => pkg/store}/zipcache_test.go | 10 ++-- 10 files changed, 84 insertions(+), 76 deletions(-) rename {cmd/searcher/search => pkg/store}/store.go (95%) rename {cmd/searcher/search => pkg/store}/store_test.go (87%) rename {cmd/searcher/search => pkg/store}/zipcache.go (79%) rename {cmd/searcher/search => pkg/store}/zipcache_test.go (87%) diff --git a/cmd/searcher/main.go b/cmd/searcher/main.go index 7a66cd4cfbb9..b14e7ad34a21 100644 --- a/cmd/searcher/main.go +++ b/cmd/searcher/main.go @@ -24,6 +24,7 @@ import ( "github.com/sourcegraph/sourcegraph/pkg/env" "github.com/sourcegraph/sourcegraph/pkg/gitserver" "github.com/sourcegraph/sourcegraph/pkg/search/rpc" + "github.com/sourcegraph/sourcegraph/pkg/store" "github.com/sourcegraph/sourcegraph/pkg/tracer" "github.com/sourcegraph/sourcegraph/pkg/vcs/git" ) @@ -49,7 +50,7 @@ func main() { } service := &search.Service{ - Store: &search.Store{ + Store: &store.Store{ FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) { return git.Archive(ctx, repo, git.ArchiveOptions{Treeish: string(commit), Format: "tar"}) }, diff --git a/cmd/searcher/search/eval.go b/cmd/searcher/search/eval.go index b71c340b2b8c..0d314b6348f4 100644 --- a/cmd/searcher/search/eval.go +++ b/cmd/searcher/search/eval.go @@ -31,6 +31,7 @@ import ( api "github.com/sourcegraph/sourcegraph/pkg/search" "github.com/sourcegraph/sourcegraph/pkg/search/matchtree" "github.com/sourcegraph/sourcegraph/pkg/search/query" + "github.com/sourcegraph/sourcegraph/pkg/store" "golang.org/x/net/trace" ) @@ -54,7 +55,7 @@ const source = api.Source("textjit") // StoreSearcher provides a pkg/search.Searcher which searches over a search // store. type StoreSearcher struct { - Store *Store + Store *store.Store } // Search implements pkg/search.Search @@ -125,7 +126,7 @@ func (s *StoreSearcher) Search(ctx context.Context, q query.Q, opts *api.Options prepareCtx, cancel = context.WithTimeout(ctx, opts.FetchTimeout) defer cancel() } - path, err := s.Store.prepareZip(prepareCtx, gitserver.Repo{Name: repo.Name}, repo.Commit) + path, err := s.Store.PrepareZip(prepareCtx, gitserver.Repo{Name: repo.Name}, repo.Commit) if err != nil { if errcode.IsTimeout(err) { return emptyResultWithStatus(api.RepositoryStatusTimedOut), nil @@ -134,7 +135,7 @@ func (s *StoreSearcher) Search(ctx context.Context, q query.Q, opts *api.Options } return nil, err } - zf, err := s.Store.zipCache.get(path) + zf, err := s.Store.ZipCache.Get(path) if err != nil { return nil, err } @@ -406,8 +407,8 @@ type candidateMatch struct { } type contentProvider struct { - zf *zipFile - file *srcFile + zf *store.ZipFile + file *store.SrcFile // Cache fileName []byte diff --git a/cmd/searcher/search/matcher.go b/cmd/searcher/search/matcher.go index d997102a9918..78b30ac6e291 100644 --- a/cmd/searcher/search/matcher.go +++ b/cmd/searcher/search/matcher.go @@ -17,6 +17,7 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/pkg/pathmatch" + "github.com/sourcegraph/sourcegraph/pkg/store" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -24,10 +25,6 @@ import ( ) const ( - // maxFileSize is the limit on file size in bytes. Only files smaller - // than this are searched. - maxFileSize = 1 << 20 // 1MB; match https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/sourcegraph/zoekt%24+%22-file_limit%22 - // maxLineSize is the maximum length of a line in bytes. // Lines larger than this are not scanned for results. // (e.g. minified javascript files that are all on one line). @@ -184,7 +181,7 @@ func (rg *readerGrep) matchString(s string) bool { // Find returns a LineMatch for each line that matches rg in reader. // LimitHit is true if some matches may not have been included in the result. // NOTE: This is not safe to use concurrently. -func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatch, limitHit bool, err error) { +func (rg *readerGrep) Find(zf *store.ZipFile, f *store.SrcFile) (matches []protocol.LineMatch, limitHit bool, err error) { if rg.ignoreCase && rg.transformBuf == nil { rg.transformBuf = make([]byte, zf.MaxLen) } @@ -262,7 +259,7 @@ func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatc } matches = append(matches, protocol.LineMatch{ // making a copy of lineBuf is intentional. - // we are not allowed to use the fileBuf data after the zipFile has been Closed, + // we are not allowed to use the fileBuf data after the ZipFile has been Closed, // which currently occurs before Preview has been serialized. // TODO: consider moving the call to Close until after we are // done with Preview, and stop making a copy here. @@ -279,7 +276,7 @@ func (rg *readerGrep) Find(zf *zipFile, f *srcFile) (matches []protocol.LineMatc } // FindZip is a convenience function to run Find on f. -func (rg *readerGrep) FindZip(zf *zipFile, f *srcFile) (protocol.FileMatch, error) { +func (rg *readerGrep) FindZip(zf *store.ZipFile, f *store.SrcFile) (protocol.FileMatch, error) { lm, limitHit, err := rg.Find(zf, f) return protocol.FileMatch{ Path: f.Name, @@ -289,7 +286,7 @@ func (rg *readerGrep) FindZip(zf *zipFile, f *srcFile) (protocol.FileMatch, erro } // concurrentFind searches files in zr looking for matches using rg. -func concurrentFind(ctx context.Context, rg *readerGrep, zf *zipFile, fileMatchLimit int, patternMatchesContent, patternMatchesPaths bool) (fm []protocol.FileMatch, limitHit bool, err error) { +func concurrentFind(ctx context.Context, rg *readerGrep, zf *store.ZipFile, fileMatchLimit int, patternMatchesContent, patternMatchesPaths bool) (fm []protocol.FileMatch, limitHit bool, err error) { span, ctx := opentracing.StartSpanFromContext(ctx, "ConcurrentFind") ext.Component.Set(span, "matcher") if rg.re != nil { diff --git a/cmd/searcher/search/matcher_test.go b/cmd/searcher/search/matcher_test.go index 39b9ba6432fb..800b53256170 100644 --- a/cmd/searcher/search/matcher_test.go +++ b/cmd/searcher/search/matcher_test.go @@ -15,6 +15,7 @@ import ( "testing/quick" "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" + "github.com/sourcegraph/sourcegraph/pkg/store" "github.com/sourcegraph/sourcegraph/pkg/testutil" ) @@ -248,13 +249,13 @@ func benchConcurrentFind(b *testing.B, p *protocol.Request) { } ctx := context.Background() - path, err := githubStore.prepareZip(ctx, p.GitserverRepo(), p.Commit) + path, err := githubStore.PrepareZip(ctx, p.GitserverRepo(), p.Commit) if err != nil { b.Fatal(err) } - var zc zipCache - zf, err := zc.get(path) + var zc store.ZipCache + zf, err := zc.Get(path) if err != nil { b.Fatal(err) } @@ -445,11 +446,11 @@ func TestLineLimit(t *testing.T) { for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - fakeZipFile := zipFile{ + fakeZipFile := store.ZipFile{ MaxLen: maxBuf, Data: bytes.Repeat([]byte("A"), test.size), } - fakeSrcFile := srcFile{Len: int32(test.size)} + fakeSrcFile := store.SrcFile{Len: int32(test.size)} matches, limitHit, err := rg.Find(&fakeZipFile, &fakeSrcFile) if err != nil { t.Fatal(err) @@ -491,7 +492,7 @@ func TestMaxMatches(t *testing.T) { if err != nil { t.Fatal(err) } - zf, err := mockZipFile(buf.Bytes()) + zf, err := store.MockZipFile(buf.Bytes()) if err != nil { t.Fatal(err) } @@ -547,7 +548,7 @@ func TestPathMatches(t *testing.T) { if err != nil { t.Fatal(err) } - zf, err := mockZipFile(zipData) + zf, err := store.MockZipFile(zipData) if err != nil { t.Fatal(err) } @@ -599,7 +600,7 @@ func createZip(files map[string]string) ([]byte, error) { } // githubStore fetches from github and caches across test runs. -var githubStore = &Store{ +var githubStore = &store.Store{ FetchTar: testutil.FetchTarFromGithub, Path: "/tmp/search_test/store", } diff --git a/cmd/searcher/search/search.go b/cmd/searcher/search/search.go index 677a461c5073..6f3c9be804b2 100644 --- a/cmd/searcher/search/search.go +++ b/cmd/searcher/search/search.go @@ -24,6 +24,7 @@ import ( log15 "gopkg.in/inconshreveable/log15.v2" "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" + "github.com/sourcegraph/sourcegraph/pkg/store" "github.com/pkg/errors" @@ -36,7 +37,7 @@ import ( // Service is the search service. It is an http.Handler. type Service struct { - Store *Store + Store *store.Store Log log15.Logger } @@ -185,11 +186,11 @@ func (s *Service) search(ctx context.Context, p *protocol.Request) (matches []pr } prepareCtx, cancel := context.WithTimeout(ctx, fetchTimeout) defer cancel() - path, err := s.Store.prepareZip(prepareCtx, p.GitserverRepo(), p.Commit) + path, err := s.Store.PrepareZip(prepareCtx, p.GitserverRepo(), p.Commit) if err != nil { return nil, false, false, err } - zf, err := s.Store.zipCache.get(path) + zf, err := s.Store.ZipCache.Get(path) if err != nil { return nil, false, false, err } diff --git a/cmd/searcher/search/search_test.go b/cmd/searcher/search/search_test.go index 03eecc39f415..a7b907ad6518 100644 --- a/cmd/searcher/search/search_test.go +++ b/cmd/searcher/search/search_test.go @@ -26,6 +26,7 @@ import ( "github.com/sourcegraph/sourcegraph/pkg/gitserver" searchapi "github.com/sourcegraph/sourcegraph/pkg/search" "github.com/sourcegraph/sourcegraph/pkg/search/query" + "github.com/sourcegraph/sourcegraph/pkg/store" ) func TestSearch(t *testing.T) { @@ -384,7 +385,7 @@ func doSearch(u string, p *protocol.Request) ([]protocol.FileMatch, error) { return r.Matches, err } -func newStore(files map[string]string) (*search.Store, func(), error) { +func newStore(files map[string]string) (*store.Store, func(), error) { buf := new(bytes.Buffer) w := tar.NewWriter(buf) for name, body := range files { @@ -415,7 +416,7 @@ func newStore(files map[string]string) (*search.Store, func(), error) { if err != nil { return nil, nil, err } - return &search.Store{ + return &store.Store{ FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil }, diff --git a/cmd/searcher/search/store.go b/pkg/store/store.go similarity index 95% rename from cmd/searcher/search/store.go rename to pkg/store/store.go index 9cfb32a9a039..47f22a92bbdc 100644 --- a/cmd/searcher/search/store.go +++ b/pkg/store/store.go @@ -1,4 +1,4 @@ -package search +package store import ( "archive/tar" @@ -27,6 +27,12 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +const ( + // maxFileSize is the limit on file size in bytes. Only files smaller + // than this are searched. + maxFileSize = 1 << 20 // 1MB; match https://sourcegraph.com/search?q=repo:%5Egithub%5C.com/sourcegraph/zoekt%24+%22-file_limit%22 +) + // Store manages the fetching and storing of git archives. Its main purpose is // keeping a local disk cache of the fetched archives to help speed up future // requests for the same archive. As a performance optimization, it is also @@ -67,8 +73,8 @@ type Store struct { // fetchLimiter limits concurrent calls to FetchTar. fetchLimiter *mutablelimiter.Limiter - // zipCache provides efficient access to repo zip files. - zipCache zipCache + // ZipCache provides efficient access to repo zip files. + ZipCache ZipCache } // SetMaxConcurrentFetchTar sets the maximum number of concurrent calls allowed @@ -96,15 +102,15 @@ func (s *Store) Start() { Dir: s.Path, Component: "store", BackgroundTimeout: 2 * time.Minute, - BeforeEvict: s.zipCache.delete, + BeforeEvict: s.ZipCache.delete, } go s.watchAndEvict() }) } -// prepareZip returns the path to a local zip archive of repo at commit. +// PrepareZip returns the path to a local zip archive of repo at commit. // It will first consult the local cache, otherwise will fetch from the network. -func (s *Store) prepareZip(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (path string, err error) { +func (s *Store) PrepareZip(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (path string, err error) { span, ctx := opentracing.StartSpanFromContext(ctx, "Store.prepareZip") ext.Component.Set(span, "store") defer func() { diff --git a/cmd/searcher/search/store_test.go b/pkg/store/store_test.go similarity index 87% rename from cmd/searcher/search/store_test.go rename to pkg/store/store_test.go index 956979effe71..e4dbeebf2890 100644 --- a/cmd/searcher/search/store_test.go +++ b/pkg/store/store_test.go @@ -1,4 +1,4 @@ -package search +package store import ( "archive/tar" @@ -41,7 +41,7 @@ func TestPrepareZip(t *testing.T) { for i := 0; i < 10; i++ { go func() { <-startPrepareZip - _, err := s.prepareZip(context.Background(), wantRepo, wantCommit) + _, err := s.PrepareZip(context.Background(), wantRepo, wantCommit) prepareZipErr <- err }() } @@ -50,7 +50,7 @@ func TestPrepareZip(t *testing.T) { for i := 0; i < 10; i++ { err := <-prepareZipErr if err != nil { - t.Fatal("expected prepareZip to succeed:", err) + t.Fatal("expected PrepareZip to succeed:", err) } } @@ -75,9 +75,9 @@ func TestPrepareZip(t *testing.T) { if !onDisk { t.Fatal("timed out waiting for items to appear in cache at", s.Path) } - _, err := s.prepareZip(context.Background(), wantRepo, wantCommit) + _, err := s.PrepareZip(context.Background(), wantRepo, wantCommit) if err != nil { - t.Fatal("expected prepareZip to succeed:", err) + t.Fatal("expected PrepareZip to succeed:", err) } } @@ -88,9 +88,9 @@ func TestPrepareZip_fetchTarFail(t *testing.T) { s.FetchTar = func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) { return nil, fetchErr } - _, err := s.prepareZip(context.Background(), gitserver.Repo{Name: "foo"}, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + _, err := s.PrepareZip(context.Background(), gitserver.Repo{Name: "foo"}, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") if errors.Cause(err) != fetchErr { - t.Fatalf("expected prepareZip to fail with %v, failed with %v", fetchErr, err) + t.Fatalf("expected PrepareZip to fail with %v, failed with %v", fetchErr, err) } } @@ -126,7 +126,7 @@ func TestIngoreSizeMax(t *testing.T) { } func tmpStore(t *testing.T) (*Store, func()) { - d, err := ioutil.TempDir("", "search_test") + d, err := ioutil.TempDir("", "store_test") if err != nil { t.Fatal(err) } diff --git a/cmd/searcher/search/zipcache.go b/pkg/store/zipcache.go similarity index 79% rename from cmd/searcher/search/zipcache.go rename to pkg/store/zipcache.go index cae7929bef9c..355d6ae797c3 100644 --- a/cmd/searcher/search/zipcache.go +++ b/pkg/store/zipcache.go @@ -1,4 +1,4 @@ -package search +package store import ( "archive/zip" @@ -16,9 +16,9 @@ import ( "golang.org/x/sys/unix" ) -// A zipCache is a shared data structure that provides efficient access to a collection of zip files. +// A ZipCache is a shared data structure that provides efficient access to a collection of zip files. // The zero value is usable. -type zipCache struct { +type ZipCache struct { // Split the cache into many parts, to minimize lock contention. // This matters because, for simplicity, // we sometimes hold the lock for long-running operations, @@ -32,23 +32,23 @@ type zipCache struct { type zipCacheShard struct { mu sync.Mutex - m map[string]*zipFile // path -> zipFile + m map[string]*ZipFile // path -> zipFile } -func (c *zipCache) shardFor(path string) *zipCacheShard { +func (c *ZipCache) shardFor(path string) *zipCacheShard { h := fnv.New32() io.WriteString(h, path) return &c.shards[h.Sum32()%uint32(len(c.shards))] } -// get returns a zipFile for the file on disk at path. +// Get returns a zipFile for the file on disk at path. // The file MUST be Closed when it is no longer needed. -func (c *zipCache) get(path string) (*zipFile, error) { +func (c *ZipCache) Get(path string) (*ZipFile, error) { shard := c.shardFor(path) shard.mu.Lock() defer shard.mu.Unlock() if shard.m == nil { - shard.m = make(map[string]*zipFile) + shard.m = make(map[string]*ZipFile) } zf, ok := shard.m[path] if ok { @@ -67,7 +67,7 @@ func (c *zipCache) get(path string) (*zipFile, error) { return zf, nil } -func (c *zipCache) delete(path string) { +func (c *ZipCache) delete(path string) { shard := c.shardFor(path) shard.mu.Lock() defer shard.mu.Unlock() @@ -93,18 +93,18 @@ func (c *zipCache) delete(path string) { delete(shard.m, path) } -// zipFile provides efficient access to a single zip file. -type zipFile struct { +// ZipFile provides efficient access to a single zip file. +type ZipFile struct { // Take care with the size of this struct. // There are many zipFiles present during typical usage. - Files []srcFile + Files []SrcFile MaxLen int Data []byte f *os.File wg sync.WaitGroup // ensures underlying file is not munmap'd or closed while in use } -func readZipFile(path string) (*zipFile, error) { +func readZipFile(path string) (*ZipFile, error) { // Open zip file at path, prepare to read it. f, err := os.Open(path) if err != nil { @@ -119,8 +119,8 @@ func readZipFile(path string) (*zipFile, error) { return nil, err } - // Create at populate zipFile from contents. - zf := &zipFile{f: f} + // Create at populate ZipFile from contents. + zf := &ZipFile{f: f} if err := zf.populateFiles(r); err != nil { return nil, err } @@ -138,8 +138,8 @@ func readZipFile(path string) (*zipFile, error) { return zf, nil } -func (f *zipFile) populateFiles(r *zip.Reader) error { - f.Files = make([]srcFile, len(r.File)) +func (f *ZipFile) populateFiles(r *zip.Reader) error { + f.Files = make([]SrcFile, len(r.File)) for i, file := range r.File { if file.Method != zip.Store { return errors.Errorf("file %s stored with compression %v, want %v", file.Name, file.Method, zip.Store) @@ -152,7 +152,7 @@ func (f *zipFile) populateFiles(r *zip.Reader) error { if uint64(size) != file.UncompressedSize64 { return errors.Errorf("file %s has size > 2gb: %v", file.Name, size) } - f.Files[i] = srcFile{Name: file.Name, Off: off, Len: int32(size)} + f.Files[i] = SrcFile{Name: file.Name, Off: off, Len: int32(size)} if size > f.MaxLen { f.MaxLen = size } @@ -168,18 +168,18 @@ func (f *zipFile) populateFiles(r *zip.Reader) error { // Close allows resources associated with f to be released. // It MUST be called exactly once for every file retrieved using get. -// Contents from any srcFile from within f MUST NOT be used after +// Contents from any SrcFile from within f MUST NOT be used after // Close has been called. -func (f *zipFile) Close() { +func (f *ZipFile) Close() { f.wg.Done() } -func mockZipFile(data []byte) (*zipFile, error) { +func MockZipFile(data []byte) (*ZipFile, error) { r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) if err != nil { return nil, err } - zf := new(zipFile) + zf := new(ZipFile) if err := zf.populateFiles(r); err != nil { return nil, err } @@ -188,36 +188,36 @@ func mockZipFile(data []byte) (*zipFile, error) { zf.Data = make([]byte, len(data)) copy(zf.Data, data) // zf.f is intentionally left nil; - // this is an indicator that this is a mock zipFile. + // this is an indicator that this is a mock ZipFile. return zf, nil } -// A srcFile is a single file inside a zipFile. -type srcFile struct { +// A SrcFile is a single file inside a ZipFile. +type SrcFile struct { // Take care with the size of this struct. // There will be *lots* of these in memory. // This is why Len is a 32 bit int. - // (Note that this means that zipCache cannot + // (Note that this means that ZipCache cannot // handle files inside the zip archive bigger than 2gb.) Name string Off int64 Len int32 } -// Data returns the contents of s, which is a srcFile in f. +// Data returns the contents of s, which is a SrcFile in f. // The contents MUST NOT be modified. // It is not safe to use the contents after f has been Closed. -func (f *zipFile) DataFor(s *srcFile) []byte { +func (f *ZipFile) DataFor(s *SrcFile) []byte { return f.Data[s.Off : s.Off+int64(s.Len)] } -func (f *srcFile) String() string { +func (f *SrcFile) String() string { return fmt.Sprintf("<%s: %d+%d bytes>", f.Name, f.Off, f.Len) } // count returns the number of elements in c, assuming c is otherwise unused during the call to c. // It is intended only for testing. -func (c *zipCache) count() int { +func (c *ZipCache) count() int { var n int for i := range c.shards { shard := &c.shards[i] diff --git a/cmd/searcher/search/zipcache_test.go b/pkg/store/zipcache_test.go similarity index 87% rename from cmd/searcher/search/zipcache_test.go rename to pkg/store/zipcache_test.go index 4d53834702d4..a9f414347bd6 100644 --- a/cmd/searcher/search/zipcache_test.go +++ b/pkg/store/zipcache_test.go @@ -1,4 +1,4 @@ -package search +package store import ( "context" @@ -21,7 +21,7 @@ func TestZipCacheDelete(t *testing.T) { } // Grab a zip. - path, err := s.prepareZip(context.Background(), gitserver.Repo{Name: "somerepo"}, "0123456789012345678901234567890123456789") + path, err := s.PrepareZip(context.Background(), gitserver.Repo{Name: "somerepo"}, "0123456789012345678901234567890123456789") if err != nil { t.Fatal(err) } @@ -33,14 +33,14 @@ func TestZipCacheDelete(t *testing.T) { } // Load into zip cache. - zf, err := s.zipCache.get(path) + zf, err := s.ZipCache.Get(path) if err != nil { t.Fatal(err) } zf.Close() // don't block eviction of this zipFile // Make sure it's there. - if n := s.zipCache.count(); n != 1 { + if n := s.ZipCache.count(); n != 1 { t.Fatalf("expected 1 item in cache, got %d", n) } @@ -51,7 +51,7 @@ func TestZipCacheDelete(t *testing.T) { } // Make sure the zipFile is gone from the zip cache, too. - if n := s.zipCache.count(); n != 0 { + if n := s.ZipCache.count(); n != 0 { t.Fatalf("expected 0 items in cache, got %d", n) } From b9a46c7ba310c46910f47ef11c756dc8afed533c Mon Sep 17 00:00:00 2001 From: Rijnard van Tonder Date: Tue, 23 Apr 2019 23:06:55 -0400 Subject: [PATCH 2/2] rebase --- cmd/searcher/search/retry.go | 4 +++- cmd/searcher/search/retry_test.go | 7 ++++--- cmd/searcher/search/search.go | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/searcher/search/retry.go b/cmd/searcher/search/retry.go index a4154b51d1e2..d17f6e9118d8 100644 --- a/cmd/searcher/search/retry.go +++ b/cmd/searcher/search/retry.go @@ -3,11 +3,13 @@ package search import ( "os" "strings" + + "github.com/sourcegraph/sourcegraph/pkg/store" ) // getZipFileWithRetry retries getting a zip file if the zip is for some reason // invalid. -func getZipFileWithRetry(get func() (string, *zipFile, error)) (zf *zipFile, err error) { +func getZipFileWithRetry(get func() (string, *store.ZipFile, error)) (zf *store.ZipFile, err error) { var path string tries := 0 for zf == nil { diff --git a/cmd/searcher/search/retry_test.go b/cmd/searcher/search/retry_test.go index a21686027dc5..b7d8dbb0aae9 100644 --- a/cmd/searcher/search/retry_test.go +++ b/cmd/searcher/search/retry_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/pkg/errors" + "github.com/sourcegraph/sourcegraph/pkg/store" ) func TestGetZipFileWithRetry(t *testing.T) { @@ -48,7 +49,7 @@ func TestGetZipFileWithRetry(t *testing.T) { }() tries := 0 - get := func() (string, *zipFile, error) { + get := func() (string, *store.ZipFile, error) { var err error tmp, err = ioutil.TempFile("", "") if err != nil { @@ -56,9 +57,9 @@ func TestGetZipFileWithRetry(t *testing.T) { } err = test.errs[tries] - var zf *zipFile + var zf *store.ZipFile if err == nil { - zf = &zipFile{} + zf = &store.ZipFile{} } tries++ return tmp.Name(), zf, err diff --git a/cmd/searcher/search/search.go b/cmd/searcher/search/search.go index 32cbe2b414f3..9bc29a5ed0f3 100644 --- a/cmd/searcher/search/search.go +++ b/cmd/searcher/search/search.go @@ -187,12 +187,12 @@ func (s *Service) search(ctx context.Context, p *protocol.Request) (matches []pr prepareCtx, cancel := context.WithTimeout(ctx, fetchTimeout) defer cancel() - getZf := func() (string, *zipFile, error) { - path, err := s.Store.prepareZip(prepareCtx, p.GitserverRepo(), p.Commit) + getZf := func() (string, *store.ZipFile, error) { + path, err := s.Store.PrepareZip(prepareCtx, p.GitserverRepo(), p.Commit) if err != nil { return "", nil, err } - zf, err := s.Store.zipCache.get(path) + zf, err := s.Store.ZipCache.Get(path) return path, zf, err }