Skip to content
Permalink
Browse files

Merge pull request #755 from ferventcoder/ticket/GH-754/index_all_ver…

…sions

(GH-754) Index All Versions
  • Loading branch information
ferventcoder committed Dec 2, 2019
2 parents c179407 + 24f30e2 commit 65cb28ab91def9aba9c683d3625d2b3a323528f1
@@ -115,6 +115,8 @@ public PackageStatisticsStoreType PackageStatisticsStoreType

public string ScanResultsKey { get { return ReadAppSettings("ScanResultsKey", (value) => value ?? string.Empty); } }

public bool IndexContainsAllVersions { get { return ReadAppSettings("IndexContainsAllVersions", (value) => bool.Parse(value ?? bool.TrueString)); } }

protected virtual string GetConfiguredSiteRoot()
{
return ReadAppSettings("SiteRoot");
@@ -65,7 +65,7 @@ public void RegisterComponents(Container container)
container.Register(() => cacheProvider, Lifestyle.Singleton);
}

container.RegisterPerWebRequest<ISearchService, LuceneSearchService>();
container.RegisterPerWebRequest<ISearchService>(() => new LuceneSearchService(configuration.IndexContainsAllVersions));
container.RegisterPerWebRequest<IEntitiesContext>(() => new EntitiesContext());
container.RegisterPerWebRequest<IEntityRepository<User>, EntityRepository<User>>();
container.RegisterPerWebRequest<IEntityRepository<PackageRegistration>, EntityRepository<PackageRegistration>>();
@@ -81,7 +81,7 @@ public void RegisterComponents(Container container)
container.RegisterPerWebRequest<IPackageService, PackageService>();
container.RegisterPerWebRequest<ICryptographyService, CryptographyService>();

container.Register<IIndexingService, LuceneIndexingService>(Lifestyle.Singleton);
container.Register<IIndexingService>(() => new LuceneIndexingService(container.GetInstance<IEntityRepository<Package>>(), configuration.IndexContainsAllVersions), Lifestyle.Singleton);
container.Register<IFormsAuthenticationService, FormsAuthenticationService>(Lifestyle.Singleton);

container.RegisterPerWebRequest<IControllerFactory, NuGetControllerFactory>();
@@ -52,6 +52,7 @@ public interface IConfiguration
bool HostImages { get; }
int PackageOperationsUserKey { get; }
string ScanResultsKey { get; }
bool IndexContainsAllVersions { get; }

}
}
@@ -155,7 +155,7 @@ protected virtual IQueryable<Package> SearchCore(IQueryable<Package> packages, s
searchFilter.IsValid = true;
}

// use Lucene
// When the SearchFilter is valid, we can use Lucene
// We can only use Lucene if the client queries for the latest versions (IsLatest \ IsLatestStable) versions of a package
// and specific sort orders that we have in the index.
if (searchFilter.IsValid)
@@ -289,25 +289,28 @@ protected static SearchFilter GetSearchFilter(bool allVersionsInIndex, string ur
}
searchFilter.QueryTerms = queryTerms;

// We'll only use the index if we the query searches for latest \ latest-stable packages
string filter;
if (queryTerms.TryGetValue("$filter", out filter))
{
if (!(filter.Equals("IsLatestVersion", StringComparison.Ordinal) || filter.Equals("IsAbsoluteLatestVersion", StringComparison.Ordinal)
|| filter.Contains("IsLatestVersion") || filter.Contains("IsAbsoluteLatestVersion")))
{
searchFilter.IsValid = false;
searchFilter.FilterInvalidReason = SearchFilterInvalidReason.DueToAllVersionsRequested;
searchFilter.IncludeAllVersions = true;
}
}
else if (!allVersionsInIndex)
else
{
searchFilter.IsValid = false;
searchFilter.FilterInvalidReason = SearchFilterInvalidReason.DueToAllVersionsRequested;
searchFilter.IncludeAllVersions = true;
}

// We'll only use the index if we the query searches for latest \ latest-stable packages
// if all versions are not available in the index
if (searchFilter.IncludeAllVersions && !allVersionsInIndex)
{
searchFilter.IsValid = false;
searchFilter.FilterInvalidReason = SearchFilterInvalidReason.DueToAllVersionsRequested;
}

string skipStr;
if (queryTerms.TryGetValue("$skip", out skipStr))
{
@@ -44,15 +44,17 @@ public class LuceneIndexingService : IIndexingService
private readonly Directory _directory;
private IndexWriter _indexWriter;
private readonly IEntityRepository<Package> _packageRepository;
private readonly bool _indexContainsAllVersions;
private readonly Func<bool> _getShouldAutoUpdate;

public string IndexPath { get { return LuceneCommon.IndexDirectory; } }

public bool IsLocal { get { return true; } }

public LuceneIndexingService(IEntityRepository<Package> packageSource)
public LuceneIndexingService(IEntityRepository<Package> packageSource, bool indexContainsAllVersions)
{
_packageRepository = packageSource;
_indexContainsAllVersions = indexContainsAllVersions;
_directory = new LuceneFileSystem(LuceneCommon.IndexDirectory);
_getShouldAutoUpdate = () => true;
}
@@ -140,14 +142,16 @@ private List<PackageIndexEntity> GetPackages(DateTime? lastIndexTime)
// We need to do this because some attributes that we index such as DownloadCount are values in the PackageRegistration table that may
// update independent of the package.
set = set.Where(
p => (p.IsLatest || p.IsLatestStable) &&
p => (_indexContainsAllVersions || p.IsLatest || p.IsLatestStable) &&
p.PackageRegistration.Packages.Any(p2 => p2.LastUpdated > lastIndexTime));
}
else
else if (!_indexContainsAllVersions)
{
set = set.Where(p => p.IsLatest || p.IsLatestStable); // which implies that p.IsListed by the way!
}

set = set.Where(p => p.Listed);

var list = set
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
@@ -241,7 +245,7 @@ private void EnsureIndexWriterCore(bool creatingIndex)
_indexWriter = new IndexWriter(_directory, analyzer, create: creatingIndex, mfl: IndexWriter.MaxFieldLength.UNLIMITED);

// Log but swallow the exception
ErrorSignal.FromCurrentContext().Raise(ex);
ErrorSignal.FromCurrentContext().Raise(ex);
}

// Should always be add, due to locking
@@ -14,16 +14,22 @@ namespace NuGetGallery
{
public class LuceneSearchService : ISearchService
{
private readonly bool _containsAllVersions;
private static readonly string[] FieldAliases = new[] { "Id", "Title", "Tag", "Tags", "Description", "Author", "Authors", "Owner", "Owners" };
private static readonly string[] Fields = new[] { "Id", "Title", "Tags", "Description", "Authors", "Owners" };
private Lucene.Net.Store.Directory _directory;

public LuceneSearchService()
public LuceneSearchService() : this(containsAllVersions: false)
{
}

public LuceneSearchService(bool containsAllVersions)
{
_containsAllVersions = containsAllVersions;
_directory = new LuceneFileSystem(LuceneCommon.IndexDirectory);
}

public bool ContainsAllVersions { get { return false; } }
public bool ContainsAllVersions { get { return _containsAllVersions; } }

public SearchResults Search(SearchFilter searchFilter)
{
@@ -66,6 +72,12 @@ private SearchResults SearchCore(SearchFilter searchFilter)
Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));

Filter filter = new QueryWrapperFilter(filterQuery);

if (searchFilter.IncludeAllVersions)
{
filter = new QueryWrapperFilter(new TermQuery(new Term("Listed", Boolean.TrueString)));
}

var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));

if (results.TotalHits == 0 || searchFilter.CountOnly)
@@ -94,6 +106,7 @@ private static Package PackageFromDoc(Document doc)
bool isLatest = Boolean.Parse(doc.Get("IsLatest"));
bool isLatestStable = Boolean.Parse(doc.Get("IsLatestStable"));
bool isPrerelease = Boolean.Parse(doc.Get("IsPrerelease"));
bool isListed = Boolean.Parse(doc.Get("Listed"));
bool requiresLicenseAcceptance = Boolean.Parse(doc.Get("RequiresLicenseAcceptance"));
DateTime created = DateTime.Parse(doc.Get("Created"), CultureInfo.InvariantCulture);
DateTime published = DateTime.Parse(doc.Get("Published"), CultureInfo.InvariantCulture);
@@ -155,6 +168,7 @@ private static Package PackageFromDoc(Document doc)
IsLatest = isLatest,
IsLatestStable = isLatestStable,
IsPrerelease = isPrerelease,
Listed = isListed,
Language = doc.Get("Language"),
LastUpdated = lastUpdated,
Published = published,
@@ -128,6 +128,7 @@ public Document ToDocument()
document.Add(new Field("IsLatest", Package.IsLatest.to_string(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("IsLatestStable", Package.IsLatestStable.to_string(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("IsPrerelease", Package.IsPrerelease.to_string(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("Listed", Package.Listed.to_string(), Field.Store.YES, Field.Index.NOT_ANALYZED));

document.Add(new Field("Language", Package.Language.to_string(), Field.Store.YES, Field.Index.NO));
document.Add(new Field("LastUpdated", Package.LastUpdated.ToString(CultureInfo.InvariantCulture), Field.Store.YES, Field.Index.NO));
@@ -63,6 +63,7 @@
<add key="Gallery:HostImages" value="true" />
<add key="Gallery:PackageOperationsUserKey" value="0" />
<add key="Gallery:ScanResultsKey" value="12345" />
<add key="Gallery:IndexContainsAllVersions" value="true" />
<add key="Gallery:StudentDiscount" value="https://somewhere.com" />

<add key="ForceSSL" value="false" />

0 comments on commit 65cb28a

Please sign in to comment.
You can’t perform that action at this time.