Skip to content

Commit

Permalink
Added a second search unit test, and converted LuceneIndexingService and
Browse files Browse the repository at this point in the history 
LuceneSearchService to use in-memory indexes during unit test runs.
  • Loading branch information
Tim Lovell-Smith committed Dec 12, 2012
1 parent 3730be7 commit a2caa0c
Show file tree
Hide file tree
Showing 10 changed files with 158 additions and 67 deletions.
5 changes: 5 additions & 0 deletions Facts/Facts.csproj
View file
Expand Up @@ -46,6 +46,11 @@
<HintPath>..\packages\EntityFramework.4.3.1\lib\net40\EntityFramework.dll</HintPath>
</Reference>
<Reference Include="MarkdownSharp">
<Reference Include="Lucene.Net, Version=2.9.2.2, Culture=neutral, PublicKeyToken=d590fe6f575d2f50, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Lucene.Net.2.9.2.2\lib\net40\Lucene.Net.dll</HintPath>
</Reference>
<Reference Include="MarkdownSharp, Version=0.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MarkdownSharp.1.13.0.0\lib\35\MarkdownSharp.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Edm, Version=5.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
Expand Down
84 changes: 79 additions & 5 deletions Facts/Infrastructure/LuceneSearchServiceFacts.cs
View file
@@ -1,5 +1,7 @@
using Moq;
using Lucene.Net.Store;
using Moq;
using NuGetGallery.Services;
using System;
using System.Collections.Generic;
using System.Linq;
using Xunit;
Expand Down Expand Up @@ -27,15 +29,18 @@ public void IndexAndSearchAPackage()
Title = "Package #1 4.2.0",
Description = "Package #1 is an awesome package",
Listed = true,
IsLatestStable = true,
IsLatest = true,
IsPrerelease = true,
DownloadCount = 100,
FlattenedAuthors = "",
}
}.AsQueryable());

var luceneIndexingService = new LuceneIndexingService(packageSource.Object);
luceneIndexingService.UpdateIndex();
var luceneIndexingService = CreateIndexingService(packageSource);
luceneIndexingService.UpdateIndex(forceRefresh: true);

var luceneSearchService = new LuceneSearchService();
var luceneSearchService = CreateSearchService(packageSource);
int totalHits = 0;
var searchFilter = new SearchFilter
{
Expand All @@ -51,7 +56,76 @@ public void IndexAndSearchAPackage()

Assert.Single(results);
Assert.Equal(3, results[0].Key);
Assert.Equal(1, results[0].Key);
Assert.Equal(1, results[0].PackageRegistrationKey);
}

[Fact]
public void IndexAndSearchDavid123For12()
{
var packageSource = new Mock<IPackageSource>();
packageSource.Setup(x => x.GetPackagesForIndexing(null)).Returns(
new List<Package>
{
new Package
{
Key = 49246,
PackageRegistrationKey = 12500,
PackageRegistration = new PackageRegistration
{
Id = "DavidTest123",
Key = 12500,
DownloadCount = 495
},
Created = DateTime.Parse("2011-07-18 22:29:46.893"),
Title = "DavidTest123",
Description = "Description",
DownloadCount = 469,
HashAlgorithm = "SHA512",
Hash = "1unECLYoz4z1C5DiIkdnptHvodvNkbLUIev28Y6wOG8EohgPLNp1w7Qa7H1M6upa4tXYlbsenDjFgQIpNHhU3w==",
LastUpdated = DateTime.Parse("2012-11-26 04:17:21.723"),
Listed = true,
IsLatest = true,
IsLatestStable = true,
Published = DateTime.Parse("1900-01-01 00:00:00.000"),
FlattenedAuthors = "",
PackageFileSize = 4429,
}
}.AsQueryable());

var luceneIndexingService = CreateIndexingService(packageSource);
luceneIndexingService.UpdateIndex();

var luceneSearchService = CreateSearchService(packageSource);
int totalHits = 0;
var searchFilter = new SearchFilter
{
Skip = 0,
Take = 10,
SearchTerm = "12",
};

var results = luceneSearchService.Search(
packageSource.Object.GetPackagesForIndexing(null),
searchFilter,
out totalHits).ToList();

Assert.Empty(results);
}

static Directory d = LuceneCommon.GetRAMDirectory();

private LuceneIndexingService CreateIndexingService(Mock<IPackageSource> packageSource)
{
d = LuceneCommon.GetRAMDirectory();
return new LuceneIndexingService(
packageSource.Object,
d);
}

private LuceneSearchService CreateSearchService(Mock<IPackageSource> packageSource)
{
return new LuceneSearchService(
d);
}
}
}
3 changes: 2 additions & 1 deletion Facts/packages.config
View file
Expand Up @@ -2,6 +2,7 @@
<packages>
<package id="AnglicanGeek.MarkdownMailer" version="1.2" />
<package id="EntityFramework" version="4.3.1" />
<package id="Lucene.Net" version="2.9.2.2" targetFramework="net45" />
<package id="MarkdownSharp" version="1.13.0.0" targetFramework="net45" />
<package id="Microsoft.AspNet.Mvc" version="4.0.20710.0" targetFramework="net45" />
<package id="Microsoft.AspNet.Razor" version="2.0.20715.0" targetFramework="net45" />
Expand All @@ -20,4 +21,4 @@
<package id="WindowsAzure.Storage" version="2.0.1.0" targetFramework="net45" />
<package id="xunit" version="1.9.1" targetFramework="net45" />
<package id="xunit.extensions" version="1.9.1" targetFramework="net45" />
</packages>
</packages>
4 changes: 4 additions & 0 deletions Website/App_Start/ContainerBindings.cs
View file
Expand Up @@ -36,6 +36,10 @@ public override void Load()

Bind<GallerySetting>().ToMethod(c => gallerySetting.Value);

Bind<Lucene.Net.Store.Directory>()
.ToMethod((_) => LuceneCommon.GetDirectory())
.InSingletonScope();

Bind<ISearchService>()
.To<LuceneSearchService>()
.InRequestScope();
Expand Down
25 changes: 21 additions & 4 deletions Website/Infrastructure/Lucene/LuceneCommon.cs
View file
Expand Up @@ -2,6 +2,7 @@
using System.Web;
using Lucene.Net.Util;
using System.Web.Hosting;
using Lucene.Net.Store;

namespace NuGetGallery
{
Expand All @@ -13,10 +14,26 @@ internal static class LuceneCommon

static LuceneCommon()
{
// Fall back to a temp folder when run with no HostingEnvironment i.e. in context of unit tests.
IndexDirectory = HostingEnvironment.MapPath("~/AppData/Lucene") ??
Path.Combine(Path.GetTempPath(), "AppData", "Lucene");
IndexMetadataPath = Path.Combine(IndexDirectory, "index.metadata");
IndexDirectory = HostingEnvironment.MapPath("~/AppData/Lucene");
IndexMetadataPath = Path.Combine(IndexDirectory ?? ".", "index.metadata");
}

// Factory method for DI/IOC that creates the directory the index is stored in.
// Used by real website. Bypassed for unit tests.
internal static Lucene.Net.Store.Directory GetDirectory()
{
if (!System.IO.Directory.Exists(IndexDirectory))
{
System.IO.Directory.CreateDirectory(IndexDirectory);
}

var directoryInfo = new DirectoryInfo(IndexDirectory);
return new SimpleFSDirectory(directoryInfo);
}

public static Lucene.Net.Store.Directory GetRAMDirectory()
{
return new RAMDirectory();
}
}
}
19 changes: 0 additions & 19 deletions Website/Infrastructure/Lucene/LuceneFileSystem.cs
View file

This file was deleted.

3 changes: 2 additions & 1 deletion Website/Infrastructure/Lucene/LuceneIndexingJob.cs
View file
Expand Up @@ -13,7 +13,8 @@ public LuceneIndexingJob(TimeSpan frequence, TimeSpan timeout)
: base("Lucene", frequence, timeout)
{
_indexingService = new LuceneIndexingService(
new PackageSource(new EntitiesContext()));
new PackageSource(new EntitiesContext()),
LuceneCommon.GetDirectory());
_indexingService.UpdateIndex();
}

Expand Down
31 changes: 16 additions & 15 deletions Website/Infrastructure/Lucene/LuceneIndexingService.cs
View file
Expand Up @@ -21,24 +21,30 @@ public class LuceneIndexingService : IIndexingService
private static readonly object IndexWriterLock = new object();
private static readonly TimeSpan IndexRecreateInterval = TimeSpan.FromDays(3);
private static readonly char[] IdSeparators = new[] { '.', '-' };
private static Lucene.Net.Store.Directory _directory;
private static IndexWriter _indexWriter;
private readonly IPackageSource _packageSource;

[Inject]
public LuceneIndexingService(IPackageSource packageSource)
public LuceneIndexingService(
IPackageSource packageSource,
Lucene.Net.Store.Directory directory)
{
_packageSource = packageSource;
_directory = directory;
}

public void UpdateIndex()
{
DateTime? lastWriteTime = GetLastWriteTime();
bool creatingIndex = lastWriteTime == null;
UpdateIndex(forceRefresh: false);
}

EnsureIndexWriter(creatingIndex);
internal void UpdateIndex(bool forceRefresh)
{
DateTime? lastWriteTime = GetLastWriteTime();

if (IndexRequiresRefresh())
if ((lastWriteTime == null) || IndexRequiresRefresh() || forceRefresh)
{
EnsureIndexWriter(creatingIndex: true);
_indexWriter.DeleteAll();
_indexWriter.Commit();

Expand All @@ -48,10 +54,11 @@ public void UpdateIndex()
// Set the index create time to now. This would tell us when we last rebuilt the index.
UpdateIndexRefreshTime();
}

var packages = GetPackages(lastWriteTime);
if (packages.Count > 0)
{
EnsureIndexWriter(creatingIndex: lastWriteTime == null);
AddPackages(packages, creatingIndex: lastWriteTime == null);
}

Expand Down Expand Up @@ -177,15 +184,8 @@ protected static void EnsureIndexWriter(bool creatingIndex)

private static void EnsureIndexWriterCore(bool creatingIndex)
{
if (!Directory.Exists(LuceneCommon.IndexDirectory))
{
Directory.CreateDirectory(LuceneCommon.IndexDirectory);
}

var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var directoryInfo = new DirectoryInfo(LuceneCommon.IndexDirectory);
var directory = new SimpleFSDirectory(directoryInfo);
_indexWriter = new IndexWriter(directory, analyzer, create: creatingIndex, mfl: IndexWriter.MaxFieldLength.UNLIMITED);
_indexWriter = new IndexWriter(_directory, analyzer, create: creatingIndex, mfl: IndexWriter.MaxFieldLength.UNLIMITED);
}

protected internal bool IndexRequiresRefresh()
Expand All @@ -195,6 +195,7 @@ protected internal bool IndexRequiresRefresh()
var creationTime = File.GetCreationTimeUtc(LuceneCommon.IndexMetadataPath);
return (DateTime.UtcNow - creationTime) > IndexRecreateInterval;
}

// If we've never created the index, it needs to be refreshed.
return true;
}
Expand Down
50 changes: 29 additions & 21 deletions Website/Infrastructure/Lucene/LuceneSearchService.cs
View file
Expand Up @@ -7,11 +7,19 @@
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Lucene.Net.Search.Function;
using System.Diagnostics;

namespace NuGetGallery
{
public class LuceneSearchService : ISearchService
{
private Lucene.Net.Store.Directory _directory;

public LuceneSearchService(Lucene.Net.Store.Directory directory)
{
_directory = directory;
}

public IQueryable<Package> Search(IQueryable<Package> packages, SearchFilter searchFilter, out int totalHits)
{
if (packages == null)
Expand Down Expand Up @@ -60,36 +68,36 @@ private static Package LookupPackage(Dictionary<int, Package> dict, int key)
return package;
}

private static IList<int> SearchCore(SearchFilter searchFilter, out int totalHits)
private IList<int> SearchCore(SearchFilter searchFilter, out int totalHits)
{
if (!Directory.Exists(LuceneCommon.IndexDirectory))
{
totalHits = 0;
return new int[0];
}

SortField sortField = GetSortField(searchFilter);
int numRecords = searchFilter.Skip + searchFilter.Take;

using (var directory = new LuceneFileSystem(LuceneCommon.IndexDirectory))
IndexSearcher searcher;
try
{
searcher = new IndexSearcher(_directory, readOnly: true);
}
catch (FileNotFoundException)
{
var searcher = new IndexSearcher(directory, readOnly: true);
var query = ParseQuery(searchFilter);
totalHits = 0;
return new List<int>();
}

var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
var termQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
Filter filter = new QueryWrapperFilter(termQuery);
var query = ParseQuery(searchFilter);

var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
var termQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
Filter filter = new QueryWrapperFilter(termQuery);

var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(sortField));
var keys = results.scoreDocs.Skip(searchFilter.Skip)
.Select(c => ParseKey(searcher.Doc(c.doc).Get("Key")))
.ToList();
var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(sortField));
var keys = results.scoreDocs.Skip(searchFilter.Skip)
.Select(c => ParseKey(searcher.Doc(c.doc).Get("Key")))
.ToList();

totalHits = results.totalHits;
searcher.Close();
return keys;
}
totalHits = results.totalHits;
searcher.Close();
return keys;
}

private static Query ParseQuery(SearchFilter searchFilter)
Expand Down
1 change: 0 additions & 1 deletion Website/Website.csproj
View file
Expand Up @@ -297,7 +297,6 @@
<Compile Include="Infrastructure\HttpHeaderValueProviderFactory.cs" />
<Compile Include="Infrastructure\HttpStatusCodeWithBodyResult.cs" />
<Compile Include="Infrastructure\JsonNetResult.cs" />
<Compile Include="Infrastructure\Lucene\LuceneFileSystem.cs" />
<Compile Include="Infrastructure\Lucene\LuceneCommon.cs" />
<Compile Include="Infrastructure\Lucene\LuceneIndexingJob.cs" />
<Compile Include="Infrastructure\Lucene\LuceneIndexingService.cs" />
Expand Down

0 comments on commit a2caa0c

Please sign in to comment.