Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize IndexingContentHandlers perf #13721

Closed
wants to merge 37 commits into from
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
6301402
avoid retrieving content from the database multiple times
hyzx86 May 20, 2023
a31de04
Fixed multiple versions of Lucene Index storage issues
hyzx86 May 20, 2023
78032d1
Merge branch 'main' into lucenePublish
hyzx86 Jan 29, 2024
26eb712
Merge branch 'main' into pr/13721
Skrypt Mar 22, 2024
383704c
Format and typo
Skrypt Mar 22, 2024
d2fbdcb
Optimize code
Skrypt Mar 22, 2024
d3944c0
Use Context assigned contentItem
Skrypt Mar 22, 2024
88ac104
Fix when saving draft
Skrypt Mar 22, 2024
c66e3fa
Move StoreLuceneDocument
Skrypt Mar 22, 2024
1628549
Optimize them all
Skrypt Mar 23, 2024
7a5b3e2
Merge branch 'main' into lucenePublish
hyzx86 Mar 23, 2024
bf927de
Merge branch 'main' into lucenePublish
hyzx86 Mar 27, 2024
e32983f
React to Sebastien comments
Skrypt Mar 28, 2024
e46cb6c
Merge branch 'main' into lucenePublish
hyzx86 Mar 29, 2024
9708f08
add test
Mar 29, 2024
cf2a4d0
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
Mar 29, 2024
4f4e10e
rename method
Mar 29, 2024
66c2068
update
Mar 29, 2024
0c5bfa2
update while
Mar 29, 2024
20c966e
fix unit test
Mar 29, 2024
ec3d6c7
update query template
Mar 29, 2024
dfcd821
Merge branch 'main' into lucenePublish
hyzx86 Mar 29, 2024
26acf51
Merge branch 'main' into lucenePublish
hyzx86 Apr 1, 2024
3d4d968
Merge branch 'main' into lucenePublish
hyzx86 Apr 5, 2024
5fc7c5b
Merge branch 'main' into lucenePublish
hyzx86 Apr 6, 2024
876d976
Merge branch 'main' into lucenePublish
hyzx86 Apr 23, 2024
98e694e
Merge branch 'main' into lucenePublish
hyzx86 Apr 30, 2024
5fe344e
Merge branch 'main' into lucenePublish
hyzx86 May 2, 2024
f2bfa57
FlushAsync error
hyzx86 May 2, 2024
36c7674
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
hyzx86 May 2, 2024
eb7ab7c
fix Flush
hyzx86 May 2, 2024
759c477
restore FlushAsync method
hyzx86 May 2, 2024
11c76ad
update unit test
hyzx86 May 2, 2024
c0075be
add ContentManager Test
hyzx86 May 2, 2024
1af75f6
Merge branch 'main' into lucenePublish
hyzx86 May 2, 2024
28e9928
update RunTask
hyzx86 May 2, 2024
a894ed5
Merge branch 'lucenePublish' of https://github.com/hyzx86/OrchardCore…
hyzx86 May 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,20 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon
{
// Only process the last context.
var context = ContextsById.Last();
ContentItem published = null;

ContentItem published = null, latest = null;
bool publishedLoaded = false, latestLoaded = false;
if (context is PublishContentContext publishContext)
{
if (publishContext.Cancel)
{
continue;
}

if (publishContext.PreviousItem != null)
{
published = publishContext.PreviousItem;
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
}
}

foreach (var indexSettings in await luceneIndexSettingsService.GetSettingsAsync())
{
Expand All @@ -88,35 +99,46 @@ private static async Task IndexingAsync(ShellScope scope, IEnumerable<ContentCon

if (indexSettings.IndexedContentTypes.Contains(context.ContentItem.ContentType) && !ignoreIndexedCulture)
{
if (!indexSettings.IndexLatest && !publishedLoaded)
if (context is RemoveContentContext)
{
publishedLoaded = true;
published = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published);
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
continue;
}

if (indexSettings.IndexLatest && !latestLoaded)
async Task storeLuceneDocument(ContentItem contentItem)
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
{
latestLoaded = true;
latest = await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Latest);
}
var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId), contentItem, new string[] { contentItem.ContentType }, new LuceneContentIndexSettings());
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);

var contentItem = !indexSettings.IndexLatest ? published : latest;
if (!indexSettings.IndexLatest)
{
await luceneIndexManager.DeleteDocumentsByVersionIdAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemVersionId });
}
else
{
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
}

if (contentItem == null)
{
await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { context.ContentItem.ContentItemId });
await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex });
}
else

if (!indexSettings.IndexLatest)
{
var buildIndexContext = new BuildIndexContext(new DocumentIndex(contentItem.ContentItemId, contentItem.ContentItemVersionId), contentItem, new string[] { contentItem.ContentType }, new LuceneContentIndexSettings());
await contentItemIndexHandlers.InvokeAsync(x => x.BuildIndexAsync(buildIndexContext), logger);
published ??= await contentManager.GetAsync(context.ContentItem.ContentItemId, VersionOptions.Published);

await luceneIndexManager.DeleteDocumentsAsync(indexSettings.IndexName, new string[] { contentItem.ContentItemId });
await luceneIndexManager.StoreDocumentsAsync(indexSettings.IndexName, new DocumentIndex[] { buildIndexContext.DocumentIndex });
if (published is not null)
{
await storeLuceneDocument(published);
}
}
Skrypt marked this conversation as resolved.
Show resolved Hide resolved

await storeLuceneDocument(context.ContentItem);
}
}
}
}



}
}
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ await WriteAsync(indexName, writer =>
}
});
}
public async Task DeleteDocumentsByVersionIdAsync(string indexName, IEnumerable<string> ContentItemVersionIds)
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
Skrypt marked this conversation as resolved.
Show resolved Hide resolved
{
await WriteAsync(indexName, writer =>
{
writer.DeleteDocuments(ContentItemVersionIds.Select(x => new Term("ContentItemVersionId", x)).ToArray());

writer.Commit();

if (_indexPools.TryRemove(indexName, out var pool))
{
pool.MakeDirty();
pool.Release();
}
});
}

public void DeleteIndex(string indexName)
{
Expand Down
Loading