songs search by title and artists

pavel-zhur · Jun 27, 2024 · 153fe54 · 153fe54
1 parent 9009355
commit 153fe54
Show file tree

Hide file tree

Showing 41 changed files with 552 additions and 160 deletions.
diff --git a/HarmonyDB and OneShelf.sln b/HarmonyDB and OneShelf.sln
@@ -169,7 +169,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HarmonyDB.Source", "Harmony
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HarmonyDB.Playground", "HarmonyDB.Playground", "{47FED0C3-B336-4FC6-86FE-51A78872C2DE}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HarmonyDB.Playground.Web", "HarmonyDB.Playground\HarmonyDB.Playground.Web\HarmonyDB.Playground.Web.csproj", "{C63D176C-03AB-4403-BE88-E8FE4EFE736D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HarmonyDB.Playground.Web", "HarmonyDB.Playground\HarmonyDB.Playground.Web\HarmonyDB.Playground.Web.csproj", "{C63D176C-03AB-4403-BE88-E8FE4EFE736D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HarmonyDB.Common.FullTextSearch", "HarmonyDB.Common\HarmonyDB.Common.FullTextSearch\HarmonyDB.Common.FullTextSearch.csproj", "{21D494EB-BDD0-49B0-A79D-57AE6A529F4D}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -421,6 +423,10 @@ Global
 		{C63D176C-03AB-4403-BE88-E8FE4EFE736D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C63D176C-03AB-4403-BE88-E8FE4EFE736D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C63D176C-03AB-4403-BE88-E8FE4EFE736D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{21D494EB-BDD0-49B0-A79D-57AE6A529F4D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{21D494EB-BDD0-49B0-A79D-57AE6A529F4D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{21D494EB-BDD0-49B0-A79D-57AE6A529F4D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{21D494EB-BDD0-49B0-A79D-57AE6A529F4D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -488,6 +494,7 @@ Global
 		{92969108-D9C8-4FF8-B083-B10463EB0FEC} = {2AD8DB5A-A1C3-4116-B8C5-D8DA505B1CC5}
 		{F2CFF546-5D77-4E7D-BE8B-611022AD6ECA} = {7548FF8E-E1B9-4939-96E9-03DBD5236709}
 		{C63D176C-03AB-4403-BE88-E8FE4EFE736D} = {47FED0C3-B336-4FC6-86FE-51A78872C2DE}
+		{21D494EB-BDD0-49B0-A79D-57AE6A529F4D} = {2950865E-9D1A-406E-B0C6-4C249E2E4D61}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {59093261-FDDA-411A-852D-EA21AEF83E07}

diff --git a/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/FullTextSearchExtensions.cs b/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/FullTextSearchExtensions.cs
@@ -0,0 +1,20 @@
+namespace HarmonyDB.Common.FullTextSearch;
+
+public static class FullTextSearchExtensions
+{
+    public static readonly char[] Separators =
+    {
+        '-',
+        ',',
+        ' ',
+        '\r',
+        '\n',
+        '\'',
+    };
+
+    public static string ToSearchSyntax(this string text) => text.ToLowerInvariant().Replace("ё", "е");
+
+    public static string SearchSyntaxRemoveSeparators(this string text) => text.Replace("'", "");
+
+    public static bool SearchSyntaxAnySeparatorsToRemove(this string text) => text.Contains('\'');
+}
diff --git a/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/HarmonyDB.Common.FullTextSearch.csproj b/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/HarmonyDB.Common.FullTextSearch.csproj
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+	  <PackageReadmeFile>readme.md</PackageReadmeFile>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<None Include="..\..\nuget readme.md" Pack="true" Link="readme.md" PackagePath="\readme.md" />
+	</ItemGroup>
+
+</Project>
diff --git a/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/SearchHighlightingTools.cs b/HarmonyDB.Common/HarmonyDB.Common.FullTextSearch/SearchHighlightingTools.cs
@@ -0,0 +1,83 @@
+namespace HarmonyDB.Common.FullTextSearch;
+
+public static class SearchHighlightingTools
+{
+    public static IEnumerable<(string fragment, bool isHighlighted)> GetFragments(string? query, string text)
+    {
+        if (string.IsNullOrWhiteSpace(text)) yield break;
+        var split = query?.ToLowerInvariant().ToSearchSyntax().Split(FullTextSearchExtensions.Separators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        if (split?.Length is null or 0)
+        {
+            yield return (text, false);
+            yield break;
+        }
+
+        var remaining = text.ToLowerInvariant().ToSearchSyntax().SearchSyntaxRemoveSeparators();
+        var rendering = text;
+        var anyToRemove = rendering.SearchSyntaxAnySeparatorsToRemove();
+
+        while (true)
+        {
+            var first = split
+                .Select(x => (x, index: (int?)remaining.IndexOf(x, StringComparison.Ordinal)))
+                .Where(x => x.index > -1)
+                .OrderBy(x => x.index)
+                .FirstOrDefault();
+
+            var index = first.index;
+
+            if (index == null)
+            {
+                yield return (remaining, false);
+                yield break;
+            }
+
+            var updatedIndex = index.Value;
+
+            if (updatedIndex > 0 && anyToRemove)
+            {
+                var expected = remaining.Substring(0, index.Value);
+                var renderingTests = rendering.ToLowerInvariant().ToSearchSyntax();
+                while (true)
+                {
+                    var test = renderingTests.Substring(0, updatedIndex).SearchSyntaxRemoveSeparators();
+                    if (test == expected) break;
+
+                    updatedIndex++;
+                    if (updatedIndex > 100) throw new("Protection. Failed.");
+                }
+            }
+
+            var matchLength = first.x.Length;
+            var updatedMatchLength = matchLength;
+
+            if (anyToRemove)
+            {
+                var expected = remaining.Substring(index.Value, matchLength);
+                var renderingTests = rendering.Substring(updatedIndex).ToLowerInvariant().ToSearchSyntax();
+                while (true)
+                {
+                    var test = renderingTests.Substring(0, updatedMatchLength).SearchSyntaxRemoveSeparators();
+                    if (test == expected) break;
+
+                    updatedMatchLength++;
+                    if (updatedMatchLength > 100) throw new("Protection. Failed.");
+                }
+            }
+
+            if (index == 0)
+            {
+                yield return (rendering.Substring(0, updatedMatchLength), true);
+            }
+            else
+            {
+                yield return (rendering.Substring(0, updatedIndex), false);
+                yield return (rendering.Substring(updatedIndex, updatedMatchLength), true);
+            }
+
+            remaining = remaining.Substring(index.Value + matchLength);
+            rendering = rendering.Substring(updatedIndex + updatedMatchLength);
+            if (remaining == string.Empty) yield break;
+        }
+    }
+}
diff --git a/...B.Common/Tools/TranspositionExtensions.cs → .../Transposition/TranspositionExtensions.cs b/...B.Common/Tools/TranspositionExtensions.cs → .../Transposition/TranspositionExtensions.cs
@@ -1,4 +1,4 @@
-namespace HarmonyDB.Common.Tools;
+namespace HarmonyDB.Common.Transposition;
 
 public static class TranspositionExtensions
 {

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Client/IndexApiClient.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Client/IndexApiClient.cs
@@ -25,8 +25,11 @@ public async Task<TryImportResponse> TryImport(string url)
             Url = url,
         });
 
-    public async Task<SearchResponse> Search(SearchRequest request, ApiTraceBag? apiTraceBag = null)
-        => await PostWithCode<SearchRequest, SearchResponse>(IndexApiUrls.VExternal1Search, request, apiTraceBag: apiTraceBag);
+    public async Task<SongsByChordsResponse> SongsByChords(SongsByChordsRequest request, ApiTraceBag? apiTraceBag = null)
+        => await PostWithCode<SongsByChordsRequest, SongsByChordsResponse>(IndexApiUrls.VExternal1SongsByChords, request, apiTraceBag: apiTraceBag);
+
+    public async Task<SongsByHeaderResponse> SongsByHeader(SongsByHeaderRequest request, ApiTraceBag? apiTraceBag = null)
+        => await PostWithCode<SongsByHeaderRequest, SongsByHeaderResponse>(IndexApiUrls.VExternal1SongsByHeader, request, apiTraceBag: apiTraceBag);
 
     public async Task<LoopsResponse> Loops(LoopsRequest request, ApiTraceBag? apiTraceBag = null)
         => await PostWithCode<LoopsRequest, LoopsResponse>(IndexApiUrls.VExternal1Loops, request, apiTraceBag: apiTraceBag);

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/IndexApiUrls.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/IndexApiUrls.cs
@@ -7,6 +7,7 @@ public static class IndexApiUrls
     public const string VInternalTryImport = nameof(VInternalTryImport);
     public const string VInternalGetLyrics = nameof(VInternalGetLyrics);
 
-    public const string VExternal1Search = nameof(VExternal1Search);
+    public const string VExternal1SongsByChords = nameof(VExternal1SongsByChords);
+    public const string VExternal1SongsByHeader = nameof(VExternal1SongsByHeader);
     public const string VExternal1Loops = nameof(VExternal1Loops);
 }
diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SearchResponse.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SearchResponse.cs
diff --git a/...dex.Api.Model/VExternal1/SearchRequest.cs → ....Model/VExternal1/SongsByChordsRequest.cs b/...dex.Api.Model/VExternal1/SearchRequest.cs → ....Model/VExternal1/SongsByChordsRequest.cs
@@ -2,7 +2,7 @@
 
 namespace HarmonyDB.Index.Api.Model.VExternal1;
 
-public record SearchRequest : PagedRequestBase
+public record SongsByChordsRequest : PagedRequestBase
 {
     public required string Query { get; init; }
 
@@ -13,5 +13,5 @@ public record SearchRequest : PagedRequestBase
     public int SongsPerPage { get; init; } = 100;
 
     [JsonConverter(typeof(JsonStringEnumConverter))]
-    public SearchRequestOrdering Ordering { get; init; } = SearchRequestOrdering.ByRating;
+    public SongsByChordsRequestOrdering Ordering { get; init; } = SongsByChordsRequestOrdering.ByRating;
 }
diff --git a/...Model/VExternal1/SearchRequestOrdering.cs → ...External1/SongsByChordsRequestOrdering.cs b/...Model/VExternal1/SearchRequestOrdering.cs → ...External1/SongsByChordsRequestOrdering.cs
@@ -1,6 +1,6 @@
 namespace HarmonyDB.Index.Api.Model.VExternal1;
 
-public enum SearchRequestOrdering
+public enum SongsByChordsRequestOrdering
 {
     ByCoverage,
     ByRating,

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByChordsResponse.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByChordsResponse.cs
@@ -0,0 +1,6 @@
+namespace HarmonyDB.Index.Api.Model.VExternal1;
+
+public record SongsByChordsResponse : PagedResponseBase
+{
+    public required List<SongsByChordsResponseSong> Songs { get; init; }
+}
diff --git a/...pi.Model/VExternal1/SearchResponseSong.cs → ...l/VExternal1/SongsByChordsResponseSong.cs b/...pi.Model/VExternal1/SearchResponseSong.cs → ...l/VExternal1/SongsByChordsResponseSong.cs
@@ -2,7 +2,7 @@
 
 namespace HarmonyDB.Index.Api.Model.VExternal1;
 
-public class SearchResponseSong
+public class SongsByChordsResponseSong
 {
     public required IndexHeader Header { get; set; }
 

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByHeaderRequest.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByHeaderRequest.cs
@@ -0,0 +1,10 @@
+namespace HarmonyDB.Index.Api.Model.VExternal1;
+
+public record SongsByHeaderRequest : PagedRequestBase
+{
+    public required string Query { get; init; }
+
+    public int MinRating { get; init; } = 70;
+
+    public int SongsPerPage { get; init; } = 100;
+}
diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByHeaderResponse.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/SongsByHeaderResponse.cs
@@ -0,0 +1,8 @@
+using HarmonyDB.Source.Api.Model.V1;
+
+namespace HarmonyDB.Index.Api.Model.VExternal1;
+
+public record SongsByHeaderResponse : PagedResponseBase
+{
+    public required List<IndexHeader> Songs { get; init; }
+}
diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api/Functions/VDev/IndexFunctions.cs b/HarmonyDB.Index/HarmonyDB.Index.Api/Functions/VDev/IndexFunctions.cs
@@ -24,8 +24,9 @@ public class IndexFunctions
     private readonly IndexHeadersCache _indexHeadersCache;
     private readonly InputParser _inputParser;
     private readonly ProgressionsSearch _progressionsSearch;
+    private readonly FullTextSearchCache _fullTextSearchCache;
 
-    public IndexFunctions(ILogger<IndexFunctions> logger, DownstreamApiClient downstreamApiClient, ProgressionsCache progressionsCache, LoopsStatisticsCache loopsStatisticsCache, SecurityContext securityContext, IndexHeadersCache indexHeadersCache, InputParser inputParser, ProgressionsSearch progressionsSearch)
+    public IndexFunctions(ILogger<IndexFunctions> logger, DownstreamApiClient downstreamApiClient, ProgressionsCache progressionsCache, LoopsStatisticsCache loopsStatisticsCache, SecurityContext securityContext, IndexHeadersCache indexHeadersCache, InputParser inputParser, ProgressionsSearch progressionsSearch, FullTextSearchCache fullTextSearchCache)
     {
         _logger = logger;
         _downstreamApiClient = downstreamApiClient;
@@ -34,6 +35,7 @@ public IndexFunctions(ILogger<IndexFunctions> logger, DownstreamApiClient downst
         _indexHeadersCache = indexHeadersCache;
         _inputParser = inputParser;
         _progressionsSearch = progressionsSearch;
+        _fullTextSearchCache = fullTextSearchCache;
 
         securityContext.InitService();
     }
@@ -163,4 +165,16 @@ public async Task<IActionResult> VDevFindAndCount([HttpTrigger(AuthorizationLeve
     {
         return new OkObjectResult(_progressionsSearch.Search((await _progressionsCache.Get()).Values, _inputParser.Parse(searchQuery)).Count);
     }
+
+    [Function(nameof(VDevFindHeaderAndCount))]
+    public async Task<IActionResult> VDevFindHeaderAndCount([HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequest req, string searchQuery)
+    {
+        var fullTextSearch = await _fullTextSearchCache.Get();
+        var found = fullTextSearch.Find(searchQuery);
+        return new OkObjectResult(new
+        {
+            found.Count,
+            Top100 = found.Take(100).ToList(),
+        });
+    }
 }
diff --git a/....Index.Api/Functions/VExternal1/Search.cs → ...Api/Functions/VExternal1/SongsByChords.cs b/....Index.Api/Functions/VExternal1/Search.cs → ...Api/Functions/VExternal1/SongsByChords.cs
@@ -18,7 +18,7 @@
 
 namespace HarmonyDB.Index.Api.Functions.VExternal1;
 
-public class Search : ServiceFunctionBase<SearchRequest, SearchResponse>
+public class SongsByChords : ServiceFunctionBase<SongsByChordsRequest, SongsByChordsResponse>
 {
     private readonly ProgressionsCache _progressionsCache;
     private readonly IndexHeadersCache _indexHeadersCache;
@@ -28,7 +28,7 @@ public class Search : ServiceFunctionBase<SearchRequest, SearchResponse>
     private readonly IndexApiOptions _options;
     private readonly IndexApiClient _indexApiClient;
 
-    public Search(ILoggerFactory loggerFactory, SecurityContext securityContext, ProgressionsCache progressionsCache, IndexHeadersCache indexHeadersCache, ProgressionsSearch progressionsSearch, InputParser inputParser, DownstreamApiClient downstreamApiClient, ConcurrencyLimiter concurrencyLimiter, IOptions<IndexApiOptions> options, IndexApiClient indexApiClient)
+    public SongsByChords(ILoggerFactory loggerFactory, SecurityContext securityContext, ProgressionsCache progressionsCache, IndexHeadersCache indexHeadersCache, ProgressionsSearch progressionsSearch, InputParser inputParser, DownstreamApiClient downstreamApiClient, ConcurrencyLimiter concurrencyLimiter, IOptions<IndexApiOptions> options, IndexApiClient indexApiClient)
         : base(loggerFactory, securityContext, concurrencyLimiter, options.Value.RedirectCachesToIndex)
     {
         _progressionsCache = progressionsCache;
@@ -40,15 +40,15 @@ public Search(ILoggerFactory loggerFactory, SecurityContext securityContext, Pro
         _options = options.Value;
     }
 
-    [Function(IndexApiUrls.VExternal1Search)]
-    public Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequest req, [FromBody] SearchRequest request)
+    [Function(IndexApiUrls.VExternal1SongsByChords)]
+    public Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "post")] HttpRequest req, [FromBody] SongsByChordsRequest request)
         => RunHandler(request);
 
-    protected override async Task<SearchResponse> Execute(SearchRequest request)
+    protected override async Task<SongsByChordsResponse> Execute(SongsByChordsRequest request)
     {
         if (_options.RedirectCachesToIndex)
         {
-            return await _indexApiClient.Search(request);
+            return await _indexApiClient.SongsByChords(request);
         }
 
         var progressions = await _progressionsCache.Get();
@@ -68,17 +68,17 @@ protected override async Task<SearchResponse> Execute(SearchRequest request)
             .Where(x => x.coverage >= request.MinCoverage && x.h.Value.Rating >= request.MinRating)
             .OrderByDescending<(KeyValuePair<string, IndexHeader> h, float coverage), int>(request.Ordering switch
             {
-                SearchRequestOrdering.ByRating => x =>
+                SongsByChordsRequestOrdering.ByRating => x =>
                     (int)(x.h.Value.Rating * 10 ?? 0) * 10 + (int)(x.coverage * 1000),
-                SearchRequestOrdering.ByCoverage => x =>
+                SongsByChordsRequestOrdering.ByCoverage => x =>
                     (int)(x.h.Value.Rating * 10 ?? 0) + (int)(x.coverage * 1000) * 10,
                 _ => throw new ArgumentOutOfRangeException(),
             })
             .ToList();
 
         return new()
         {
-            Songs = results.Skip((request.PageNumber - 1) * request.SongsPerPage).Take(request.SongsPerPage).Select(x => new SearchResponseSong
+            Songs = results.Skip((request.PageNumber - 1) * request.SongsPerPage).Take(request.SongsPerPage).Select(x => new SongsByChordsResponseSong
             {
                 Header = x.h.Value with
                 {