loops cache roots statistics for correct representation

pavel-zhur · Jun 29, 2024 · 90fb969 · 90fb969
1 parent cdd31e1
commit 90fb969
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 15 deletions.
diff --git a/HarmonyDB.Index/HarmonyDB.Index.Analysis.Tests/LoopTests.cs b/HarmonyDB.Index/HarmonyDB.Index.Analysis.Tests/LoopTests.cs
@@ -41,7 +41,7 @@ public void NormalizationIsCorrectForMultipleOptions(int repetitions)
             Successions = 0,
         };
 
-        loop.GetNormalizedProgression(out var invariants);
+        loop.GetNormalizedProgression(out _, out var invariants);
         Assert.Equal(repetitions, invariants);
     }
 

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Analysis/Models/Loop.cs b/HarmonyDB.Index/HarmonyDB.Index.Analysis/Models/Loop.cs
@@ -47,8 +47,11 @@ public static string Serialize(ReadOnlyMemory<CompactHarmonyMovement> progressio
         return Convert.ToBase64String(bytes);
     }
 
-    public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression() => GetNormalizedProgression(out _);
-    public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression(out int invariants)
+    public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression() => GetNormalizedProgression(out _, out _);
+
+    public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression(out int shift) => GetNormalizedProgression(out shift, out _);
+
+    public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression(out int shift, out int invariants)
     {
         var buffer = new byte[4];
         var idSequences = MemoryMarshal.ToEnumerable(Progression)
@@ -81,7 +84,8 @@ public ReadOnlyMemory<CompactHarmonyMovement> GetNormalizedProgression(out int i
             iteration++;
         }
 
-        return Enumerable.Range(shifts.Single(), Length)
+        shift = shifts.Single();
+        return Enumerable.Range(shift, Length)
             .Select(s => Progression.Span[s % Length])
             .ToArray();
     }

diff --git a/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/LoopStatistics.cs b/HarmonyDB.Index/HarmonyDB.Index.Api.Model/VExternal1/LoopStatistics.cs
@@ -13,4 +13,6 @@ public class LoopStatistics
     public required int TotalOccurrences { get; init; }
 
     public required bool IsCompound { get; init; }
+
+    public required IReadOnlyList<byte> RootsStatistics { get; init; }
 }
diff --git a/HarmonyDB.Index/HarmonyDB.Index.BusinessLogic/Models/CompactLoopStatistics.cs b/HarmonyDB.Index/HarmonyDB.Index.BusinessLogic/Models/CompactLoopStatistics.cs
@@ -12,4 +12,7 @@ public record CompactLoopStatistics
 
     [JsonPropertyName("s")]
     public int TotalSuccessions { get; set; }
+
+    [JsonPropertyName("c")]
+    public required string Counts { get; set; }
 }
diff --git a/HarmonyDB.Index/HarmonyDB.Index.BusinessLogic/Services/LoopsStatisticsCache.cs b/HarmonyDB.Index/HarmonyDB.Index.BusinessLogic/Services/LoopsStatisticsCache.cs
@@ -33,11 +33,14 @@ protected override List<LoopStatistics> ToPresentationModel(IReadOnlyDictionary<
     {
         string ToChord(int note, ChordType chordType) => $"{new Note(note, NoteAlteration.Sharp).Representation(new())}{chordType.ChordTypeToString()}";
 
+        Logger.LogInformation("{count} unique songs participating in the loops statistics.", fileModel.Values.SelectMany(x => x.ExternalIds).Distinct().Count());
+
         return fileModel
             .Select(l =>
             {
                 var sequence = Loop.Deserialize(l.Key);
-                var note = sequence.Span[0].FromType == ChordType.Minor ? 0 : 3;
+                var rootsStatistics = Convert.FromBase64String(l.Value.Counts).AsIReadOnlyList();
+                var note = rootsStatistics.WithIndices().MaxBy(x => x.x).i;
                 return new LoopStatistics
                 {
                     Progression = string.Join(" ", ToChord(note, sequence.Span[0].FromType)
@@ -55,6 +58,7 @@ protected override List<LoopStatistics> ToPresentationModel(IReadOnlyDictionary<
                     TotalSuccessions = l.Value.TotalSuccessions,
                     TotalSongs = l.Value.ExternalIds.Count,
                     IsCompound = _progressionsSearch.IsCompound(sequence),
+                    RootsStatistics = rootsStatistics,
                 };
             })
             .OrderByDescending(x => x.TotalOccurrences)
@@ -67,11 +71,11 @@ public async Task Rebuild()
         await StreamCompressSerialize(await GetAllLoops(await _progressionsCache.Get()));
     }
 
-    private async Task<ConcurrentDictionary<string, CompactLoopStatistics>> GetAllLoops(IReadOnlyDictionary<string, CompactChordsProgression> dictionary)
+    private async Task<Dictionary<string, CompactLoopStatistics>> GetAllLoops(IReadOnlyDictionary<string, CompactChordsProgression> dictionary)
     {
         var cc = 0;
         var cf = 0;
-        ConcurrentDictionary<string, CompactLoopStatistics> loopsBag = new();
+        ConcurrentDictionary<string, (CompactLoopStatistics loopStatistics, int[] counts)> loopStatisticsBag = new();
 
         await Parallel.ForEachAsync(dictionary, (x, _) =>
         {
@@ -82,17 +86,28 @@ public async Task Rebuild()
 
                 foreach (var loop in loops)
                 {
-                    var serialized = Loop.Serialize(loop.GetNormalizedProgression());
-                    var bag = loopsBag.GetOrAdd(serialized, _ => new()
+                    var serialized = Loop.Serialize(loop.GetNormalizedProgression(out var shift));
+                    var root = Note.Normalize(
+                        compactChordsProgression
+                            .ExtendedHarmonyMovementsSequences[loop.SequenceIndex]
+                            .FirstRoot
+                        + MemoryMarshal.ToEnumerable(compactChordsProgression
+                                .ExtendedHarmonyMovementsSequences[loop.SequenceIndex].Movements)
+                            .Take(loop.Start + shift)
+                            .Sum(x => x.RootDelta));
+
+                    var (loopStatistics, counts) = loopStatisticsBag.GetOrAdd(serialized, _ => (new()
                     {
                         ExternalIds = new(),
-                    });
+                        Counts = null!, // will be overridden later
+                    }, new int[12]));
 
-                    lock (bag)
+                    lock (loopStatistics)
                     {
-                        bag.ExternalIds.Add(externalId);
-                        bag.TotalOccurrences += loop.Occurrences;
-                        bag.TotalSuccessions += loop.Successions;
+                        loopStatistics.ExternalIds.Add(externalId);
+                        loopStatistics.TotalOccurrences += loop.Occurrences;
+                        loopStatistics.TotalSuccessions += loop.Successions;
+                        counts[root]++;
                     }
                 }
 
@@ -107,6 +122,16 @@ public async Task Rebuild()
             return ValueTask.CompletedTask;
         });
 
-        return loopsBag;
+        return loopStatisticsBag
+            .ToDictionary(
+                x => x.Key, 
+                x =>
+                {
+                    var max = x.Value.counts.Max();
+                    return x.Value.loopStatistics with
+                    {
+                        Counts = Convert.ToBase64String(x.Value.counts.Select(x => (byte)(max <= 255 ? x : x * 255 / max)).ToArray()),
+                    };
+                });
     }
 }
diff --git a/HarmonyDB.Playground/HarmonyDB.Playground.Web/Models/LoopsModel.cs b/HarmonyDB.Playground/HarmonyDB.Playground.Web/Models/LoopsModel.cs
@@ -7,4 +7,6 @@ public record LoopsModel : LoopsRequest
     public bool IncludeTrace { get; init; }
 
     public bool JustForm { get; set; }
+
+    public bool IncludeRootsStatistics { get; init; }
 }
diff --git a/HarmonyDB.Playground/HarmonyDB.Playground.Web/Views/Home/Loops.cshtml b/HarmonyDB.Playground/HarmonyDB.Playground.Web/Views/Home/Loops.cshtml
@@ -48,6 +48,12 @@
             @ViewLocalizer["Include the Index API trace for developers"]
         </label>
     </p>
+    <p>
+        <label>
+            @Html.CheckBoxFor(x => x.IncludeRootsStatistics)
+            @ViewLocalizer["Include the roots statistics trace"]
+        </label>
+    </p>
     <p>
         @ViewLocalizer["Compound"]:
         @Html.DropDownListFor(x => x.Compound, Enum.GetValues<LoopsRequestCompoundFilter>().Select(x => new SelectListItem(x.ToString(), x.ToString())))
@@ -73,6 +79,10 @@
                 <td>@ViewLocalizer["Total Occurrences"]</td>
                 <td>@ViewLocalizer["Total Successions"]</td>
                 <td>@ViewLocalizer["Compound"]</td>
+                @if (Model.IncludeRootsStatistics)
+                {
+                    <td>@ViewLocalizer["Roots Statistics"]</td>
+                }
             </tr>
         </thead>
         @foreach (var loop in response.Loops)
@@ -84,6 +94,10 @@
                 <td>@loop.TotalOccurrences.ToString("N0")</td>
                 <td>@loop.TotalSuccessions.ToString("N0")</td>
                 <td>@loop.IsCompound</td>
+                @if (Model.IncludeRootsStatistics)
+                {
+                    <td>@string.Join(", ", loop.RootsStatistics)</td>
+                }
             </tr>
         }
     </table>

diff --git a/HarmonyDB.Playground/HarmonyDB.Playground.Web/Views/Home/Loops.ru.resx b/HarmonyDB.Playground/HarmonyDB.Playground.Web/Views/Home/Loops.ru.resx
@@ -126,6 +126,9 @@
   <data name="Include the Index API trace for developers" xml:space="preserve">
     <value>Показать запросы к Index API для разработчиков</value>
   </data>
+  <data name="Include the roots statistics trace" xml:space="preserve">
+    <value>Показать отладку статистики тональностей цикла</value>
+  </data>
   <data name="Length" xml:space="preserve">
     <value>Длина</value>
   </data>
@@ -168,6 +171,9 @@
   <data name="Results" xml:space="preserve">
     <value>Результаты</value>
   </data>
+  <data name="Roots Statistics" xml:space="preserve">
+    <value>Статистика тональностей</value>
+  </data>
   <data name="See the most popular chord progressions" xml:space="preserve">
     <value>Популярные аккордовые последовательности</value>
   </data>