From bfa69f96a4c2a113728392902127e0591ae82af1 Mon Sep 17 00:00:00 2001 From: Khang Date: Sun, 16 Oct 2022 11:22:11 -0400 Subject: [PATCH 1/3] Add data versioning --- Mapperator.ConsoleApp/Program.cs | 10 +++++----- .../Mapperator.DemoApp.Game/MainScreen.cs | 3 ++- Mapperator/DataExtractor.cs | 6 ++++-- Mapperator/DataSerializer.cs | 14 ++++++++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Mapperator.ConsoleApp/Program.cs b/Mapperator.ConsoleApp/Program.cs index 298d814..98453dc 100644 --- a/Mapperator.ConsoleApp/Program.cs +++ b/Mapperator.ConsoleApp/Program.cs @@ -173,7 +173,7 @@ class SearchOptions { if (opts.OutputStructName is null) throw new ArgumentNullException(nameof(opts)); if (opts.DataPath is null) throw new ArgumentNullException(nameof(opts)); - var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt"))); + var (_, trainData) = DataSerializer.DeserializeBeatmapData(Path.ChangeExtension(opts.DataPath, ".txt")); var data = new RhythmDistanceTrieStructure(); if (data is not ISerializable sMatcher) { @@ -229,9 +229,9 @@ class SearchOptions { stopwatch.Start(); Console.WriteLine(Strings.Program_DoMapConvert_Extracting_data___); - var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt"))); + var (trainVersion, trainData) = DataSerializer.DeserializeBeatmapData(Path.ChangeExtension(opts.DataPath, ".txt")); var map = new BeatmapEditor(Path.ChangeExtension(opts.InputBeatmapPath, ".osu")).ReadFile(); - var input = new DataExtractor().ExtractBeatmapData(map).ToArray(); + var input = new DataExtractor(trainVersion).ExtractBeatmapData(map).ToArray(); // TODO: add options to automatically add distance spacing // TODO: also add options for ignoring angles, nc, or slider attributes @@ -243,7 +243,7 @@ class SearchOptions { if (opts.SpacingBeatmapPath is not null) { Console.WriteLine(Strings.Program_DoMapConvert_Converting_spacing_to_reference_beatmap___); var spacingMap = new BeatmapEditor(Path.ChangeExtension(opts.SpacingBeatmapPath, ".osu")).ReadFile(); - var spacingMapData = new DataExtractor().ExtractBeatmapData(spacingMap).ToArray(); + var spacingMapData = new DataExtractor(trainVersion).ExtractBeatmapData(spacingMap).ToArray(); input = TransferSpacing(spacingMapData, input); } @@ -346,7 +346,7 @@ class SearchOptions { } }).Where(ValidBeatmap) .SelectMany(b => mirrors.Select(m => extractor.ExtractBeatmapData(b!, m))) - )); + ).Prepend(DataSerializer.CurrentHeader)); return 0; } diff --git a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs index e0032d7..f064ff0 100644 --- a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs +++ b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs @@ -221,7 +221,8 @@ private void OnBeatmapChange(ValueChangedEvent obj) pos.MaxValue = obj.NewValue.HitObjects.Count - length - 1; - pattern = new DataExtractor().ExtractBeatmapData(obj.NewValue).ToArray(); + // TODO: Add versioning support. Currently hardcoded to v1 + pattern = new DataExtractor(1).ExtractBeatmapData(obj.NewValue).ToArray(); matcher = new TrieDataMatcher2(dataStruct, pattern); filter = new OnScreenFilter(); sorter = new BestScoreOrderFilter(new SuperJudge(), pattern, matcher); diff --git a/Mapperator/DataExtractor.cs b/Mapperator/DataExtractor.cs index cca245e..86c0165 100644 --- a/Mapperator/DataExtractor.cs +++ b/Mapperator/DataExtractor.cs @@ -10,11 +10,13 @@ namespace Mapperator { public class DataExtractor { private readonly HitObjectEncoder encoder; + private readonly int dataVersion; - public DataExtractor() : this(new HitObjectEncoder()) { } + public DataExtractor(int dataVersion = DataSerializer.CurrentDataVersion) : this(new HitObjectEncoder(), dataVersion) { } - public DataExtractor(HitObjectEncoder encoder) { + public DataExtractor(HitObjectEncoder encoder, int dataVersion) { this.encoder = encoder; + this.dataVersion = dataVersion; } public IEnumerable ExtractBeatmapData(IBeatmap beatmap, bool mirror = false) { diff --git a/Mapperator/DataSerializer.cs b/Mapperator/DataSerializer.cs index 8dd9dd2..190f563 100644 --- a/Mapperator/DataSerializer.cs +++ b/Mapperator/DataSerializer.cs @@ -5,7 +5,11 @@ namespace Mapperator { public static class DataSerializer { + public const int CurrentDataVersion = 1; private const string BeatmapSeparator = "/-\\_/-\\_/-\\"; + private const string DataHeader = "Mapperator file format v"; + + public static string CurrentHeader => $"{DataHeader}{CurrentDataVersion}"; public static IEnumerable SerializeBeatmapData(IEnumerable> data) { foreach (var beatmap in data) { @@ -21,6 +25,16 @@ public static class DataSerializer { return data.ToString(); } + public static (int, IEnumerable>) DeserializeBeatmapData(string dataPath) { + // ReSharper disable twice PossibleMultipleEnumeration + var lines = File.ReadLines(dataPath); + var firstLine = lines.FirstOrDefault() ?? ""; + if (firstLine.StartsWith(DataHeader)) { + return (int.Parse(firstLine.Split('v').Last()), DeserializeBeatmapData(lines.Skip(1))); + } + return (1, DeserializeBeatmapData(lines)); + } + public static IEnumerable> DeserializeBeatmapData(IEnumerable data) { return data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample)); } From 11d5566a53e728845660870bea71c6a7a1f49ca0 Mon Sep 17 00:00:00 2001 From: Khang Date: Sun, 16 Oct 2022 11:40:07 -0400 Subject: [PATCH 2/3] Properly count linear segments and increment data version --- Mapperator/DataExtractor.cs | 14 +++++++++++--- Mapperator/DataSerializer.cs | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Mapperator/DataExtractor.cs b/Mapperator/DataExtractor.cs index 86c0165..891ef90 100644 --- a/Mapperator/DataExtractor.cs +++ b/Mapperator/DataExtractor.cs @@ -45,9 +45,17 @@ public class DataExtractor { var segments = 0; var controlPoints = path.ControlPoints; - for (var i = 0; i < controlPoints.Count; i++) { - if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) { - segments++; + if (dataVersion >= 2 && slider.SliderType == PathType.Linear) { + for (var i = 0; i < controlPoints.Count - 1; i++) { + if (controlPoints[i] != controlPoints[i + 1]) { + segments++; + } + } + } else { + for (var i = 0; i < controlPoints.Count; i++) { + if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) { + segments++; + } } } diff --git a/Mapperator/DataSerializer.cs b/Mapperator/DataSerializer.cs index 190f563..955fc11 100644 --- a/Mapperator/DataSerializer.cs +++ b/Mapperator/DataSerializer.cs @@ -5,7 +5,7 @@ namespace Mapperator { public static class DataSerializer { - public const int CurrentDataVersion = 1; + public const int CurrentDataVersion = 2; private const string BeatmapSeparator = "/-\\_/-\\_/-\\"; private const string DataHeader = "Mapperator file format v"; From 7dd6645a4e7ae7f21209ccc4496c392d61343c2c Mon Sep 17 00:00:00 2001 From: Khang Date: Sun, 16 Oct 2022 15:03:41 -0400 Subject: [PATCH 3/3] Rework to use IEnumerable --- Mapperator.ConsoleApp/Program.cs | 4 ++-- .../Mapperator.DemoApp.Game/MainScreen.cs | 1 - .../Mapperator.DemoApp.Game/MapDataStore.cs | 11 +++++++++-- Mapperator/DataSerializer.cs | 13 ++++--------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Mapperator.ConsoleApp/Program.cs b/Mapperator.ConsoleApp/Program.cs index 98453dc..594e495 100644 --- a/Mapperator.ConsoleApp/Program.cs +++ b/Mapperator.ConsoleApp/Program.cs @@ -173,7 +173,7 @@ class SearchOptions { if (opts.OutputStructName is null) throw new ArgumentNullException(nameof(opts)); if (opts.DataPath is null) throw new ArgumentNullException(nameof(opts)); - var (_, trainData) = DataSerializer.DeserializeBeatmapData(Path.ChangeExtension(opts.DataPath, ".txt")); + var (_, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt"))); var data = new RhythmDistanceTrieStructure(); if (data is not ISerializable sMatcher) { @@ -229,7 +229,7 @@ class SearchOptions { stopwatch.Start(); Console.WriteLine(Strings.Program_DoMapConvert_Extracting_data___); - var (trainVersion, trainData) = DataSerializer.DeserializeBeatmapData(Path.ChangeExtension(opts.DataPath, ".txt")); + var (trainVersion, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt"))); var map = new BeatmapEditor(Path.ChangeExtension(opts.InputBeatmapPath, ".osu")).ReadFile(); var input = new DataExtractor(trainVersion).ExtractBeatmapData(map).ToArray(); diff --git a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs index f064ff0..cb04084 100644 --- a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs +++ b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs @@ -221,7 +221,6 @@ private void OnBeatmapChange(ValueChangedEvent obj) pos.MaxValue = obj.NewValue.HitObjects.Count - length - 1; - // TODO: Add versioning support. Currently hardcoded to v1 pattern = new DataExtractor(1).ExtractBeatmapData(obj.NewValue).ToArray(); matcher = new TrieDataMatcher2(dataStruct, pattern); filter = new OnScreenFilter(); diff --git a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs index e9214d5..b07a09d 100644 --- a/Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs +++ b/Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Collections.Generic; using System.IO; @@ -36,7 +37,10 @@ public IEnumerable> Get(string name) using Stream stream = store.GetStream(name); if (stream is null) return null; using StreamReader reader = new StreamReader(stream); - return DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray()); + var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray()); + if (version != 1) + throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore"); + return data; } public Task>> GetAsync(string name, CancellationToken cancellationToken = new()) @@ -44,7 +48,10 @@ public Task>> GetAsync(string name, Cancel using Stream stream = store.GetStream(name); if (stream is null) return null; using StreamReader reader = new StreamReader(stream); - return Task.FromResult(DataSerializer.DeserializeBeatmapData(iterateLines(reader))); + var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader)); + if (version != 1) + throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore"); + return Task.FromResult(data); } public Stream GetStream(string name) diff --git a/Mapperator/DataSerializer.cs b/Mapperator/DataSerializer.cs index 955fc11..9c960a2 100644 --- a/Mapperator/DataSerializer.cs +++ b/Mapperator/DataSerializer.cs @@ -25,18 +25,13 @@ public static class DataSerializer { return data.ToString(); } - public static (int, IEnumerable>) DeserializeBeatmapData(string dataPath) { + public static (int, IEnumerable>) DeserializeBeatmapData(IEnumerable data) { // ReSharper disable twice PossibleMultipleEnumeration - var lines = File.ReadLines(dataPath); - var firstLine = lines.FirstOrDefault() ?? ""; + var firstLine = data.FirstOrDefault() ?? ""; if (firstLine.StartsWith(DataHeader)) { - return (int.Parse(firstLine.Split('v').Last()), DeserializeBeatmapData(lines.Skip(1))); + return (int.Parse(firstLine.Split('v').Last()), data.Skip(1).Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample))); } - return (1, DeserializeBeatmapData(lines)); - } - - public static IEnumerable> DeserializeBeatmapData(IEnumerable data) { - return data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample)); + return (1, data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample))); } public static MapDataPoint DeserializeBeatmapDataSample(string data) {