Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add data versioning and properly count linear segments #4

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add data versioning and properly count linear segments #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bfa69f9
Add data versioning
khang06 Oct 16, 2022
11d5566
Properly count linear segments and increment data version
khang06 Oct 16, 2022
7dd6645
Rework to use IEnumerable<string>
khang06 Oct 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Mapperator.ConsoleApp/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ private static int DoBuildGraph(BuildOptions opts) {
if (opts.OutputStructName is null) throw new ArgumentNullException(nameof(opts));
if (opts.DataPath is null) throw new ArgumentNullException(nameof(opts));

var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt")));
var (_, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt")));
var data = new RhythmDistanceTrieStructure();

if (data is not ISerializable sMatcher) {
Expand Down Expand Up @@ -229,9 +229,9 @@ private static int DoMapConvert(ConvertOptions opts) {
stopwatch.Start();

Console.WriteLine(Strings.Program_DoMapConvert_Extracting_data___);
var trainData = DataSerializer.DeserializeBeatmapData(File.ReadLines(Path.ChangeExtension(opts.DataPath, ".txt")));
var (trainVersion, trainData) = DataSerializer.DeserializeBeatmapData(File.ReadAllLines(Path.ChangeExtension(opts.DataPath, ".txt")));
var map = new BeatmapEditor(Path.ChangeExtension(opts.InputBeatmapPath, ".osu")).ReadFile();
var input = new DataExtractor().ExtractBeatmapData(map).ToArray();
var input = new DataExtractor(trainVersion).ExtractBeatmapData(map).ToArray();

// TODO: add options to automatically add distance spacing
// TODO: also add options for ignoring angles, nc, or slider attributes
Expand All @@ -243,7 +243,7 @@ private static int DoMapConvert(ConvertOptions opts) {
if (opts.SpacingBeatmapPath is not null) {
Console.WriteLine(Strings.Program_DoMapConvert_Converting_spacing_to_reference_beatmap___);
var spacingMap = new BeatmapEditor(Path.ChangeExtension(opts.SpacingBeatmapPath, ".osu")).ReadFile();
var spacingMapData = new DataExtractor().ExtractBeatmapData(spacingMap).ToArray();
var spacingMapData = new DataExtractor(trainVersion).ExtractBeatmapData(spacingMap).ToArray();
input = TransferSpacing(spacingMapData, input);
}

Expand Down Expand Up @@ -346,7 +346,7 @@ private static int DoDataExtraction(ExtractOptions opts) {
}
}).Where(ValidBeatmap)
.SelectMany(b => mirrors.Select(m => extractor.ExtractBeatmapData(b!, m)))
));
).Prepend(DataSerializer.CurrentHeader));
return 0;
}

Expand Down
2 changes: 1 addition & 1 deletion Mapperator.DemoApp/Mapperator.DemoApp.Game/MainScreen.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ private void OnBeatmapChange(ValueChangedEvent<Beatmap> obj)

pos.MaxValue = obj.NewValue.HitObjects.Count - length - 1;

pattern = new DataExtractor().ExtractBeatmapData(obj.NewValue).ToArray();
pattern = new DataExtractor(1).ExtractBeatmapData(obj.NewValue).ToArray();
matcher = new TrieDataMatcher2(dataStruct, pattern);
filter = new OnScreenFilter();
sorter = new BestScoreOrderFilter(new SuperJudge(), pattern, matcher);
Expand Down
11 changes: 9 additions & 2 deletions Mapperator.DemoApp/Mapperator.DemoApp.Game/MapDataStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
Expand Down Expand Up @@ -36,15 +37,21 @@ public IEnumerable<IEnumerable<MapDataPoint>> Get(string name)
using Stream stream = store.GetStream(name);
if (stream is null) return null;
using StreamReader reader = new StreamReader(stream);
return DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray());
var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader).ToArray());
if (version != 1)
throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore");
return data;
}

public Task<IEnumerable<IEnumerable<MapDataPoint>>> GetAsync(string name, CancellationToken cancellationToken = new())
{
using Stream stream = store.GetStream(name);
if (stream is null) return null;
using StreamReader reader = new StreamReader(stream);
return Task.FromResult(DataSerializer.DeserializeBeatmapData(iterateLines(reader)));
var (version, data) = DataSerializer.DeserializeBeatmapData(iterateLines(reader));
if (version != 1)
throw new NotImplementedException($"Data version {version} is not currently supported in MapDataStore");
return Task.FromResult(data);
}

public Stream GetStream(string name)
Expand Down
20 changes: 15 additions & 5 deletions Mapperator/DataExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@
namespace Mapperator {
public class DataExtractor {
private readonly HitObjectEncoder encoder;
private readonly int dataVersion;

public DataExtractor() : this(new HitObjectEncoder()) { }
public DataExtractor(int dataVersion = DataSerializer.CurrentDataVersion) : this(new HitObjectEncoder(), dataVersion) { }

public DataExtractor(HitObjectEncoder encoder) {
public DataExtractor(HitObjectEncoder encoder, int dataVersion) {
this.encoder = encoder;
this.dataVersion = dataVersion;
}

public IEnumerable<MapDataPoint> ExtractBeatmapData(IBeatmap beatmap, bool mirror = false) {
Expand Down Expand Up @@ -43,9 +45,17 @@ public IEnumerable<MapDataPoint> ExtractBeatmapData(IEnumerable<HitObject> hitob
var segments = 0;
var controlPoints = path.ControlPoints;

for (var i = 0; i < controlPoints.Count; i++) {
if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) {
segments++;
if (dataVersion >= 2 && slider.SliderType == PathType.Linear) {
for (var i = 0; i < controlPoints.Count - 1; i++) {
if (controlPoints[i] != controlPoints[i + 1]) {
segments++;
}
}
} else {
for (var i = 0; i < controlPoints.Count; i++) {
if (i == controlPoints.Count - 1 || controlPoints[i] == controlPoints[i + 1] && i != controlPoints.Count - 2) {
segments++;
}
}
}

Expand Down
13 changes: 11 additions & 2 deletions Mapperator/DataSerializer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@

namespace Mapperator {
public static class DataSerializer {
public const int CurrentDataVersion = 2;
private const string BeatmapSeparator = "/-\\_/-\\_/-\\";
private const string DataHeader = "Mapperator file format v";

public static string CurrentHeader => $"{DataHeader}{CurrentDataVersion}";

public static IEnumerable<string> SerializeBeatmapData(IEnumerable<IEnumerable<MapDataPoint>> data) {
foreach (var beatmap in data) {
Expand All @@ -21,8 +25,13 @@ public static string SerializeBeatmapDataSample(MapDataPoint data) {
return data.ToString();
}

public static IEnumerable<IEnumerable<MapDataPoint>> DeserializeBeatmapData(IEnumerable<string> data) {
return data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample));
public static (int, IEnumerable<IEnumerable<MapDataPoint>>) DeserializeBeatmapData(IEnumerable<string> data) {
// ReSharper disable twice PossibleMultipleEnumeration
var firstLine = data.FirstOrDefault() ?? "";
if (firstLine.StartsWith(DataHeader)) {
return (int.Parse(firstLine.Split('v').Last()), data.Skip(1).Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample)));
}
return (1, data.Split(BeatmapSeparator, beatmapData => beatmapData.Select(DeserializeBeatmapDataSample)));
}

public static MapDataPoint DeserializeBeatmapDataSample(string data) {
Expand Down