Skip to content

Commit

Permalink
Update JSON schema, remove XML and Avro, update tests (#38)
Browse files Browse the repository at this point in the history
  • Loading branch information
@martinothamar
martinothamar committed Feb 22, 2024
1 parent aaacd3d commit ca9e9d2
Show file tree
Hide file tree
Showing 102 changed files with 2,094 additions and 47,161 deletions.
14 changes: 0 additions & 14 deletions csharp/Vista.SDK.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{093029D2-21C
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK", "src\Vista.SDK\Vista.SDK.csproj", "{36063914-B923-446B-ACCB-C9DFD87E48BA}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK.Apache.Avro", "src\Vista.SDK.Apache.Avro\Vista.SDK.Apache.Avro.csproj", "{20C7791E-19BA-4711-B1E8-580394A010CA}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK.SourceGenerator", "src\Vista.SDK.SourceGenerator\Vista.SDK.SourceGenerator.csproj", "{80707032-DEC8-4C25-9F0D-744818E706EB}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK.System.Text.Json", "src\Vista.SDK.System.Text.Json\Vista.SDK.System.Text.Json.csproj", "{64282827-0A92-4038-A53D-8F1458993E26}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK.System.Xml", "src\Vista.SDK.System.Xml\Vista.SDK.System.Xml.csproj", "{3BBE2FB8-7F53-4772-9936-045F718DF73C}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{A70BCD8E-E1DB-49FF-855D-AD8ED1179080}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vista.SDK.Tests", "test\Vista.SDK.Tests\Vista.SDK.Tests.csproj", "{04C3E604-AEC3-47F0-8EF9-9E180022CD41}"
Expand Down Expand Up @@ -51,10 +47,6 @@ Global
{36063914-B923-446B-ACCB-C9DFD87E48BA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{36063914-B923-446B-ACCB-C9DFD87E48BA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{36063914-B923-446B-ACCB-C9DFD87E48BA}.Release|Any CPU.Build.0 = Release|Any CPU
{20C7791E-19BA-4711-B1E8-580394A010CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{20C7791E-19BA-4711-B1E8-580394A010CA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{20C7791E-19BA-4711-B1E8-580394A010CA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{20C7791E-19BA-4711-B1E8-580394A010CA}.Release|Any CPU.Build.0 = Release|Any CPU
{80707032-DEC8-4C25-9F0D-744818E706EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{80707032-DEC8-4C25-9F0D-744818E706EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{80707032-DEC8-4C25-9F0D-744818E706EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand All @@ -63,10 +55,6 @@ Global
{64282827-0A92-4038-A53D-8F1458993E26}.Debug|Any CPU.Build.0 = Debug|Any CPU
{64282827-0A92-4038-A53D-8F1458993E26}.Release|Any CPU.ActiveCfg = Release|Any CPU
{64282827-0A92-4038-A53D-8F1458993E26}.Release|Any CPU.Build.0 = Release|Any CPU
{3BBE2FB8-7F53-4772-9936-045F718DF73C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3BBE2FB8-7F53-4772-9936-045F718DF73C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3BBE2FB8-7F53-4772-9936-045F718DF73C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3BBE2FB8-7F53-4772-9936-045F718DF73C}.Release|Any CPU.Build.0 = Release|Any CPU
{04C3E604-AEC3-47F0-8EF9-9E180022CD41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{04C3E604-AEC3-47F0-8EF9-9E180022CD41}.Debug|Any CPU.Build.0 = Debug|Any CPU
{04C3E604-AEC3-47F0-8EF9-9E180022CD41}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand All @@ -93,10 +81,8 @@ Global
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{36063914-B923-446B-ACCB-C9DFD87E48BA} = {093029D2-21CA-4470-B16A-0291ABA2109F}
{20C7791E-19BA-4711-B1E8-580394A010CA} = {093029D2-21CA-4470-B16A-0291ABA2109F}
{80707032-DEC8-4C25-9F0D-744818E706EB} = {093029D2-21CA-4470-B16A-0291ABA2109F}
{64282827-0A92-4038-A53D-8F1458993E26} = {093029D2-21CA-4470-B16A-0291ABA2109F}
{3BBE2FB8-7F53-4772-9936-045F718DF73C} = {093029D2-21CA-4470-B16A-0291ABA2109F}
{04C3E604-AEC3-47F0-8EF9-9E180022CD41} = {A70BCD8E-E1DB-49FF-855D-AD8ED1179080}
{CBACE153-372D-479A-9663-529846C16DB9} = {95D04DE8-37F5-4D86-9D0B-D0819DAA0E7A}
{BF1C3D9F-2A43-4F90-866C-623BD65176E2} = {093029D2-21CA-4470-B16A-0291ABA2109F}
Expand Down
97 changes: 49 additions & 48 deletions csharp/benchmark/Vista.SDK.Benchmarks/Transport/DataChannelListSerialization.cs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
using System.Globalization;
using System.IO.Compression;
using System.Text.Json;
using Avro.IO;
using Avro.Specific;
// using Avro.IO;
// using Avro.Specific;
using BenchmarkDotNet.Analysers;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Engines;
Expand All @@ -18,10 +18,10 @@
using BenchmarkDotNet.Validators;
using ICSharpCode.SharpZipLib.BZip2;
using Microsoft.Diagnostics.Tracing.Parsers.IIS_Trace;
using Vista.SDK.Transport.Avro.DataChannel;
// using Vista.SDK.Transport.Avro.DataChannel;
using Vista.SDK.Transport.Json;
using Vista.SDK.Transport.Json.DataChannel;
using DataChannelListAvroPackage = Vista.SDK.Transport.Avro.DataChannel.DataChannelListPackage;
// using DataChannelListAvroPackage = Vista.SDK.Transport.Avro.DataChannel.DataChannelListPackage;
using DataChannelListJsonPackage = Vista.SDK.Transport.Json.DataChannel.DataChannelListPackage;
using RunMode = BenchmarkDotNet.Diagnosers.RunMode;

Expand All @@ -35,9 +35,10 @@ public class DataChannelListSerialization
private MemoryStream _compressionStream;

private DataChannelListJsonPackage _jsonPackage;
private DataChannelListAvroPackage _avroPackage;
private BinaryEncoder _avroEncoder;
private SpecificWriter<DataChannelListAvroPackage> _avroWriter;

// private DataChannelListAvroPackage _avroPackage;
// private BinaryEncoder _avroEncoder;
// private SpecificWriter<DataChannelListAvroPackage> _avroWriter;

private static readonly Lazy<DataChannelListSerialization> _bench =
new(() =>
Expand Down Expand Up @@ -104,24 +105,24 @@ public void Setup()
var text = File.ReadAllText("schemas/json/DataChannelList.sample.compact.json");
_jsonPackage = Serializer.DeserializeDataChannelList(text);
var domainModel = _jsonPackage!.ToDomainModel();
_avroPackage = domainModel.ToAvroDto();
// _avroPackage = domainModel.ToAvroDto();

_memoryStream = new MemoryStream();
_compressionStream = new MemoryStream();
_avroWriter = new SpecificWriter<DataChannelListAvroPackage>(DataChannelListAvroPackage._SCHEMA);
_avroEncoder = new BinaryEncoder(_memoryStream);
// _avroWriter = new SpecificWriter<DataChannelListAvroPackage>(DataChannelListAvroPackage._SCHEMA);
// _avroEncoder = new BinaryEncoder(_memoryStream);

Json();
_payloadSizes[nameof(Json)] = _memoryStream.Length;

Avro();
_payloadSizes[nameof(Avro)] = _memoryStream.Length;
// Avro();
// _payloadSizes[nameof(Avro)] = _memoryStream.Length;

Json_Brotli();
_payloadSizes[nameof(Json_Brotli)] = _compressionStream.Length;

Avro_Brotli();
_payloadSizes[nameof(Avro_Brotli)] = _compressionStream.Length;
// Avro_Brotli();
// _payloadSizes[nameof(Avro_Brotli)] = _compressionStream.Length;

foreach (var compressionLevelArgs in GetCompressionLevels())
{
Expand All @@ -130,8 +131,8 @@ public void Setup()
Json_Bzip2(compressionLevel);
_payloadSizes[$"{nameof(Json_Bzip2)}-{compressionLevel}"] = _compressionStream.Length;

Avro_Bzip2(compressionLevel);
_payloadSizes[$"{nameof(Avro_Bzip2)}-{compressionLevel}"] = _compressionStream.Length;
// Avro_Bzip2(compressionLevel);
// _payloadSizes[$"{nameof(Avro_Bzip2)}-{compressionLevel}"] = _compressionStream.Length;
}
}

Expand Down Expand Up @@ -174,38 +175,38 @@ public void Json_Brotli()
_brotliStream.Flush();
}

[Benchmark(Description = "Avro")]
[BenchmarkCategory("Uncompressed")]
public void Avro()
{
_memoryStream.SetLength(0);
_avroWriter.Write(_avroPackage, _avroEncoder);
}

[Benchmark(Description = "Avro")]
[ArgumentsSource(nameof(GetCompressionLevels))]
[BenchmarkCategory("Bzip2")]
public void Avro_Bzip2(int CompressionLevel)
{
_memoryStream.SetLength(0);
_compressionStream.SetLength(0);
_avroWriter.Write(_avroPackage, _avroEncoder);
_memoryStream.Position = 0;
BZip2.Compress(_memoryStream, _compressionStream, false, CompressionLevel);
}

[Benchmark(Description = "Avro")]
[BenchmarkCategory("Brotli")]
public void Avro_Brotli()
{
_memoryStream.SetLength(0);
_compressionStream.SetLength(0);
_avroWriter.Write(_avroPackage, _avroEncoder);
_memoryStream.Position = 0;
_brotliStream = new BrotliStream(_compressionStream, CompressionLevel.SmallestSize, true);
_memoryStream.CopyTo(_brotliStream);
_brotliStream.Flush();
}
// [Benchmark(Description = "Avro")]
// [BenchmarkCategory("Uncompressed")]
// public void Avro()
// {
// _memoryStream.SetLength(0);
// _avroWriter.Write(_avroPackage, _avroEncoder);
// }

// [Benchmark(Description = "Avro")]
// [ArgumentsSource(nameof(GetCompressionLevels))]
// [BenchmarkCategory("Bzip2")]
// public void Avro_Bzip2(int CompressionLevel)
// {
// _memoryStream.SetLength(0);
// _compressionStream.SetLength(0);
// _avroWriter.Write(_avroPackage, _avroEncoder);
// _memoryStream.Position = 0;
// BZip2.Compress(_memoryStream, _compressionStream, false, CompressionLevel);
// }

// [Benchmark(Description = "Avro")]
// [BenchmarkCategory("Brotli")]
// public void Avro_Brotli()
// {
// _memoryStream.SetLength(0);
// _compressionStream.SetLength(0);
// _avroWriter.Write(_avroPackage, _avroEncoder);
// _memoryStream.Position = 0;
// _brotliStream = new BrotliStream(_compressionStream, CompressionLevel.SmallestSize, true);
// _memoryStream.CopyTo(_brotliStream);
// _brotliStream.Flush();
// }

public static IEnumerable<object[]> GetCompressionLevels()
{
Expand Down
2 changes: 0 additions & 2 deletions csharp/benchmark/Vista.SDK.Benchmarks/Vista.SDK.Benchmarks.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\Vista.SDK.Apache.Avro\Vista.SDK.Apache.Avro.csproj" />
<ProjectReference Include="..\..\src\Vista.SDK.System.Text.Json\Vista.SDK.System.Text.Json.csproj" />
<ProjectReference Include="..\..\src\Vista.SDK.System.Xml\Vista.SDK.System.Xml.csproj" />
<ProjectReference Include="..\..\src\Vista.SDK\Vista.SDK.csproj" />
</ItemGroup>

Expand Down
8 changes: 4 additions & 4 deletions csharp/samples/simple/TimeSeriesDataPackage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,16 @@ internal static TimeSeriesDataPackage GetPackage(IReadOnlyList<BallastEvent> sou

var data = new TimeSeriesData(
dataConfiguration: null, // May optionally refer to a specific DataChannelList,
eventData: new EventData(Array.Empty<EventDataSet>(), "0"),
eventData: new EventData([], 0),
tabularData: new[]
{
new TabularData(
dataChannelID: dataChannelId,
dataSet: dataAsTable
.Select(d => new TabularDataSet(quality: null, timeStamp: d.Timestamp, value: d.Data))
.Select(d => new DataSet_Tabular(quality: null, timeStamp: d.Timestamp, value: d.Data))
.ToArray(),
numberOfDataChannel: dataChannelId.Length.ToString(CultureInfo.InvariantCulture),
numberOfDataSet: dataAsTable.Length.ToString(CultureInfo.InvariantCulture) // One sample per datachannel/timestamp for this scenario
numberOfDataChannel: dataChannelId.Length,
numberOfDataSet: dataAsTable.Length // One sample per datachannel/timestamp for this scenario
)
}
);
Expand Down
10 changes: 0 additions & 10 deletions csharp/src/Vista.SDK.Apache.Avro/Common.cs
View file

This file was deleted.

83 changes: 0 additions & 83 deletions csharp/src/Vista.SDK.Apache.Avro/DataChannelList/ConfigurationReference.cs
View file

This file was deleted.

0 comments on commit ca9e9d2

Please sign in to comment.