diff --git a/src/ApiService/ApiService/FeatureFlags.cs b/src/ApiService/ApiService/FeatureFlags.cs index e74396e882..5a88d7232d 100644 --- a/src/ApiService/ApiService/FeatureFlags.cs +++ b/src/ApiService/ApiService/FeatureFlags.cs @@ -9,4 +9,5 @@ public static class FeatureFlagConstants { public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention"; public const string EnableWorkItemCreation = "EnableWorkItemCreation"; public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies"; + public const string EnableSlimEventSerialization = "EnableSlimEventSerialization"; } diff --git a/src/ApiService/ApiService/onefuzzlib/Events.cs b/src/ApiService/ApiService/onefuzzlib/Events.cs index 0b806c5806..b6343ac95b 100644 --- a/src/ApiService/ApiService/onefuzzlib/Events.cs +++ b/src/ApiService/ApiService/onefuzzlib/Events.cs @@ -35,7 +35,9 @@ public class Events : IEvents { private readonly IContainers _containers; private readonly ICreds _creds; private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; private readonly JsonSerializerOptions _deserializingFromBlobOptions; + private readonly IOnefuzzContext _context; public Events(ILogger log, IOnefuzzContext context) { _queue = context.Queue; @@ -47,9 +49,12 @@ public class Events : IEvents { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; _options.Converters.Add(new RemoveUserInfo()); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); _deserializingFromBlobOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + _context = context; } public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) { @@ -58,7 +63,13 @@ public class Events : IEvents { ("event_id", message.EventId.ToString()) }; var ev = new SignalREvent("events", new List() { message }); - var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: _options); + + var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch { + true => _optionsSlim, + false => _options + }; + + var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: opts); if (!queueResult) { _log.AddTags(tags); @@ -155,16 +166,4 @@ public class Events : IEvents { ); } } - - - public class RemoveUserInfo : JsonConverter { - public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - throw new NotSupportedException("reading UserInfo is not supported"); - } - - public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { - writer.WriteStartObject(); - writer.WriteEndObject(); - } - } } diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index cb43a536e5..cc5367b211 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -20,10 +20,15 @@ public interface IWebhookOperations : IOrm { public class WebhookOperations : Orm, IWebhookOperations { private readonly IHttpClientFactory _httpFactory; + private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; public WebhookOperations(IHttpClientFactory httpFactory, ILogger log, IOnefuzzContext context) : base(log, context) { _httpFactory = httpFactory; + _options = EntityConverter.GetJsonSerializerOptions(); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); } public async Async.Task SendEvent(DownloadableEventMessage eventMessage) { @@ -139,11 +144,15 @@ public WebhookOperations(IHttpClientFactory httpFactory, ILogger _optionsSlim, + false => _options + }; if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) { var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookMessage, DataVersion: "2.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) }; - data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(eventGridMessage, options: opts); } else { - data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(webhookMessage, options: opts); } string? digest = null; diff --git a/src/ApiService/ApiService/onefuzzlib/events/Converters.cs b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs new file mode 100644 index 0000000000..47050a496d --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs @@ -0,0 +1,76 @@ +using System.Collections; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.OneFuzz.Service { + public class RemoveUserInfo : JsonConverter { + public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("reading UserInfo is not supported"); + } + + public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { + writer.WriteStartObject(); + writer.WriteEndObject(); + } + } + + /// + /// THIS IS A WRITE ONLY JSON CONVERTER + ///
+ /// It should only be used when serializing events to be sent outside of the service + /// + public class EventExportConverter : JsonConverter + where T : DownloadableEventMessage { + public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("This converter should only be used when serializing event messages to sent outside of the service"); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) { + BoundedSerializer.WriteInternal(writer, value, options); + } + } + + public class BoundedSerializer { + private static HashSet boundedTypes = new HashSet{ + typeof(Guid), + typeof(DateTime), + typeof(int), + typeof(bool), + typeof(float), + typeof(double), + typeof(long), + typeof(char), + typeof(Uri) + }; + + public static void WriteInternal(Utf8JsonWriter writer, object type, JsonSerializerOptions options) { + writer.WriteStartObject(); + var properties = type.GetType().GetProperties(); + foreach (var property in properties) { + if (property.GetValue(type, null) == null + || typeof(IEnumerable).IsAssignableFrom(property.PropertyType) + || type.GetType() == property.PropertyType) { + continue; + } + if (HasBoundedSerialization(property)) { + var serialized = JsonSerializer.Serialize(property.GetValue(type, null), property.PropertyType, options); + if (!string.IsNullOrEmpty(serialized)) { + writer.WritePropertyName(property.Name); + writer.WriteRawValue(serialized); + } + } else if (property.PropertyType.IsClass) { + writer.WritePropertyName(property.Name); + WriteInternal(writer, property.GetValue(type, null)!, options); + } + } + writer.WriteEndObject(); + } + + public static bool HasBoundedSerialization(PropertyInfo propertyInfo) { + return propertyInfo.PropertyType.IsEnum || + boundedTypes.Contains(propertyInfo.PropertyType) || + typeof(IValidatedString).IsAssignableFrom(propertyInfo.PropertyType); + } + } +} diff --git a/src/ApiService/Tests/EventExportConverterTests.cs b/src/ApiService/Tests/EventExportConverterTests.cs new file mode 100644 index 0000000000..4d129c18c0 --- /dev/null +++ b/src/ApiService/Tests/EventExportConverterTests.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using FluentAssertions; +using FsCheck; +using FsCheck.Xunit; +using Microsoft.OneFuzz.Service; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using Xunit; + + +namespace Tests; + +public class EventExportConverterTests { + enum Color { + Red, + Blue + } + + [Fact] + public void BaseTypesAreBounded() { + var a = new { + guid = Guid.NewGuid(), + date = new DateTime(), + en = Color.Red, + b = 1, + boo = false, + flo = float.Pi, + doub = double.Tau, + lon = long.MinValue, + cha = 'a' + }; + + a.GetType().GetProperties().All(p => BoundedSerializer.HasBoundedSerialization(p)).Should().BeTrue(); + } + + [Fact] + public void StringIsNotBounded() { + var a = new { + bad = "this is not bounded" + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("bad")!).Should().BeFalse(); + } + + [Fact] + public void ValidatedStringIsBounded() { + var a = new { + scalesetid = ScalesetId.Parse("abc-123") + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("scalesetid")!).Should().BeTrue(); + } + + [Fact] + public void ComplexObjectsAreSerialized() { + var randomGuid = Guid.NewGuid(); + var a = new DownloadableEventMessage( + randomGuid, + EventType.CrashReported, + new EventCrashReported( + new Report( + "https://example.com", + null, + "target.exe", + "crash", + string.Empty, + new List { "this", "is", "a", "stacktrace" }, + string.Empty, + string.Empty, + null, + Guid.NewGuid(), + Guid.NewGuid(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ), + Container.Parse("this-is-a-container"), + "crash-abc123", + null + ), + Guid.NewGuid(), + "onefuzz", + DateTime.Now, + new Uri("https://example.com"), + null + ); + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().NotBeNullOrEmpty(); + serialized.Should().NotContain("stacktrace"); // List is not serialized + serialized.Should().NotContain("crash-abc123"); // string is not serialized + serialized.Should().Contain("this-is-a-container"); // ValidatedString is serialized + serialized.Should().Contain("crash_reported"); // Enum is serialized + serialized.Should().Contain(DateTime.Now.Year.ToString()); // DateTime is serialized + serialized.Should().Contain(randomGuid.ToString()); // Guid id serialized + } + + [Fact] + public void TestWebhookMessage() { + var a = new WebhookMessageEventGrid( + "2.0.0", + "eventsubject", + EventType.JobCreated, + DateTime.Now, + Guid.NewGuid(), + new WebhookMessage( + Guid.NewGuid(), + EventType.JobCreated, + new EventJobCreated( + Guid.NewGuid(), + new JobConfig("some project", "some name", "some build", 1, "some logs"), + null, + "8.0"), + Guid.NewGuid(), + "onefuzz", + Guid.NewGuid(), + DateTime.Now, + new Uri("https://example.com") + ) + ); + + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().Contain("eventsubject"); + serialized.Should().NotContain("some project"); + } + + public class EventExportConverterSerializationTests { + private readonly JsonSerializerOptions _opts = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + public EventExportConverterSerializationTests() { + _ = Arb.Register(); + _opts.Converters.Add(new EventExportConverter()); + } + + void Test(T v) { + // TODO: Try cloning/creating a new serializer options from the existing one? + var serialized = JsonSerializer.Serialize(v, _opts); + var _ = JsonSerializer.Deserialize(serialized); + } + + [Property] + public void EventNodeHeartbeat(EventNodeHeartbeat e) => Test(e); + + + [Property] + public void EventTaskHeartbeat(EventTaskHeartbeat e) => Test(e); + + [Property] + public void EventTaskStopped(EventTaskStopped e) => Test(e); + + [Property] + public void EventInstanceConfigUpdated(EventInstanceConfigUpdated e) => Test(e); + + [Property] + public void EventProxyCreated(EventProxyCreated e) => Test(e); + + [Property] + public void EventProxyDeleted(EventProxyDeleted e) => Test(e); + + [Property] + public void EventProxyFailed(EventProxyFailed e) => Test(e); + + [Property] + public void EventProxyStateUpdated(EventProxyStateUpdated e) => Test(e); + + + [Property] + public void EventCrashReported(EventCrashReported e) => Test(e); + + + [Property] + public void EventRegressionReported(EventRegressionReported e) => Test(e); + + + [Property] + public void EventFileAdded(EventFileAdded e) => Test(e); + + [Property] + public void EventTaskFailed(EventTaskFailed e) => Test(e); + + [Property] + public void EventTaskStateUpdated(EventTaskStateUpdated e) => Test(e); + + [Property] + public void EventScalesetFailed(EventScalesetFailed e) => Test(e); + + [Property] + public void EventScalesetResizeScheduled(EventScalesetResizeScheduled e) => Test(e); + + [Property] + public void EventScalesetStateUpdated(EventScalesetStateUpdated e) => Test(e); + + [Property] + public void EventNodeDeleted(EventNodeDeleted e) => Test(e); + + [Property] + public void EventNodeCreated(EventNodeCreated e) => Test(e); + + [Property] + public void EventMessage(DownloadableEventMessage e) => Test(e); + } +} diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 956d0c30c5..6828d7f2fd 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -530,12 +530,13 @@ public class OrmJsonSerialization { //Sample function on how repro a failing test run, using Replay //functionality of FsCheck. Feel free to - /* + + [Fact] void Replay() { - var seed = FsCheck.Random.StdGen.NewStdGen(811038773, 297085737); - var p = Prop.ForAll((NotificationTemplate x) => NotificationTemplate(x)); + var seed = FsCheck.Random.StdGen.NewStdGen(1687595065, 297240661); + var p = Prop.ForAll((DownloadableEventMessage x) => EventMessage(x)); p.Check(new Configuration { Replay = seed }); } - */ + } } diff --git a/src/deployment/bicep-templates/feature-flags.bicep b/src/deployment/bicep-templates/feature-flags.bicep index 46fccb0856..cd39edb52f 100644 --- a/src/deployment/bicep-templates/feature-flags.bicep +++ b/src/deployment/bicep-templates/feature-flags.bicep @@ -102,4 +102,17 @@ resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurat } } +resource enableSlimEventSerialization 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = { + parent: featureFlags + name: '.appconfig.featureflag~2FEnableSlimEventSerialization' + properties: { + value: string({ + id: 'EnableSlimEventSerialization' + description: 'Enable serializing events as smaller payloads' + enabled: false + }) + contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8' + } +} + output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io'