Fix offline storage during network errors (#38832)

* Fix offline storage during network errors * fix net462 * changelog
Azure · Sep 19, 2023 · 9864850 · 9864850
1 parent 108a37e
commit 9864850
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 14 deletions.
diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md
@@ -8,6 +8,10 @@
 
 ### Bugs Fixed
 
+* Fixed an issue during network failures which prevented the exporter to store
+the telemetry offline for retrying at a later time.
+([#38832](https://github.com/Azure/azure-sdk-for-net/pull/38832))
+
 ### Other Changes
 
 * Update OpenTelemetry dependencies

diff --git a/.../Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs b/.../Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs
@@ -42,7 +42,7 @@ internal async Task<HttpMessage> InternalTrackAsync(IEnumerable<TelemetryItem> b
             catch (Exception ex)
             {
                 AzureMonitorExporterEventSource.Log.FailedToTransmit(ex);
-                if (ex.InnerException?.Source != "System.Net.Http")
+                if (ex.InnerException?.Source != "System.Net.Http" && ex.InnerException?.Source != "System")
                 {
                     message?.Dispose();
                     throw;
@@ -70,7 +70,7 @@ internal async Task<HttpMessage> InternalTrackAsync(ReadOnlyMemory<byte> body, C
             catch (Exception ex)
             {
                 AzureMonitorExporterEventSource.Log.FailedToTransmit(ex);
-                if (ex.InnerException?.Source != "System.Net.Http")
+                if (ex.InnerException?.Source != "System.Net.Http" && ex.InnerException?.Source != "System")
                 {
                     message?.Dispose();
                     throw;

diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs
@@ -178,7 +178,7 @@ public async ValueTask<ExportResult> TrackAsync(IEnumerable<TelemetryItem> telem
 
                     if (result == ExportResult.Failure && _fileBlobProvider != null)
                     {
-                        _transmissionStateManager.EnableBackOff(httpMessage.Response);
+                        _transmissionStateManager.EnableBackOff(httpMessage.HasResponse ? httpMessage.Response : null);
                         result = HttpPipelineHelper.HandleFailures(httpMessage, _fileBlobProvider, _connectionVars, origin, _isAadEnabled);
                     }
                     else

diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs
@@ -96,7 +96,7 @@ internal void ResetTransmission(object? source, System.Timers.ElapsedEventArgs e
             CloseTransmission();
         }
 
-        internal void EnableBackOff(Response response)
+        internal void EnableBackOff(Response? response)
         {
             if (Interlocked.Exchange(ref _syncBackOffIntervalCalculation, 1) == 0)
             {

diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs
@@ -69,7 +69,7 @@ internal void TransmitFromStorage(object? sender, ElapsedEventArgs? e)
                         }
                         else
                         {
-                            _transmissionStateManager.EnableBackOff(httpMessage.Response);
+                            _transmissionStateManager.EnableBackOff(httpMessage.HasResponse ? httpMessage.Response : null);
                             HttpPipelineHelper.HandleFailures(httpMessage, blob, _blobProvider, _connectionVars, _isAadEnabled);
                             break;
                         }

diff --git a/...elemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs b/...elemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs
@@ -98,6 +98,23 @@ public void FailureResponseCode429()
             Assert.Single(transmitter._fileBlobProvider.GetBlobs());
         }
 
+        [Fact]
+        public void NetworkFailure()
+        {
+            using var activity = CreateActivity("TestActivity");
+            var telemetryItem = CreateTelemetryItem(activity);
+            List<TelemetryItem> telemetryItems = new List<TelemetryItem>();
+            telemetryItems.Add(telemetryItem);
+
+            // Transmit
+            using var transmitter = GetTransmitter(null);
+            transmitter.TrackAsync(telemetryItems, TelemetryItemOrigin.UnitTest, false, CancellationToken.None).EnsureCompleted();
+
+            //Assert
+            Assert.NotNull(transmitter._fileBlobProvider);
+            Assert.Single(transmitter._fileBlobProvider.GetBlobs());
+        }
+
         [Fact]
         public void FailureResponseCode206()
         {
@@ -260,17 +277,32 @@ public void TelemetryIsStoredOfflineWhenTransmissionStateIsSetToOpen()
             transmitter.Dispose();
         }
 
-        private static AzureMonitorTransmitter GetTransmitter(params MockResponse[] mockResponse)
+        private static AzureMonitorTransmitter GetTransmitter(params MockResponse[]? mockResponse)
         {
-            MockTransport mockTransport = new MockTransport(mockResponse);
-            AzureMonitorExporterOptions options = new AzureMonitorExporterOptions
+            AzureMonitorTransmitter transmitter;
+            AzureMonitorExporterOptions options;
+            if (mockResponse == null)
             {
-                ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}",
-                StorageDirectory = "C:\\test",
-                Transport = mockTransport,
-                EnableStatsbeat = false, // disabled in tests.
-            };
-            AzureMonitorTransmitter transmitter = new AzureMonitorTransmitter(options, new MockPlatform());
+                options = new AzureMonitorExporterOptions
+                {
+                    ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}",
+                    StorageDirectory = "C:\\test",
+                    EnableStatsbeat = false, // disabled in tests.
+                };
+            }
+            else
+            {
+                MockTransport mockTransport = new MockTransport(mockResponse);
+                options = new AzureMonitorExporterOptions
+                {
+                    ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}",
+                    StorageDirectory = "C:\\test",
+                    Transport = mockTransport,
+                    EnableStatsbeat = false, // disabled in tests.
+                };
+            }
+
+            transmitter = new AzureMonitorTransmitter(options, new MockPlatform());
 
             // Overwrite storage with mock
             transmitter._fileBlobProvider = new MockFileProvider();