Updating previews/RBAC branch - Master to previews/RBAC (#1825)

* fix: samples readme links were out-of-date (#1784) * fix: samples arguments and remove 1 more dead link (#1785) * fix(readme): Update the location of ConsoleEventListener in our readme * doc(service-client) - Updating readme (#1799) * fix(doc): Update amqp transport exception doc to have detailed description for quota exceeded error mapping * feature(device-client): Make the DeviceClient and ModuleClient extensible (#1802) * doc(service-client): Added extra comments to clarify true and false in dispose (#1805) * feature,fix (device-client) Handle Twin failures using Amqp (#1796) * fix(iot-service): Update xml comments for ServiceClient * fix(iot-device): Update MqttTransportHandler to not use SemaphoreSlim.WaitAsync(TimeSpan, CancellationToken) * fix(iot-device): Update dotnetty task calls to use ConfigureAwait(true) * fix(iot-device): Fix MqttTransportHandler to not await on user supplied C2D callback * IoTHub Exception for Get and Patch Twin failures (#1815) * fix(edge): UnixDomainSocketEndPoint is available in .NET 2.1 and greater (#1816) * UnixDomainSocketEndPoint has been standard since 2.1. Allowing later versions to use the correct class The edge client HSM provider uses UnixDomainSockets (UDS) for communication. Before .NET 2.1 to implement a Unix Socket you had to create your own class to do so. Since 2.1 there has been a native UnixDomainSocketEndPoint class in the runtime. In 2.1 and 3.1 there is no issue. However in 5.0 there are some changes to the way the Socket class handles the native UnixDomainSocketEndPoint class. I didn't dig down extremely deep, but I suspect it's due to the way the endpoint handles the SocketAddress and the string manipulation there seeing as how there is a specific implementation for Windows and for Unix. * feat(e2e) - Enabling soft delete when creating keyvaults (#1820) * fix(e2e) - Change event logging to opt in to specific events only. (#1824) * (service-client): Design for IoT hub AAD authentication * (service-client: Refactor and add implementation for token credential input) (#1781) * (service-client): Refactor and add sas credential (#1786) * (service-client): Add constructors in service client to accept aad and sas tokens. (#1787) * (service-client): Add constructors in registry manager to accept aad and sas tokens. (#1788) * (service-client): Add constructors in job client to accept aad and sas tokens. (#1789) * (service-client): Add constructors to accept aad and sas tokens for digital twins client. (#1790) * fix(service-client): Support for AzureSasCredential for a better user experience (#1797) * doc(service-client): Update readme about the differnt client and operations (#1798) * tests(service-client): E2E tests for aad auth on all our clients (#1800) * test(service-client): Adding e2e tests for sas credential auth for IoT hub. (#1806) * feature(service-client): Adding chaching for aad tokens. (#1807) * fix(service-client)- Add IoT hub token scope. (#1812) Co-authored-by: David R. Williamson <drwill@microsoft.com> Co-authored-by: Abhipsa Misra <abhipsa.misra@microsoft.com> Co-authored-by: bikamani <41314966+bikamani@users.noreply.github.com> Co-authored-by: jamdavi <73593426+jamdavi@users.noreply.github.com>
Azure · Mar 11, 2021 · ff88ada · ff88ada
1 parent 818f332
commit ff88ada
Show file tree

Hide file tree

Showing 24 changed files with 365 additions and 364 deletions.
diff --git a/e2e/test/E2EMsTestBase.cs b/e2e/test/E2EMsTestBase.cs
@@ -23,7 +23,6 @@ namespace Microsoft.Azure.Devices.E2ETests
     /// </summary>
     public class E2EMsTestBase : IDisposable
     {
-        private static readonly string[] s_eventProviders = new string[] { "DotNetty-Default", "Microsoft-Azure-", };
         private ConsoleEventListener _listener;
 
         // Test specific logger instance
@@ -41,7 +40,7 @@ public void TestInitialize()
             // Note: Events take long and increase run time of the test suite, so only using trace.
             Logger.Trace($"Starting test - {TestContext.TestName}", SeverityLevel.Information);
 
-            _listener = new ConsoleEventListener(s_eventProviders);
+            _listener = new ConsoleEventListener();
         }
 
         [TestCleanup]

diff --git a/e2e/test/Helpers/ConsoleEventListener.cs b/e2e/test/Helpers/ConsoleEventListener.cs
@@ -1,89 +1,55 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
 
 namespace System.Diagnostics.Tracing
 {
     public sealed class ConsoleEventListener : EventListener
     {
-        private readonly string[] _eventFilters;
-        private readonly object _lock = new object();
-
-        public ConsoleEventListener(string filter)
-        {
-            _eventFilters = new string[1];
-            _eventFilters[0] = filter ?? throw new ArgumentNullException(nameof(filter));
-
-            InitializeEventSources();
-        }
-
-        public ConsoleEventListener(string[] filters)
-        {
-            _eventFilters = filters ?? throw new ArgumentNullException(nameof(filters));
-            if (_eventFilters.Length == 0)
-            {
-                throw new ArgumentException("Filters cannot be empty", nameof(filters));
-            }
-
-            foreach (string filter in _eventFilters)
-            {
-                if (string.IsNullOrWhiteSpace(filter))
-                {
-                    throw new ArgumentNullException(nameof(filters));
-                }
-            }
+        // Configure this value to filter all the necessary events when OnEventSourceCreated is called.
+        // OnEventSourceCreated is triggered as soon as the EventListener is registered and an event source is created.
+        // So trying to configure this value in the ConsoleEventListener constructor does not work.
+        // The OnEventSourceCreated can be triggered sooner than the filter is initialized in the ConsoleEventListener constructor.
+        private static string[] _eventFilters = new string[] { "DotNetty-Default", "Microsoft-Azure-Devices" };
 
-            InitializeEventSources();
-        }
-
-        private void InitializeEventSources()
-        {
-            foreach (EventSource source in EventSource.GetSources())
-            {
-                EnableEvents(source, EventLevel.LogAlways);
-            }
-        }
+        private readonly object _lock = new object();
 
         protected override void OnEventSourceCreated(EventSource eventSource)
         {
-            base.OnEventSourceCreated(eventSource);
-            EnableEvents(
-                eventSource,
-                EventLevel.LogAlways
+            if (_eventFilters.Any(filter => eventSource.Name.StartsWith(filter, StringComparison.OrdinalIgnoreCase)))
+            {
+                base.OnEventSourceCreated(eventSource);
+                EnableEvents(
+                    eventSource,
+                    EventLevel.LogAlways
 #if !NET451
                 , EventKeywords.All
 #endif
                 );
+            }
         }
 
         protected override void OnEventWritten(EventWrittenEventArgs eventData)
         {
-            if (_eventFilters == null)
-            {
-                return;
-            }
-
             lock (_lock)
             {
-                if (_eventFilters.Any(ef => eventData.EventSource.Name.StartsWith(ef, StringComparison.Ordinal)))
-                {
-                    string eventIdent;
+                string eventIdent;
 #if NET451
                     // net451 doesn't have EventName, so we'll settle for EventId
                     eventIdent = eventData.EventId.ToString(CultureInfo.InvariantCulture);
 #else
-                    eventIdent = eventData.EventName;
+                eventIdent = eventData.EventName;
 #endif
-                    string text = $"{DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture)} [{eventData.EventSource.Name}-{eventIdent}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : "")}";
+                string text = $"{DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture)} [{eventData.EventSource.Name}-{eventIdent}]{(eventData.Payload != null ? $" ({string.Join(", ", eventData.Payload)})." : "")}";
 
-                    ConsoleColor origForeground = Console.ForegroundColor;
-                    Console.ForegroundColor = ConsoleColor.DarkYellow;
-                    Console.WriteLine(text);
-                    Debug.WriteLine(text);
-                    Console.ForegroundColor = origForeground;
-                }
+                ConsoleColor origForeground = Console.ForegroundColor;
+                Console.ForegroundColor = ConsoleColor.DarkYellow;
+                Console.WriteLine(text);
+                Debug.WriteLine(text);
+                Console.ForegroundColor = origForeground;
             }
         }
     }

diff --git a/e2e/test/iothub/twin/TwinE2ETests.cs b/e2e/test/iothub/twin/TwinE2ETests.cs
@@ -6,11 +6,11 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.Devices.Client;
+using Microsoft.Azure.Devices.Client.Exceptions;
 using Microsoft.Azure.Devices.E2ETests.Helpers;
 using Microsoft.Azure.Devices.Shared;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Azure.Devices.E2ETests.Twins
 {
@@ -709,12 +709,12 @@ await deviceClient
                         })
                     .ConfigureAwait(false);
             }
-            catch (Exception)
+            catch (IotHubException)
             {
                 exceptionThrown = true;
             }
 
-            Assert.IsTrue(exceptionThrown, "Exception was expected, but not thrown.");
+            Assert.IsTrue(exceptionThrown, "IotHubException was expected for updating reported property with an invalid property name, but was not thrown.");
 
             Twin serviceTwin = await registryManager.GetTwinAsync(testDevice.Id).ConfigureAwait(false);
             Assert.IsFalse(serviceTwin.Properties.Reported.Contains(propName1));

diff --git a/e2e/test/prerequisites/E2ETestsSetup/test-resources.bicep b/e2e/test/prerequisites/E2ETestsSetup/test-resources.bicep
@@ -163,7 +163,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2018-02-14' = {
       name: 'standard'
       family: 'A'
     }
-    enableSoftDelete: false
+    enableSoftDelete: true
     networkAcls: {
       defaultAction: 'Allow'
       bypass: 'AzureServices'

diff --git a/e2e/test/prerequisites/E2ETestsSetup/test-resources.json b/e2e/test/prerequisites/E2ETestsSetup/test-resources.json
@@ -176,7 +176,7 @@
           "name": "standard",
           "family": "A"
         },
-        "enableSoftDelete": false,
+        "enableSoftDelete": true,
         "networkAcls": {
           "defaultAction": "Allow",
           "bypass": "AzureServices",
@@ -342,7 +342,7 @@
       "properties": {}
     },
     {
-      "type": "Microsoft.Security/IoTSecuritySolutions",
+      "type": "Microsoft.Security/iotSecuritySolutions",
       "apiVersion": "2019-08-01",
       "name": "[parameters('SecuritySolutionName')]",
       "location": "[resourceGroup().location]",
@@ -484,7 +484,7 @@
     },
     "workspaceId": {
       "type": "string",
-      "value": "[format('{0}', reference(resourceId('Microsoft.OperationalInsights/workspaces', parameters('OperationalInsightsName')), '2017-03-15-preview').customerId)]"
+      "value": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', parameters('OperationalInsightsName')), '2017-03-15-preview').customerId]"
     },
     "customAllocationPolicyWebhook": {
       "type": "string",

diff --git a/iothub/device/devdoc/amqpTransportExceptions.md b/iothub/device/devdoc/amqpTransportExceptions.md
@@ -41,7 +41,7 @@ Below is the behavior of the SDK on receiving an exception over Amqp transport p
 | amqp:not-implemented | NotSupportedException | InnerException: AmqpException.Error.Condition = AmqpSymbol.NotImplemented | Inspect the exception details, collect logs and contact service |
 | amqp:precondition-failed | IotHubException | InnerException: AmqpException.Error.Condition = AmqpSymbol.PreconditionFailed | Inspect the exception details, collect logs and contact service |
 | amqp:resource-deleted | IotHubException | InnerException: AmqpException.Error.Condition = AmqpSymbol.ResourceDeleted | Inspect the exception details, collect logs and contact service |
-| amqp:resource-limit-exceeded | IotHubException | InnerException: AmqpException.Error.Condition = AmqpSymbol.ResourceLimitExceeded | Inspect the exception details, collect logs and contact service |
+| amqp:resource-limit-exceeded | DeviceMaximumQueueDepthExceededException | The correct exception type for this error code is `QuotaExceededException` but it was incorrectly mapped to `DeviceMaximumQueueDepthExceededException`. In order to avoid a breaking change, we now return the correct exception details as an inner exception within the `DeviceMaximumQueueDepthExceededException` thrown. | Upgrade or increase the number of units on your IoT Hub or wait until the next UTC day for the daily quota to refresh and then retry the operation. |
 | amqp:unauthorized-access | UnauthorizedException | InnerException: AmqpException.Error.Condition = AmqpSymbol.UnauthorizedAccess | Inspect your credentials |
 | com.microsoft:message-lock-lost | DeviceMessageLockLostException | The device client attempted to complete/reject/abandon a received cloud-to-device message, but the lock token was expired (took > 1min after receiving the message to complete/reject/abandon it) | Call `ReceiveAsync()` again to retrieve an updated lock token, and then complete/reject/abandon the message. De-duplication logic wil need to be implemented at the application level |
 | amqp:transaction :unknown-id | IotHubException | InnerException: AmqpException.Error.Condition = AmqpSymbol.TransactionUnknownId | Inspect the exception details, collect logs and contact service |

diff --git a/iothub/device/samples/readme.md b/iothub/device/samples/readme.md
@@ -2,70 +2,81 @@
 This folder contains simple samples showing how to use the various features of Microsoft Azure IoT Hub service, from a device running C# code.
 
 ### [Device samples][device-samples]
-* [Message Sample][d-message-sample]
-* [Method Sample][d-method-sample]
-* [Twin Sample][d-twin-sample]
-* [File Upload Sample][d-file-upload-sample]
-* [Keys Rollover Sample][d-keys-rollover-sample]
-* [Import Export Devices Sample][d-import-export-devices-sample]
-* [Plug And Play Device Sample][d-pnp-sample]
-* [Xamarin Sample][d-xamarin-sample]
-
-### [Module samples][module-samples]
-* [Message Sample][m-message-sample]
-* [Twin Sample][m-twin-sample]
+
+- [Reconnection sample][d-message-sample]
+    - This sample illustrates how to write a device application to handle connection issues, connection-related exceptions, and how to manage the lifetime of the `DeviceClient`
+    - Includes sending messages and symmetric key failover
+- [Method sample][d-method-sample]
+- [Receive message sample][d-receive-message-sample]
+- [Twin sample][d-twin-sample]
+- [File upload sample][d-file-upload-sample]
+- [Import/export devices sample][d-import-export-devices-sample]
+- [Connect with X509 certificate sample][d-x509-cert-sample]
+- [Plug and Play device samples][d-pnp-sample]
+- [Xamarin sample][d-xamarin-sample]
+
+### Module sample
+
+- [Message sample][m-message-sample]
+    - This sample illustrates how to write an IoT Hub module to handle connection issues, connection-related exceptions, and how to manage the lifetime of the `ModuleClient`
+    - Includes sending messages and symmetric key failover
 
 ### Prerequisites
+
 In order to run the device samples on Linux or Windows, you will first need the following prerequisites:
-* [Setup your IoT hub][lnk-setup-iot-hub]
-* [Provision your device and get its credentials][lnk-manage-iot-device]
+
+- [Setup your IoT hub][lnk-setup-iot-hub]
+- [Provision your device and get its credentials][lnk-manage-iot-device]
 
 ### Setup environment
 
 The following prerequisite is the minimum requirement to build and run the samples. 
 
-Visual Studio is **not required** to run the samples.
+- Install the latest .NET Core from <https://dot.net>
 
-- Install the latest .NET Core from https://dot.net
+> Visual Studio is **not required** to run the samples.
 
 ### Get and run the samples
-You need to clone the repository or download the sample (the one you want to try) project's folder on your device.
 
-#### Build and run the samples:
-1. Preparing the sample application:
-   1. Set the following environment variables on the terminal from which you want to run the application.
+You need to clone the repository or download the sample (the one you want to try) project's folder on your device.
 
-      * IOTHUB_DEVICE_CONNECTION_STRING
+#### Build and run the samples
 
-2. Building the sample application:
+1. Building the sample application
 
     To build the sample application using dotnet, from terminal navigate to the sample folder (where the .csproj file lives). Then execute the following command and check for build errors:
 
-    ```
+    ```console
     dotnet build
     ```
 
-3. Running the sample application:
+1. Preparing the sample application:
+   1. Many of these samples take parameters. To see the parameters required, type:
+
+      ```console
+      dotnet run --help
+      ```
 
-	To run the sample application using dotnet, execute the following command.
+1. Running the sample application:
 
-    ```
+    To run the sample application using dotnet, execute the following command with any required parameters discovered in the previous step.
+
+    ```console
     dotnet run
     ```
 
 [device-samples]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device
-[d-message-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/MessageSample
+[d-message-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/DeviceReconnectionSample
+[d-receive-message-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/MessageReceiveSample
 [d-method-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/MethodSample
 [d-twin-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/TwinSample
 [d-file-upload-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/FileUploadSample
-[d-keys-rollover-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/KeysRolloverSample
+[d-x509-cert-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/X509DeviceCertWithChainSample
 [d-import-export-devices-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/ImportExportDevicesSample
 [d-pnp-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/PnpDeviceSamples
 [d-xamarin-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/device/XamarinSample
 
-[module-samples]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/module
-[m-message-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/module/MessageSample
-[m-twin-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/module/TwinSample
+[m-message-sample]: https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Samples/module/ModuleSample
 
 [lnk-setup-iot-hub]: https://aka.ms/howtocreateazureiothub
 [lnk-manage-iot-device]: https://github.com/Azure/azure-iot-device-ecosystem/blob/master/setup_iothub.md#create-new-device-in-the-iot-hub-device-identity-registry
diff --git a/iothub/device/src/DeviceClient.cs b/iothub/device/src/DeviceClient.cs
@@ -15,7 +15,7 @@ namespace Microsoft.Azure.Devices.Client
     /// Contains methods that a device can use to send messages to and receive from the service.
     /// </summary>
     /// <threadsafety static="true" instance="true" />
-    public sealed class DeviceClient : IDisposable
+    public class DeviceClient : IDisposable
     {
         /// <summary>
         /// Default operation timeout.
@@ -608,7 +608,26 @@ public void SetRetryPolicy(IRetryPolicy retryPolicy)
         /// <summary>
         /// Releases the unmanaged resources used by the DeviceClient and optionally disposes of the managed resources.
         /// </summary>
-        public void Dispose() => InternalClient?.Dispose();
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases the unmanaged resources used by the DeviceClient and allows for any derived class to override and
+        /// provide custom implementation.
+        /// </summary>
+        /// <param name="disposing">Setting to true will release both managed and unmanaged resources. Setting to
+        /// false will only release the unmanaged resources.</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                InternalClient?.Dispose();
+                InternalClient = null;
+            }
+        }
 
         /// <summary>
         /// Set a callback that will be called whenever the client receives a state update

diff --git a/iothub/device/src/ModernDotNet/HsmAuthentication/Transport/HttpUdsMessageHandler.cs b/iothub/device/src/ModernDotNet/HsmAuthentication/Transport/HttpUdsMessageHandler.cs
@@ -25,6 +25,7 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
             using var stream = new HttpBufferedStream(new NetworkStream(socket, true));
 
             byte[] requestBytes = HttpRequestResponseSerializer.SerializeRequest(request);
+
 #if NET451 || NET472 || NETSTANDARD2_0
             await stream.WriteAsync(requestBytes, 0, requestBytes.Length, cancellationToken).ConfigureAwait(false);
 #else
@@ -42,10 +43,25 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
 
         private async Task<Socket> GetConnectedSocketAsync()
         {
-            var endpoint = new UnixDomainSocketEndPoint(_providerUri.LocalPath);
             Socket socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified);
-            await socket.ConnectAsync(endpoint).ConfigureAwait(false);
 
+            // The Edge Agent uses unix sockets for communication with the modules deployed in docker for HSM.
+            // For netstandard 2.0 there was no implementation for a Unix Domain Socket (UDS) so we used a version
+            // that was part of a test that was reused in a number of libraries on the internet.
+            //
+            // https://github.com/dotnet/corefx/blob/12b51c6bf153cc237b251a4e264d5e7c0ee84a33/src/System.IO.Pipes/src/System/Net/Sockets/UnixDomainSocketEndPoint.cs
+            // https://github.com/dotnet/corefx/blob/12b51c6bf153cc237b251a4e264d5e7c0ee84a33/src/System.Net.Sockets/tests/FunctionalTests/UnixDomainSocketTest.cs#L248
+            //
+            // Since then the UnixDomainSocketEndpoint has been added to the dotnet framework and there has been considerable work
+            // around unix sockets in the BCL. For older versions of the framework we will continue to use the existing class since it works
+            // fine. For netcore 2.1 and greater as well as .NET 5.0 and greater we'll use the native framework version.
+
+#if NET451 || NET472 || NETSTANDARD2_0
+            var endpoint = new Microsoft.Azure.Devices.Client.HsmAuthentication.Transport.UnixDomainSocketEndPoint(_providerUri.LocalPath);
+#else
+            var endpoint = new System.Net.Sockets.UnixDomainSocketEndPoint(_providerUri.LocalPath);
+#endif
+            await socket.ConnectAsync(endpoint).ConfigureAwait(false);
             return socket;
         }
     }