Merge pull request #14 from Microsoft/staging

init

DAT - Getting the most out of your data/DAT10/README.md

@@ -0,0 +1,11 @@
+# Dealing with a Massive Onset of Data Ingestion
+
+Instructions and code to recreate the demo in the Microsoft Ignite | The Tour session titled [_Dealing with a Massive Onset of Data Ingestion_](./session).
+
+# Edge Nodes
+
+A living repository of best practices and examples for developing [Azure IoT Edge](./edge).
+
+# Data Generator
+
+An IoT Edge Module that [Generates Sample Data](./data-generator)

DAT - Getting the most out of your data/DAT10/data-generator/.gitignore

@@ -0,0 +1,2 @@
+config/
+.env

DAT - Getting the most out of your data/DAT10/data-generator/.vscode/launch.json

@@ -0,0 +1,43 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "azure-iot-edge-bogus-data-generator Remote Debug (.NET Core)",
+      "type": "coreclr",
+      "request": "attach",
+      "processId": "${command:pickRemoteProcess}",
+      "pipeTransport": {
+        "pipeProgram": "docker",
+        "pipeArgs": [
+          "exec",
+          "-i",
+          "azure-iot-edge-bogus-data-generator",
+          "sh",
+          "-c"
+        ],
+        "debuggerPath": "~/vsdbg/vsdbg",
+        "pipeCwd": "${workspaceFolder}",
+        "quoteArgs": true
+      },
+      "sourceFileMap": {
+        "/app": "${workspaceFolder}/modules/IoTEdgeBogusDataGenerator"
+      },
+      "justMyCode": true
+    },
+    {
+      "name": "azure-iot-edge-bogus-data-generator Local Debug (.NET Core)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceRoot}/modules/IoTEdgeBogusDataGenerator/bin/Debug/netcoreapp2.1/IoTEdgeBogusDataGenerator.dll",
+      "args": [],
+      "cwd": "${workspaceRoot}/modules/IoTEdgeBogusDataGenerator",
+      "internalConsoleOptions": "openOnSessionStart",
+      "stopAtEntry": false,
+      "console": "internalConsole",
+      "env": {
+        "EdgeHubConnectionString": "${config:azure-iot-edge.EdgeHubConnectionString}",
+        "EdgeModuleCACertificateFile": "${config:azure-iot-edge.EdgeModuleCACertificateFile}"
+      }
+    }
+  ]
+}

DAT - Getting the most out of your data/DAT10/data-generator/README.MD

@@ -0,0 +1,16 @@
+# azure-iot-edge-bogus-data-generator
+An IoT Edge Module that generates sample data using [Bogus](https://github.com/bchavez/Bogus)
+
+[![Docker Build Status](https://dockerbuildbadges.quelltext.eu/status.svg?organization=toolboc&repository=azure-iot-edge-bogus-data-generator)](https://hub.docker.com/r/toolboc/azure-iot-edge-bogus-data-generator/builds) [![Docker Pulls](https://img.shields.io/docker/pulls/toolboc/azure-iot-edge-bogus-data-generator.svg?style=flat-square)](https://hub.docker.com/r/toolboc/azure-iot-edge-bogus-data-generator]/)
+
+![Demo](https://raw.githubusercontent.com/toolboc/azure-iot-edge-bogus-data-generator/master/assets/demo.png)
+
+## Getting Started
+Modify the `CreateBogusData` method in `BogusDataFatory.cs` to customize the simulated data.  See the [Bogus README](https://github.com/bchavez/Bogus#the-great-c-example) for examples.
+
+## Module Twin Properties
+
+| Parameter      | Description |   Type        |
+| -------------- | ------------| --------- |
+| SendData      | Controls whether to send data (default: true) | Boolean  |
+| SendInterval   | Interval to generate and send data (in seconds) | Integer |

DAT - Getting the most out of your data/DAT10/data-generator/deployment.template.json

@@ -0,0 +1,58 @@
+{
+  "modulesContent": {
+    "$edgeAgent": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "runtime": {
+          "type": "docker",
+          "settings": {
+            "minDockerVersion": "v1.25",
+            "loggingOptions": "",
+            "registryCredentials": {}
+          }
+        },
+        "systemModules": {
+          "edgeAgent": {
+            "type": "docker",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-agent:1.0",
+              "createOptions": "{}"
+            }
+          },
+          "edgeHub": {
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "mcr.microsoft.com/azureiotedge-hub:1.0",
+              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}], \"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
+            }
+          }
+        },
+        "modules": {
+          "IoTEdgeBogusDataGenerator": {
+            "version": "1.0",
+            "type": "docker",
+            "status": "running",
+            "restartPolicy": "always",
+            "settings": {
+              "image": "${MODULES.azure-iot-edge-bogus-data-generator.amd64}",
+              "createOptions": "{}"
+            }
+          }
+        }
+      }
+    },
+    "$edgeHub": {
+      "properties.desired": {
+        "schemaVersion": "1.0",
+        "routes": {
+          "IoTEdgeBogusDataGeneratorToIoTHub": "FROM /messages/modules/IoTEdgeBogusDataGenerator/outputs/* INTO $upstream"
+        },
+        "storeAndForwardConfiguration": {
+          "timeToLiveSecs": 7200
+        }
+      }
+    }
+  }
+}

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/.gitignore

@@ -0,0 +1,34 @@
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+**/Properties/launchSettings.json
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+.vs
+
+[Bb]in/
+[Oo]bj/

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/BogusDataFactory.cs

@@ -0,0 +1,97 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using Bogus;
+using Bogus.DataSets;
+using IoTEdgeBogusDataGenerator.Models;
+using System.Collections.Generic;
+
+namespace IoTEdgeBogusDataGenerator
+{
+    public class BogusDataFactory
+    {
+
+        static List<Item> ItemCatalog = new List<Item>()
+        {
+            new Item(){ Sku = "TT2099", Name = "Philips Head Screwdriver" , Price = 2 },
+            new Item(){ Sku = "TT2136", Name = "Screw" , Price = 19 },
+            new Item(){ Sku = "TT2152", Name = "Screw" , Price = 7 },
+            new Item(){ Sku = "TT2178", Name = "Screw" , Price = 8 },
+            new Item(){ Sku = "TT2051", Name = "Extension Cord" , Price = 15 },
+            new Item(){ Sku = "TT2053", Name = "LED Lamp" , Price = 14 },
+            new Item(){ Sku = "TT2057", Name = "Timer Outlet" , Price = 9 },
+            new Item(){ Sku = "TT2059", Name = "Paint Roller" , Price = 2 },
+            new Item(){ Sku = "TT2061", Name = "Sponge" , Price = 1 },
+            new Item(){ Sku = "TT2062", Name = "Tile Backsplash" , Price = 2 }
+        };
+        static List<string> Stores = new List<string>()
+        {
+            "tailwind-traders.com (web site)",
+            "Redmond Center"
+        };
+
+        public static Object CreateBogusData(int counter)
+        {
+            //Set the randomzier seed if you wish to generate repeatable data sets.
+            Randomizer.Seed = new Random(counter);
+
+            var testOrder = new Faker<Order>()
+                //Ensure all properties have rules. By default, StrictMode is false
+                //Set a global policy by using Faker.DefaultStrictMode
+                .StrictMode(true)
+                //OrderNumber is deterministic
+                .RuleFor(o => o.OrderNumber, f => counter)
+                //Generate Random GUID for PurchaseId
+                .RuleFor(o => o.PurchaseId, f => Guid.NewGuid())
+                //Pick a Random Store
+                .RuleFor(o => o.StoreName, f => f.PickRandom(Stores))
+                //Generate Fake Name
+                .RuleFor(o => o.CustomerName, f => f.Name.FullName())
+                //Generate Fake Email
+                .RuleFor(o => o.Email, (f,o) => f.Internet.Email(o.CustomerName))
+                //Generate Fake Address
+                .RuleFor(o => o.ShippingAddress, f => f.Address.StreetAddress())
+                //Get Fake Items
+                .RuleFor(o => o.Items, f => GetFakeItems(f))
+                //Get Amount Due
+                .RuleFor(o => o.Amount_Due, (f,o) => GetAmountDue(o.Items))
+                //Stamp with the current time
+                .RuleFor(o => o.Timestamp, f => DateTime.UtcNow);
+
+            return testOrder.Generate();
+        }
+
+        public static List<Item> GetFakeItems(Faker f)
+        {
+            List<Item> Items = new List<Item>();
+            int numberOfItems = f.Random.Number(1,5);
+
+            for(int i = 0; i < numberOfItems; i++)
+            {
+                var item = f.PickRandom(ItemCatalog);
+
+                if(item.Name == "Screw")
+                  {
+                    Items.Add(new Item(){Sku = "TT2099" , Name = "Philips Head Screwdriver", Price = 2});
+                    Console.WriteLine("Correlation Forced");
+                  }  
+
+                Items.Add(item);
+            }
+
+            return Items;
+        }
+        public static int GetAmountDue(List<Item> items)
+        {
+            int total = 0;
+
+            foreach(var item in items)
+            {
+                total += item.Price;
+            }
+
+            return total;
+        }
+    }
+}

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/DesiredPropertiesData.cs

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using Microsoft.Azure.Devices.Shared;
+using Newtonsoft.Json;
+
+namespace IoTEdgeBogusDataGenerator
+{
+    public class DesiredPropertiesData
+    {
+        private bool _sendData = true;
+        private int _sendInterval = 5;
+
+        public DesiredPropertiesData(TwinCollection twinCollection)
+        {
+            Console.WriteLine($"Updating desired properties {twinCollection.ToJson(Formatting.Indented)}");
+            try
+            {
+                if(twinCollection.Contains("SendData") && twinCollection["SendData"] != null)
+                {
+                    _sendData = twinCollection["SendData"];
+                }
+
+                if(twinCollection.Contains("SendInterval") && twinCollection["SendInterval"] != null)
+                {
+                    _sendInterval = twinCollection["SendInterval"];
+                }
+            }
+            catch(AggregateException aexc)
+            {
+                foreach(var exception in aexc.InnerExceptions)
+                {
+                    Console.WriteLine($"[ERROR] Could not retrieve desired properties {aexc.Message}");
+                }
+            }
+            catch(Exception ex)
+            {
+                Console.WriteLine($"[ERROR] Reading desired properties failed with {ex.Message}");
+            }
+            finally
+            {
+                Console.WriteLine($"Value for SendData = {_sendData}");
+                Console.WriteLine($"Value for SendInterval = {_sendInterval}");
+            }
+        }
+
+        public bool SendData => _sendData;
+        public int SendInterval => _sendInterval;
+    }
+}

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/Dockerfile

@@ -0,0 +1,17 @@
+FROM microsoft/dotnet:2.1-sdk AS build-env
+WORKDIR /app
+
+COPY *.csproj ./
+RUN dotnet restore
+
+COPY . ./
+RUN dotnet publish -c Release -o out
+
+FROM microsoft/dotnet:2.1-runtime-stretch-slim
+WORKDIR /app
+COPY --from=build-env /app/out ./
+
+RUN useradd -ms /bin/bash moduleuser
+USER moduleuser
+
+ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/Dockerfile.amd64

@@ -0,0 +1,17 @@
+FROM microsoft/dotnet:2.1-sdk AS build-env
+WORKDIR /app
+
+COPY *.csproj ./
+RUN dotnet restore
+
+COPY . ./
+RUN dotnet publish -c Release -o out
+
+FROM microsoft/dotnet:2.1-runtime-stretch-slim
+WORKDIR /app
+COPY --from=build-env /app/out ./
+
+RUN useradd -ms /bin/bash moduleuser
+USER moduleuser
+
+ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]

DAT - Getting the most out of your data/DAT10/data-generator/modules/azure-iot-edge-bogus-data-generator/Dockerfile.amd64.debug

@@ -0,0 +1,24 @@
+FROM microsoft/dotnet:2.1-runtime-stretch-slim AS base
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends unzip procps && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -ms /bin/bash moduleuser
+USER moduleuser
+RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg
+
+FROM microsoft/dotnet:2.1-sdk AS build-env
+WORKDIR /app
+
+COPY *.csproj ./
+RUN dotnet restore
+
+COPY . ./
+RUN dotnet publish -c Debug -o out
+
+FROM base
+WORKDIR /app
+COPY --from=build-env /app/out ./
+
+ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]