Skip to content
Permalink
Browse files

init

  • Loading branch information...
Scott Cate Scott Cate
Scott Cate authored and Scott Cate committed Dec 6, 2018
1 parent 3a8b5ae commit 8a0b48436b2fd112bd064124a99e6fab354654ae
Showing 1,532 changed files with 304,700 additions and 0 deletions.
@@ -0,0 +1,11 @@
# Dealing with a Massive Onset of Data Ingestion

Instructions and code to recreate the demo in the Microsoft Ignite | The Tour session titled [_Dealing with a Massive Onset of Data Ingestion_](./session).

# Edge Nodes

A living repository of best practices and examples for developing [Azure IoT Edge](./edge).

# Data Generator

An IoT Edge Module that [Generates Sample Data](./data-generator)
@@ -0,0 +1,2 @@
config/
.env
@@ -0,0 +1,43 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "azure-iot-edge-bogus-data-generator Remote Debug (.NET Core)",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickRemoteProcess}",
"pipeTransport": {
"pipeProgram": "docker",
"pipeArgs": [
"exec",
"-i",
"azure-iot-edge-bogus-data-generator",
"sh",
"-c"
],
"debuggerPath": "~/vsdbg/vsdbg",
"pipeCwd": "${workspaceFolder}",
"quoteArgs": true
},
"sourceFileMap": {
"/app": "${workspaceFolder}/modules/IoTEdgeBogusDataGenerator"
},
"justMyCode": true
},
{
"name": "azure-iot-edge-bogus-data-generator Local Debug (.NET Core)",
"type": "coreclr",
"request": "launch",
"program": "${workspaceRoot}/modules/IoTEdgeBogusDataGenerator/bin/Debug/netcoreapp2.1/IoTEdgeBogusDataGenerator.dll",
"args": [],
"cwd": "${workspaceRoot}/modules/IoTEdgeBogusDataGenerator",
"internalConsoleOptions": "openOnSessionStart",
"stopAtEntry": false,
"console": "internalConsole",
"env": {
"EdgeHubConnectionString": "${config:azure-iot-edge.EdgeHubConnectionString}",
"EdgeModuleCACertificateFile": "${config:azure-iot-edge.EdgeModuleCACertificateFile}"
}
}
]
}
@@ -0,0 +1,16 @@
# azure-iot-edge-bogus-data-generator
An IoT Edge Module that generates sample data using [Bogus](https://github.com/bchavez/Bogus)

[![Docker Build Status](https://dockerbuildbadges.quelltext.eu/status.svg?organization=toolboc&repository=azure-iot-edge-bogus-data-generator)](https://hub.docker.com/r/toolboc/azure-iot-edge-bogus-data-generator/builds) [![Docker Pulls](https://img.shields.io/docker/pulls/toolboc/azure-iot-edge-bogus-data-generator.svg?style=flat-square)](https://hub.docker.com/r/toolboc/azure-iot-edge-bogus-data-generator]/)

![Demo](https://raw.githubusercontent.com/toolboc/azure-iot-edge-bogus-data-generator/master/assets/demo.png)

## Getting Started
Modify the `CreateBogusData` method in `BogusDataFatory.cs` to customize the simulated data. See the [Bogus README](https://github.com/bchavez/Bogus#the-great-c-example) for examples.

## Module Twin Properties

| Parameter | Description | Type |
| -------------- | ------------| --------- |
| SendData | Controls whether to send data (default: true) | Boolean |
| SendInterval | Interval to generate and send data (in seconds) | Integer |
Binary file not shown.
@@ -0,0 +1,58 @@
{
"modulesContent": {
"$edgeAgent": {
"properties.desired": {
"schemaVersion": "1.0",
"runtime": {
"type": "docker",
"settings": {
"minDockerVersion": "v1.25",
"loggingOptions": "",
"registryCredentials": {}
}
},
"systemModules": {
"edgeAgent": {
"type": "docker",
"settings": {
"image": "mcr.microsoft.com/azureiotedge-agent:1.0",
"createOptions": "{}"
}
},
"edgeHub": {
"type": "docker",
"status": "running",
"restartPolicy": "always",
"settings": {
"image": "mcr.microsoft.com/azureiotedge-hub:1.0",
"createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}], \"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
}
}
},
"modules": {
"IoTEdgeBogusDataGenerator": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"settings": {
"image": "${MODULES.azure-iot-edge-bogus-data-generator.amd64}",
"createOptions": "{}"
}
}
}
}
},
"$edgeHub": {
"properties.desired": {
"schemaVersion": "1.0",
"routes": {
"IoTEdgeBogusDataGeneratorToIoTHub": "FROM /messages/modules/IoTEdgeBogusDataGenerator/outputs/* INTO $upstream"
},
"storeAndForwardConfiguration": {
"timeToLiveSecs": 7200
}
}
}
}
}
@@ -0,0 +1,34 @@
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
**/Properties/launchSettings.json

*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
.vs

[Bb]in/
[Oo]bj/
@@ -0,0 +1,97 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using Bogus;
using Bogus.DataSets;
using IoTEdgeBogusDataGenerator.Models;
using System.Collections.Generic;

namespace IoTEdgeBogusDataGenerator
{
public class BogusDataFactory
{

static List<Item> ItemCatalog = new List<Item>()
{
new Item(){ Sku = "TT2099", Name = "Philips Head Screwdriver" , Price = 2 },
new Item(){ Sku = "TT2136", Name = "Screw" , Price = 19 },
new Item(){ Sku = "TT2152", Name = "Screw" , Price = 7 },
new Item(){ Sku = "TT2178", Name = "Screw" , Price = 8 },
new Item(){ Sku = "TT2051", Name = "Extension Cord" , Price = 15 },
new Item(){ Sku = "TT2053", Name = "LED Lamp" , Price = 14 },
new Item(){ Sku = "TT2057", Name = "Timer Outlet" , Price = 9 },
new Item(){ Sku = "TT2059", Name = "Paint Roller" , Price = 2 },
new Item(){ Sku = "TT2061", Name = "Sponge" , Price = 1 },
new Item(){ Sku = "TT2062", Name = "Tile Backsplash" , Price = 2 }
};
static List<string> Stores = new List<string>()
{
"tailwind-traders.com (web site)",
"Redmond Center"
};

public static Object CreateBogusData(int counter)
{
//Set the randomzier seed if you wish to generate repeatable data sets.
Randomizer.Seed = new Random(counter);

var testOrder = new Faker<Order>()
//Ensure all properties have rules. By default, StrictMode is false
//Set a global policy by using Faker.DefaultStrictMode
.StrictMode(true)
//OrderNumber is deterministic
.RuleFor(o => o.OrderNumber, f => counter)
//Generate Random GUID for PurchaseId
.RuleFor(o => o.PurchaseId, f => Guid.NewGuid())
//Pick a Random Store
.RuleFor(o => o.StoreName, f => f.PickRandom(Stores))
//Generate Fake Name
.RuleFor(o => o.CustomerName, f => f.Name.FullName())
//Generate Fake Email
.RuleFor(o => o.Email, (f,o) => f.Internet.Email(o.CustomerName))
//Generate Fake Address
.RuleFor(o => o.ShippingAddress, f => f.Address.StreetAddress())
//Get Fake Items
.RuleFor(o => o.Items, f => GetFakeItems(f))
//Get Amount Due
.RuleFor(o => o.Amount_Due, (f,o) => GetAmountDue(o.Items))
//Stamp with the current time
.RuleFor(o => o.Timestamp, f => DateTime.UtcNow);

return testOrder.Generate();
}

public static List<Item> GetFakeItems(Faker f)
{
List<Item> Items = new List<Item>();
int numberOfItems = f.Random.Number(1,5);

for(int i = 0; i < numberOfItems; i++)
{
var item = f.PickRandom(ItemCatalog);

if(item.Name == "Screw")
{
Items.Add(new Item(){Sku = "TT2099" , Name = "Philips Head Screwdriver", Price = 2});
Console.WriteLine("Correlation Forced");
}

Items.Add(item);
}

return Items;
}
public static int GetAmountDue(List<Item> items)
{
int total = 0;

foreach(var item in items)
{
total += item.Price;
}

return total;
}
}
}
@@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using Microsoft.Azure.Devices.Shared;
using Newtonsoft.Json;

namespace IoTEdgeBogusDataGenerator
{
public class DesiredPropertiesData
{
private bool _sendData = true;
private int _sendInterval = 5;

public DesiredPropertiesData(TwinCollection twinCollection)
{
Console.WriteLine($"Updating desired properties {twinCollection.ToJson(Formatting.Indented)}");
try
{
if(twinCollection.Contains("SendData") && twinCollection["SendData"] != null)
{
_sendData = twinCollection["SendData"];
}

if(twinCollection.Contains("SendInterval") && twinCollection["SendInterval"] != null)
{
_sendInterval = twinCollection["SendInterval"];
}
}
catch(AggregateException aexc)
{
foreach(var exception in aexc.InnerExceptions)
{
Console.WriteLine($"[ERROR] Could not retrieve desired properties {aexc.Message}");
}
}
catch(Exception ex)
{
Console.WriteLine($"[ERROR] Reading desired properties failed with {ex.Message}");
}
finally
{
Console.WriteLine($"Value for SendData = {_sendData}");
Console.WriteLine($"Value for SendInterval = {_sendInterval}");
}
}

public bool SendData => _sendData;
public int SendInterval => _sendInterval;
}
}
@@ -0,0 +1,17 @@
FROM microsoft/dotnet:2.1-sdk AS build-env
WORKDIR /app

COPY *.csproj ./
RUN dotnet restore

COPY . ./
RUN dotnet publish -c Release -o out

FROM microsoft/dotnet:2.1-runtime-stretch-slim
WORKDIR /app
COPY --from=build-env /app/out ./

RUN useradd -ms /bin/bash moduleuser
USER moduleuser

ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]
@@ -0,0 +1,17 @@
FROM microsoft/dotnet:2.1-sdk AS build-env
WORKDIR /app

COPY *.csproj ./
RUN dotnet restore

COPY . ./
RUN dotnet publish -c Release -o out

FROM microsoft/dotnet:2.1-runtime-stretch-slim
WORKDIR /app
COPY --from=build-env /app/out ./

RUN useradd -ms /bin/bash moduleuser
USER moduleuser

ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]
@@ -0,0 +1,24 @@
FROM microsoft/dotnet:2.1-runtime-stretch-slim AS base

RUN apt-get update && \
apt-get install -y --no-install-recommends unzip procps && \
rm -rf /var/lib/apt/lists/*

RUN useradd -ms /bin/bash moduleuser
USER moduleuser
RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg

FROM microsoft/dotnet:2.1-sdk AS build-env
WORKDIR /app

COPY *.csproj ./
RUN dotnet restore

COPY . ./
RUN dotnet publish -c Debug -o out

FROM base
WORKDIR /app
COPY --from=build-env /app/out ./

ENTRYPOINT ["dotnet", "azure-iot-edge-bogus-data-generator.dll"]

0 comments on commit 8a0b484

Please sign in to comment.
You can’t perform that action at this time.