From 9348fba7464e31b7707d9cb7d65d5da596c357c5 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Thu, 26 Sep 2019 08:38:33 -0400 Subject: [PATCH 01/22] initial commit --- .gitignore | 7 +- NaveegoGrpcPlugin/NaveegoGrpcPlugin.sln | 25 ++++ .../NaveegoGrpcPlugin.csproj | 15 ++ .../NaveegoGrpcPlugin/Program.cs | 34 +++++ .../Properties/launchSettings.json | 12 ++ .../NaveegoGrpcPlugin/Protos/plugin.proto | 134 ++++++++++++++++++ .../Services/PluginService.cs | 29 ++++ .../NaveegoGrpcPlugin/Startup.cs | 44 ++++++ .../appsettings.Development.json | 10 ++ .../NaveegoGrpcPlugin/appsettings.json | 14 ++ 10 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin.sln create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Properties/launchSettings.json create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Protos/plugin.proto create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.Development.json create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json diff --git a/.gitignore b/.gitignore index aa2fc52..ead6c2b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ goplugin/ bin/ .log -**/node_modules/ \ No newline at end of file +**/node_modules/ +NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/Debug/netcoreapp3.0/ + +NaveegoGrpcPlugin/.vs/NaveegoGrpcPlugin/v16/ + +NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/ diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin.sln b/NaveegoGrpcPlugin/NaveegoGrpcPlugin.sln new file mode 100644 index 0000000..8089e20 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29318.209 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NaveegoGrpcPlugin", "NaveegoGrpcPlugin\NaveegoGrpcPlugin.csproj", "{65952571-BC78-418F-A4B6-1A46A1225AAB}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {65952571-BC78-418F-A4B6-1A46A1225AAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {65952571-BC78-418F-A4B6-1A46A1225AAB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {65952571-BC78-418F-A4B6-1A46A1225AAB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {65952571-BC78-418F-A4B6-1A46A1225AAB}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4F447E73-BAC3-49DF-AFD2-4EF27FF8529D} + EndGlobalSection +EndGlobal diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj new file mode 100644 index 0000000..a84f9f5 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp3.0 + + + + + + + + + + + diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs new file mode 100644 index 0000000..d5c7978 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Hosting; + +namespace NaveegoGrpcPlugin +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + // Additional configuration is required to successfully run gRPC on macOS. + // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.ConfigureKestrel(options => + { + // Setup a HTTP/2 endpoint without TLS. + options.ListenLocalhost(50051, o => o.Protocols = + HttpProtocols.Http2); + }); + webBuilder.UseStartup(); + }); + } +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Properties/launchSettings.json b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Properties/launchSettings.json new file mode 100644 index 0000000..be249f5 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "NaveegoGrpcPlugin": { + "commandName": "Project", + "launchBrowser": false, + "applicationUrl": "https://localhost:5001", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Protos/plugin.proto b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Protos/plugin.proto new file mode 100644 index 0000000..f38973b --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Protos/plugin.proto @@ -0,0 +1,134 @@ +syntax = "proto3"; +package plugin; + +// The Plugin service is implemented by plugins which are responsible for discovering +// and publishing data. In this challenge the plugin will target CSV files, but Naveego +// writes plugins against many different kinds of data sources. +// + +// One difficulty in creating a unified interface for interacting with multiple data sources +// is that the configuration settings needed to connect to data sources can differ radically. +// Naveego handles this by having each plugin define a JSONSchema for its settings, +// then dynamically rendering a form based on that schema which the user is asked to fill in. +// The resulting JSON object is then sent to the plugin for validation. In this challenge, +// we'll keep things simple and just hardcode the settings the plugin needs. +// +// Another complication is that data sources differ in the degree to which their schemas +// are discoverable. A SQL database usually provides metadata about the structure of tables, +// but a NoSQL database may not have any structure defined, and a collection of CSV files +// may provide a minimal structure by having headers, but with no type information. +// We handle that variation by providing a UI where a user can strongly specify the available +// schemas in collaboration with the plugin. The user provides some basic connection or +// selection information - such as a database name or a file path - and the plugin uses that +// to do its best to discover the schemas of the available data. We refer to this as the "discovery" +// phase. The user can then look at the discovered schemas, usually along with a sample of the +// data from the data source, and can assign types (like number, date, or boolean) to the +// properties of the schema, as well as annotating the properties with clear names and descriptions. +// The schemas authored by technical users are then made available to business users who can use +// them to construct their business-level data collection and merge flows. +// +// To help users, we'd like to be able to heuristically infer the types of properties that are +// discovered in data sources which do not provide strong type information. If you find +// implementing a passing solution not challenging enough, try adding some type inference logic +// which will examine the values in the data source in order to specify the types without user +// intervention. This may be tricky, since some records may have invalid data. If you've provided +// inferred types during discovery, you can mark records with bad data as invalid, but you +// should still publish the record. + +service Plugin { + // The Discover method is responsible for taking the provided settings + // and using them to find and describe all the schemas which the settings make available. + // In this case, the plugin will look for CSV files which match a pattern. + rpc Discover (DiscoverRequest) returns (DiscoverResponse) { + } + + // The Publish method is responsible for collecting all the records + // which belong to a single schema and streaming them back to the host. + // The schema which is passed in will be one of the schemas returned by the + // Discover method, so you can share data between Discover and Publish by + // means of the `settings` string on the Schema message. + rpc Publish (PublishRequest) returns (stream PublishRecord) { + } +} + +// The request message containing the user's name. +message DiscoverRequest { + // In a real plugin the settings would be conveyed in a JSON object + // with a schema defined by the plugin and populated by a user through a UI. + // For this challenge we'll define a message for the settings + // to make it easier for the host to create the settings. + Settings settings = 1; +} + +message Settings { + // This a glob specifying the pattern to use to find files. + // This will be an absolute path something like /src/data/*/*.csv. + // The plugin should find all files matching the pattern, then + // analyze them to find the unique schemas among them (multiple files + // may have the same schema). + // + // For this challenge you can assume that all CSV files have a header row, + // and that all files are comma delimited + string fileGlob = 1; +} + +message DiscoverResponse { + // Array of schemas discovered. + repeated Schema schemas = 1; +} + +message Schema { + // The unique name of the schema; if there is no unique name + // the plugin can generate one. + string name = 1; + // The settings the plugin will use for publishing when + // this schema is included in a PublishRequest. This + // can be any data the plugin wants to capture; the host + // will treat it as an opaque blob. + // Hint: this is a good place to store the file paths of all the + // files which contain records with this schema. + string settings = 2; + // Array of the properties discovered for this schema, + // in the order they have in the CSV. + repeated Property properties = 3; +} + +message Property { + // Name of the property, from the column header. + string name = 1; + // Type of the property; can be "string", "integer", "number", "datetime", "boolean" + // This should be inferred if possible by analyzing the data. + // This is an optional part of the challenge; you can pass the tests + // without populating this field. + string type = 2; +} + +message PublishRequest { + // The settings will be the same as the settings sent to the Discover method. + Settings settings = 1; + // The schema will be one of the schemas returned by the Discover method. + Schema schema = 2; +} + +message PublishRecord { + // This should be set to true if the record is not valid + // because it violates the inferred schema in some way. + // The invalid property should be written as `null` in data. + bool invalid = 1; + + // If the record is invalid this field should explain why. + // This should include the property name and the original value. + // This is a user-directed (as opposed to machine-readable) field, + // so you can format it however you want. + string error = 2; + // Data contains the values for a single record. + // The values should be provided as a JSON serialized array. + // For example, if the CSV has a row + // 17,Alabama,true + // then the data field should contain the string + // "[17,\"Alabama\",true]" + // however, if you have not inferred the types of the properties, + // it's OK to make everything a string, which would be + // "[\"17\",\"Alabama\",\"true\"]" + string data = 3; +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs new file mode 100644 index 0000000..901c983 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Grpc.Core; +using Microsoft.Extensions.Logging; +using Plugin; + +namespace NaveegoGrpcPlugin +{ + public class PluginService : Plugin.Plugin.PluginBase + { + private readonly ILogger _logger; + public PluginService(ILogger logger) + { + _logger = logger; + } + + public override Task Discover(DiscoverRequest request, ServerCallContext context) + { + var examine = request.Settings.FileGlob; + return Task.FromResult(new DiscoverResponse { Schemas = { new Schema { Name = "Test" } } }); + } + + + + + } +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs new file mode 100644 index 0000000..b824c6b --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace NaveegoGrpcPlugin +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + services.AddGrpc(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + Console.WriteLine("50051"); + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGrpcService(); + + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909"); + }); + }); + } + } +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.Development.json b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.Development.json new file mode 100644 index 0000000..fe20c40 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.Development.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Grpc": "Information", + "Microsoft": "Information" + } + } +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json new file mode 100644 index 0000000..efb2625 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json @@ -0,0 +1,14 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*", + "Kestrel": { + "EndpointDefaults": { + "Protocols": "Http2" + } + } +} From 0f6185bc0eed094cea3c6f349b9ee2f759aa2894 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Thu, 26 Sep 2019 09:06:02 -0400 Subject: [PATCH 02/22] discover --- .../NaveegoGrpcPlugin.csproj | 2 + .../Services/PluginService.cs | 107 +++++++++++++++++- 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj index a84f9f5..18b75fb 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj @@ -5,6 +5,8 @@ + + diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 901c983..b4ec8df 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -5,25 +5,126 @@ using Grpc.Core; using Microsoft.Extensions.Logging; using Plugin; +using Ganss.IO; +using System.IO; +using CsvHelper; namespace NaveegoGrpcPlugin { public class PluginService : Plugin.Plugin.PluginBase { private readonly ILogger _logger; + private List discoveredSchemas; public PluginService(ILogger logger) { _logger = logger; + discoveredSchemas = new List(); } public override Task Discover(DiscoverRequest request, ServerCallContext context) { var examine = request.Settings.FileGlob; - return Task.FromResult(new DiscoverResponse { Schemas = { new Schema { Name = "Test" } } }); + LookForFiles(examine); + return Task.FromResult(new DiscoverResponse { Schemas = { discoveredSchemas } }); } - + private void LookForFiles(string fileGlob) + { + _logger.LogInformation("Looking for files in {glob}", fileGlob); + try + { + + var foundFiles = Glob.Expand(fileGlob); + + if (!foundFiles.Any()) + { + _logger.LogInformation("Found no file matches"); + } + + foreach (var file in foundFiles) + { + _logger.LogInformation("Found {file}. Looking into it...", file.Name); + InvestigateFile(file.FullName); + } + + } + catch (Exception e) + { + _logger.LogError(e, "Error looking for files in glob"); + } + + } + private void InvestigateFile(string filePath) + { + try + { + using (var reader = new StreamReader(filePath)) + using (var csv = new CsvReader(reader)) + { + csv.Read(); + csv.ReadHeader(); + + var headers = csv.Context.HeaderRecord; + var props = CreateProperties(headers); + var foundSchema = CheckForExistingSchema(props); + if (foundSchema != null) + { + AppendFileToSchema(foundSchema, filePath); + } + else + { + discoveredSchemas.Add(CreateSchema(filePath, props)); + } + + + } + } + catch (Exception e) + { + _logger.LogError(e, "Error opening csv"); + } + + + } + private Schema CheckForExistingSchema(List props) + { + + foreach (var schema in discoveredSchemas) + { + var schemaArray = schema.Properties.ToArray(); + var propsArray = props.ToArray(); + if (schemaArray.Length == propsArray.Length && schemaArray.Intersect(propsArray).Count() == schemaArray.Length) + { + _logger.LogInformation("Found same schema called {name}. ", schema.Name); + return schema; + } + } + return null; + } + + private void AppendFileToSchema(Schema schema, string filePath) + { + schema.Settings += ";" + filePath; + } + + private Schema CreateSchema(string fileName, List props) + { + string schemaName = "Schema" + (discoveredSchemas.Count + 1); + _logger.LogInformation("Creating schema called {name}. ", schemaName); + return new Schema { Name = schemaName, Settings = fileName, Properties = { props } }; + } + + private List CreateProperties(string[] propsToCreate) + { + List newProps = new List(); + foreach (var header in propsToCreate) + { + newProps.Add(new Property { Name = header }); // not including Type yet + } + + return newProps; + } } -} +} \ No newline at end of file From d8a10199a615b838d9933aa440c3df2147af96d7 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Thu, 26 Sep 2019 09:12:10 -0400 Subject: [PATCH 03/22] publish --- .../Services/PluginService.cs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index b4ec8df..50d5f61 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -8,6 +8,8 @@ using Ganss.IO; using System.IO; using CsvHelper; +using Google.Protobuf.Collections; +using System.Text.Json; namespace NaveegoGrpcPlugin { @@ -28,6 +30,16 @@ public override Task Discover(DiscoverRequest request, ServerC return Task.FromResult(new DiscoverResponse { Schemas = { discoveredSchemas } }); } + public override async Task Publish(PublishRequest request, IServerStreamWriter responseStream, ServerCallContext context) + { + var filePaths = request.Schema.Settings.Split(';'); + var props = request.Schema.Properties; + foreach (var file in filePaths) + { + await GetDataToStream(file, props, responseStream); + } + } + private void LookForFiles(string fileGlob) { _logger.LogInformation("Looking for files in {glob}", fileGlob); @@ -126,5 +138,34 @@ private List CreateProperties(string[] propsToCreate) return newProps; } + + private async Task GetDataToStream(string filePath, RepeatedField props, IServerStreamWriter responseStream) + { + try + { + using (var reader = new StreamReader(filePath)) + using (var csv = new CsvReader(reader)) + { + + foreach (var record in csv.GetRecords()) + { + var fullRecord = new List(); + foreach (KeyValuePair col in record) + { + fullRecord.Add(col.Value.ToString()); + } + + var data = JsonSerializer.Serialize(fullRecord); + await responseStream.WriteAsync(new PublishRecord { Data = data, Invalid = false }); + } + + } + } + catch (Exception e) + { + _logger.LogError(e, "Error opening csv"); + } + } + } -} \ No newline at end of file +} From f4c96b9fc57a58c901c2e1c382e79c1ec1f8736f Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Thu, 26 Sep 2019 10:31:40 -0400 Subject: [PATCH 04/22] attempt to catch sigkill --- NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index d5c7978..c8bd493 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -13,9 +13,15 @@ public class Program { public static void Main(string[] args) { + AppDomain.CurrentDomain.ProcessExit += ProcessExitHandler; CreateHostBuilder(args).Build().Run(); } + private static void ProcessExitHandler(object sender, EventArgs e) + { + Console.WriteLine("Shutting down"); + } + // Additional configuration is required to successfully run gRPC on macOS. // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 public static IHostBuilder CreateHostBuilder(string[] args) => From b6ecd3c0922718ae020d0b32ddab949a871426b6 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Thu, 26 Sep 2019 10:31:56 -0400 Subject: [PATCH 05/22] build in docker --- Dockerfile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f3a8d89..167f76e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,24 @@ ADD host.go . ENTRYPOINT ["go", "run", "host.go"] # Build your implementation here +FROM mcr.microsoft.com/dotnet/core/runtime:3.0 AS base +WORKDIR /app +FROM mcr.microsoft.com/dotnet/core/sdk:3.0 AS build +WORKDIR /src +COPY NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj NaveegoGrpcPlugin/ +RUN dotnet restore NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj +COPY . . +WORKDIR /src/NaveegoGrpcPlugin/NaveegoGrpcPlugin +RUN dotnet build NaveegoGrpcPlugin.csproj -c Release -o /app + +FROM build AS publish +RUN dotnet publish NaveegoGrpcPlugin.csproj -c Release -o /app -r linux-x64 --self-contained true + +FROM base AS final +WORKDIR /app +COPY --from=publish /app . # Put your implementation here -CMD ["plugin"] +CMD ["./NaveegoGrpcPlugin"] From 7e1221cd18092d8bc97d5e6f52532cc8271dbbdc Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 10:08:14 -0400 Subject: [PATCH 06/22] improved logging, set port from appsettings file --- .../NaveegoGrpcPlugin.csproj | 1 + .../NaveegoGrpcPlugin/Program.cs | 42 ++++++++++++++++--- .../NaveegoGrpcPlugin/Startup.cs | 25 ++++++++++- .../NaveegoGrpcPlugin/appsettings.json | 7 +--- .../runtimeconfig.template.json | 5 +++ 5 files changed, 67 insertions(+), 13 deletions(-) create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/runtimeconfig.template.json diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj index 18b75fb..e00b9ad 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/NaveegoGrpcPlugin.csproj @@ -8,6 +8,7 @@ + diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index c8bd493..6dafb1c 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -5,16 +5,46 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; +using Serilog; +using Serilog.Events; namespace NaveegoGrpcPlugin { public class Program { - public static void Main(string[] args) + public static int Main(string[] args) { - AppDomain.CurrentDomain.ProcessExit += ProcessExitHandler; - CreateHostBuilder(args).Build().Run(); + + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + //.MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.FromLogContext() + //.WriteTo.Console() + .WriteTo.File("PluginLog.txt", rollingInterval: RollingInterval.Day) + .CreateLogger(); + + try + { + AppDomain.CurrentDomain.ProcessExit += ProcessExitHandler; + var configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json") + .Build(); + CreateHostBuilder(args, configuration).Build().Run(); + return 0; + } + catch (Exception ex) + { + Log.Fatal(ex, "Host terminated unexpectedly"); + return 1; + } + finally + { + Log.CloseAndFlush(); + } + } private static void ProcessExitHandler(object sender, EventArgs e) @@ -24,17 +54,19 @@ private static void ProcessExitHandler(object sender, EventArgs e) // Additional configuration is required to successfully run gRPC on macOS. // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 - public static IHostBuilder CreateHostBuilder(string[] args) => + public static IHostBuilder CreateHostBuilder(string[] args, IConfiguration configuration) => Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => { webBuilder.ConfigureKestrel(options => { + var port = configuration.GetValue("GrpcPort"); // Setup a HTTP/2 endpoint without TLS. - options.ListenLocalhost(50051, o => o.Protocols = + options.ListenLocalhost(port, o => o.Protocols = HttpProtocols.Http2); }); webBuilder.UseStartup(); + webBuilder.UseSerilog(); }); } } diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs index b824c6b..d78be90 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs @@ -5,13 +5,20 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Serilog; namespace NaveegoGrpcPlugin { public class Startup { + private IConfiguration Configuration { get; } + public Startup(IConfiguration configuration) + { + Configuration = configuration; + } // This method gets called by the runtime. Use this method to add services to the container. // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) @@ -20,13 +27,13 @@ public void ConfigureServices(IServiceCollection services) } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime applicationLifetime) { if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } - Console.WriteLine("50051"); + app.UseSerilogRequestLogging(); app.UseRouting(); @@ -39,6 +46,20 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909"); }); }); + + applicationLifetime.ApplicationStarted.Register(ShowPort); + + + } + + private void ShowPort() + { + var port = Configuration.GetValue("GrpcPort").ToString(); + if (!string.IsNullOrEmpty(port)) + { + Console.WriteLine(port); + } + } } } diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json index efb2625..77bbd4d 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/appsettings.json @@ -1,11 +1,6 @@ { - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft.Hosting.Lifetime": "Information" - } - }, "AllowedHosts": "*", + "GrpcPort": "50051", "Kestrel": { "EndpointDefaults": { "Protocols": "Http2" diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/runtimeconfig.template.json b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/runtimeconfig.template.json new file mode 100644 index 0000000..d43f4c0 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/runtimeconfig.template.json @@ -0,0 +1,5 @@ +{ + "configProperties": { + "System.Globalization.Invariant": true + } +} From 7a2f8ed3893d4d726b20a41fdb76cdebfb55547a Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 10:35:49 -0400 Subject: [PATCH 07/22] delimiter now a property --- NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs | 4 ++-- .../NaveegoGrpcPlugin/Services/PluginService.cs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index 6dafb1c..713f0ec 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -18,8 +18,8 @@ public static int Main(string[] args) { Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - //.MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) .Enrich.FromLogContext() //.WriteTo.Console() .WriteTo.File("PluginLog.txt", rollingInterval: RollingInterval.Day) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 50d5f61..97d05be 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -17,9 +17,11 @@ public class PluginService : Plugin.Plugin.PluginBase { private readonly ILogger _logger; private List discoveredSchemas; + private readonly char delimiter; public PluginService(ILogger logger) { _logger = logger; + delimiter = ';'; discoveredSchemas = new List(); } @@ -32,7 +34,7 @@ public override Task Discover(DiscoverRequest request, ServerC public override async Task Publish(PublishRequest request, IServerStreamWriter responseStream, ServerCallContext context) { - var filePaths = request.Schema.Settings.Split(';'); + var filePaths = request.Schema.Settings.Split(delimiter); var props = request.Schema.Properties; foreach (var file in filePaths) { @@ -118,7 +120,7 @@ private Schema CheckForExistingSchema(List props) private void AppendFileToSchema(Schema schema, string filePath) { - schema.Settings += ";" + filePath; + schema.Settings += delimiter + filePath; } private Schema CreateSchema(string fileName, List props) From f3c17de417a21164b20d9174b6acfd6fc21f4f26 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 10:42:34 -0400 Subject: [PATCH 08/22] working Dockerfile --- Dockerfile | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 167f76e..d07be5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,4 @@ -FROM golang:1.11-stretch - -WORKDIR /code-challenge-plugin - -ADD go.mod . -ADD go.sum . - -RUN go mod download - -ADD ./plugin/ ./plugin -ADD ./data/ ./data - -ADD host.go . - -ENTRYPOINT ["go", "run", "host.go"] - -# Build your implementation here +# I had to build my implementation first FROM mcr.microsoft.com/dotnet/core/runtime:3.0 AS base WORKDIR /app @@ -33,6 +17,24 @@ FROM base AS final WORKDIR /app COPY --from=publish /app . +FROM golang:1.11-stretch + +WORKDIR /code-challenge-plugin +#Then copy it into the go environment +COPY --from=final /app . + +ADD go.mod . +ADD go.sum . + +RUN go mod download + +ADD ./plugin/ ./plugin +ADD ./data/ ./data + +ADD host.go . + +ENTRYPOINT ["go", "run", "host.go"] -# Put your implementation here CMD ["./NaveegoGrpcPlugin"] + +#ENTRYPOINT ["/bin/sh"] From eb17c7b611f8c6820b87fb63b35065499e5928ae Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 15:38:51 -0400 Subject: [PATCH 09/22] WiP type inference --- .gitignore | 4 ++ .../NaveegoGrpcPlugin/Classes/Types.cs | 68 +++++++++++++++++++ .../Services/PluginService.cs | 51 ++++++++------ 3 files changed, 103 insertions(+), 20 deletions(-) create mode 100644 NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs diff --git a/.gitignore b/.gitignore index ead6c2b..9b9b2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/Debug/netcoreapp3.0/ NaveegoGrpcPlugin/.vs/NaveegoGrpcPlugin/v16/ NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/ + +*.txt + +NaveegoGrpcPlugin/.vs/NaveegoGrpcPlugin/DesignTimeBuild/.dtbcache diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs new file mode 100644 index 0000000..0f8f046 --- /dev/null +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace NaveegoGrpcPlugin +{ + public class Types + { + public bool String { get; set; } + public bool Integer { get; set; } + public bool Number { get; set; } + public bool Datetime { get; set; } + public bool Boolean { get; set; } + + private string _value; + private bool foundType; + + public Types(string value) + { + _value = value; + foundType = false; + IsNumber(); + IsInt(); + IsDatetime(); + IsBoolean(); + + if (!Integer && !Number && !Datetime && !Boolean) + { + //if nothing else, it's a string + String = true; + } + } + + + private void IsNumber() + { + decimal decimalCheck; + decimal.TryParse(_value, out decimalCheck); + Number = decimalCheck != 0 ? true : false; + } + + private void IsInt() + { + int integerCheck; + int.TryParse(_value, out integerCheck); + Integer = integerCheck != 0 ? true : false; + } + + + private void IsDatetime() + { + DateTime datetimeCheck; + DateTime.TryParse(_value, out datetimeCheck); + Datetime = datetimeCheck != DateTime.MinValue ? true : false; + } + + private void IsBoolean() + { + bool booleanCheck; + bool.TryParse(_value, out booleanCheck); + Boolean = booleanCheck; + } + + } + + +} diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 97d05be..ed2a42c 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -73,25 +73,16 @@ private void InvestigateFile(string filePath) { try { - using (var reader = new StreamReader(filePath)) - using (var csv = new CsvReader(reader)) - { - csv.Read(); - csv.ReadHeader(); - - var headers = csv.Context.HeaderRecord; - var props = CreateProperties(headers); - var foundSchema = CheckForExistingSchema(props); - if (foundSchema != null) - { - AppendFileToSchema(foundSchema, filePath); - } - else - { - discoveredSchemas.Add(CreateSchema(filePath, props)); - } - + var props = CreateProperties(filePath); + var foundSchema = CheckForExistingSchema(props); + if (foundSchema != null) + { + AppendFileToSchema(foundSchema, filePath); + } + else + { + discoveredSchemas.Add(CreateSchema(filePath, props)); } } catch (Exception e) @@ -130,10 +121,29 @@ private Schema CreateSchema(string fileName, List props) return new Schema { Name = schemaName, Settings = fileName, Properties = { props } }; } - private List CreateProperties(string[] propsToCreate) + private List CreateProperties(string filePath) { + string[] headers; + using (var reader = new StreamReader(filePath)) + using (var csv = new CsvReader(reader)) + { + csv.Read(); + csv.ReadHeader(); + + headers = csv.Context.HeaderRecord; + foreach (var record in csv.GetRecords()) + { + List types = new List(); + foreach (KeyValuePair col in record) + { + types.Add(new Types(col.Value.ToString())); + } + + } + } + List newProps = new List(); - foreach (var header in propsToCreate) + foreach (var header in headers) { newProps.Add(new Property { Name = header }); // not including Type yet } @@ -141,6 +151,7 @@ private List CreateProperties(string[] propsToCreate) return newProps; } + private async Task GetDataToStream(string filePath, RepeatedField props, IServerStreamWriter responseStream) { try From 314c8136f3b703698bab55bf65e686172d9574ab Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 16:14:37 -0400 Subject: [PATCH 10/22] votes for types, WiP change to dict --- .../NaveegoGrpcPlugin/Classes/Types.cs | 84 ++++++++++++------- .../Services/PluginService.cs | 16 +++- 2 files changed, 68 insertions(+), 32 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index 0f8f046..5b4956e 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -7,59 +7,83 @@ namespace NaveegoGrpcPlugin { public class Types { - public bool String { get; set; } - public bool Integer { get; set; } - public bool Number { get; set; } - public bool Datetime { get; set; } - public bool Boolean { get; set; } + public string ColumnName { get; set; } + public int String { get; set; } + public int Integer { get; set; } + public int Number { get; set; } + public int Datetime { get; set; } + public int Boolean { get; set; } - private string _value; - private bool foundType; + private bool typeFound; - public Types(string value) + public Types(string columnName, string value) { - _value = value; - foundType = false; - IsNumber(); - IsInt(); - IsDatetime(); - IsBoolean(); - - if (!Integer && !Number && !Datetime && !Boolean) + ColumnName = columnName; + DetectTypes(value); + } + + public void DetectTypes(string value) + { + typeFound = false; + VoteForNumber(value); + VoteForInt(value); + VoteForDatetime(value); + VoteForBoolean(value); + if (!typeFound) { - //if nothing else, it's a string - String = true; + VoteForString(); } } - private void IsNumber() + private void VoteForNumber(string value) { decimal decimalCheck; - decimal.TryParse(_value, out decimalCheck); - Number = decimalCheck != 0 ? true : false; + decimal.TryParse(value, out decimalCheck); + if (decimalCheck != 0) + { + Number = Number + 1; + typeFound = true; + } } - private void IsInt() + private void VoteForInt(string value) { int integerCheck; - int.TryParse(_value, out integerCheck); - Integer = integerCheck != 0 ? true : false; + int.TryParse(value, out integerCheck); + if (integerCheck != 0) + { + Integer = Integer + 1; + typeFound = true; + } } - private void IsDatetime() + private void VoteForDatetime(string value) { DateTime datetimeCheck; - DateTime.TryParse(_value, out datetimeCheck); - Datetime = datetimeCheck != DateTime.MinValue ? true : false; + DateTime.TryParse(value, out datetimeCheck); + if (datetimeCheck != DateTime.MinValue) + { + Datetime = Datetime + 1; + typeFound = true; + } } - private void IsBoolean() + private void VoteForBoolean(string value) { bool booleanCheck; - bool.TryParse(_value, out booleanCheck); - Boolean = booleanCheck; + bool.TryParse(value, out booleanCheck); + if (booleanCheck) + { + Boolean = Boolean + 1; + typeFound = true; + } + } + + private void VoteForString() + { + String = String + 1; } } diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index ed2a42c..3b3f0d0 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -131,12 +131,24 @@ private List CreateProperties(string filePath) csv.ReadHeader(); headers = csv.Context.HeaderRecord; + List types = new List(); foreach (var record in csv.GetRecords()) { - List types = new List(); + foreach (KeyValuePair col in record) { - types.Add(new Types(col.Value.ToString())); + var colName = col.Key.ToString(); + var field = col.Value.ToString(); + if (types.Where(w => w.ColumnName == colName).Any()) + { + var colType = types.Where(w => w.ColumnName == colName).First(); + colType.DetectTypes(field); + } + else + { + types.Add(new Types(colName, field)); + } + } } From 83bc6d905cc0b07e11eec61e4ee975472977fa24 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 16:47:54 -0400 Subject: [PATCH 11/22] changed to dictionary storage --- .../NaveegoGrpcPlugin/Classes/Types.cs | 53 +++++++++++++++---- .../Services/PluginService.cs | 17 ++++-- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index 5b4956e..ff89e99 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -8,20 +8,37 @@ namespace NaveegoGrpcPlugin public class Types { public string ColumnName { get; set; } - public int String { get; set; } - public int Integer { get; set; } - public int Number { get; set; } - public int Datetime { get; set; } - public int Boolean { get; set; } + + public Dictionary TypeVotes { + get + { + return typeDict; + } + } private bool typeFound; + private Dictionary typeDict; public Types(string columnName, string value) { + ColumnName = columnName; + SetUpTypes(); DetectTypes(value); } + private void SetUpTypes() + { + typeDict = new Dictionary(); + typeDict.Add(typeof(int), 0); + typeDict.Add(typeof(decimal), 0); + typeDict.Add(typeof(DateTime), 0); + typeDict.Add(typeof(bool), 0); + typeDict.Add(typeof(string), 0); + } + + + public void DetectTypes(string value) { typeFound = false; @@ -42,7 +59,7 @@ private void VoteForNumber(string value) decimal.TryParse(value, out decimalCheck); if (decimalCheck != 0) { - Number = Number + 1; + typeDict[typeof(decimal)] = typeDict[typeof(decimal)] + 1; typeFound = true; } } @@ -53,7 +70,7 @@ private void VoteForInt(string value) int.TryParse(value, out integerCheck); if (integerCheck != 0) { - Integer = Integer + 1; + typeDict[typeof(int)] = typeDict[typeof(int)] + 1; typeFound = true; } } @@ -65,7 +82,7 @@ private void VoteForDatetime(string value) DateTime.TryParse(value, out datetimeCheck); if (datetimeCheck != DateTime.MinValue) { - Datetime = Datetime + 1; + typeDict[typeof(DateTime)] = typeDict[typeof(DateTime)] + 1; typeFound = true; } } @@ -76,14 +93,30 @@ private void VoteForBoolean(string value) bool.TryParse(value, out booleanCheck); if (booleanCheck) { - Boolean = Boolean + 1; + typeDict[typeof(bool)] = typeDict[typeof(bool)] + 1; typeFound = true; } } private void VoteForString() { - String = String + 1; + typeDict[typeof(string)] = typeDict[typeof(string)] + 1; + } + + public string TypeNameConvert(Type type) + { + if (type == typeof(string)) + return "string"; + else if (type == typeof(int)) + return "integer"; + else if (type == typeof(decimal)) + return "number"; + else if (type == typeof(DateTime)) + return "datetame"; + else if (type == typeof(bool)) + return "boolean"; + else + return string.Empty; } } diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 3b3f0d0..2efe457 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -124,6 +124,7 @@ private Schema CreateSchema(string fileName, List props) private List CreateProperties(string filePath) { string[] headers; + List types = new List(); using (var reader = new StreamReader(filePath)) using (var csv = new CsvReader(reader)) { @@ -131,7 +132,7 @@ private List CreateProperties(string filePath) csv.ReadHeader(); headers = csv.Context.HeaderRecord; - List types = new List(); + foreach (var record in csv.GetRecords()) { @@ -155,9 +156,19 @@ private List CreateProperties(string filePath) } List newProps = new List(); - foreach (var header in headers) + foreach (var type in types) { - newProps.Add(new Property { Name = header }); // not including Type yet + int max = 0; + string typeName = string.Empty; + foreach (var myType in type.TypeVotes) + { + if (myType.Value > max) + { + max = myType.Value; + typeName = type.TypeNameConvert(myType.Key); + } + } + newProps.Add(new Property { Name = type.ColumnName, Type = typeName }); } return newProps; From 6dd18fdf5eb93f0cd9cd9caef903d91fddd211b1 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Fri, 27 Sep 2019 16:48:40 -0400 Subject: [PATCH 12/22] gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index ead6c2b..88dafa3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/Debug/netcoreapp3.0/ NaveegoGrpcPlugin/.vs/NaveegoGrpcPlugin/v16/ NaveegoGrpcPlugin/NaveegoGrpcPlugin/obj/ + +NaveegoGrpcPlugin/.vs/NaveegoGrpcPlugin/DesignTimeBuild/.dtbcache + +*.txt From d7d1be6e6ee4575eb3eb64159d01ba094f637b2f Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 14:32:15 -0400 Subject: [PATCH 13/22] simplify Types class --- .../NaveegoGrpcPlugin/Classes/Types.cs | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index ff89e99..cdbb893 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -9,19 +9,12 @@ public class Types { public string ColumnName { get; set; } - public Dictionary TypeVotes { - get - { - return typeDict; - } - } + public Dictionary TypeVotes { get; private set; } private bool typeFound; - private Dictionary typeDict; public Types(string columnName, string value) { - ColumnName = columnName; SetUpTypes(); DetectTypes(value); @@ -29,12 +22,12 @@ public Types(string columnName, string value) private void SetUpTypes() { - typeDict = new Dictionary(); - typeDict.Add(typeof(int), 0); - typeDict.Add(typeof(decimal), 0); - typeDict.Add(typeof(DateTime), 0); - typeDict.Add(typeof(bool), 0); - typeDict.Add(typeof(string), 0); + TypeVotes = new Dictionary(); + TypeVotes.Add(typeof(int), 0); + TypeVotes.Add(typeof(decimal), 0); + TypeVotes.Add(typeof(DateTime), 0); + TypeVotes.Add(typeof(bool), 0); + TypeVotes.Add(typeof(string), 0); } @@ -59,7 +52,7 @@ private void VoteForNumber(string value) decimal.TryParse(value, out decimalCheck); if (decimalCheck != 0) { - typeDict[typeof(decimal)] = typeDict[typeof(decimal)] + 1; + TypeVotes[typeof(decimal)] = TypeVotes[typeof(decimal)] + 1; typeFound = true; } } @@ -70,7 +63,7 @@ private void VoteForInt(string value) int.TryParse(value, out integerCheck); if (integerCheck != 0) { - typeDict[typeof(int)] = typeDict[typeof(int)] + 1; + TypeVotes[typeof(int)] = TypeVotes[typeof(int)] + 1; typeFound = true; } } @@ -82,7 +75,7 @@ private void VoteForDatetime(string value) DateTime.TryParse(value, out datetimeCheck); if (datetimeCheck != DateTime.MinValue) { - typeDict[typeof(DateTime)] = typeDict[typeof(DateTime)] + 1; + TypeVotes[typeof(DateTime)] = TypeVotes[typeof(DateTime)] + 1; typeFound = true; } } @@ -93,14 +86,14 @@ private void VoteForBoolean(string value) bool.TryParse(value, out booleanCheck); if (booleanCheck) { - typeDict[typeof(bool)] = typeDict[typeof(bool)] + 1; + TypeVotes[typeof(bool)] = TypeVotes[typeof(bool)] + 1; typeFound = true; } } private void VoteForString() { - typeDict[typeof(string)] = typeDict[typeof(string)] + 1; + TypeVotes[typeof(string)] = TypeVotes[typeof(string)] + 1; } public string TypeNameConvert(Type type) From 90679189f3820b4bef358a5b3730f4a9dbe85df2 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 15:55:24 -0400 Subject: [PATCH 14/22] converting records to types --- .../NaveegoGrpcPlugin/Classes/Types.cs | 4 ++- .../Services/PluginService.cs | 32 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index cdbb893..cab72b2 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -105,13 +105,15 @@ public string TypeNameConvert(Type type) else if (type == typeof(decimal)) return "number"; else if (type == typeof(DateTime)) - return "datetame"; + return "datetime"; else if (type == typeof(bool)) return "boolean"; else return string.Empty; } + + } diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 2efe457..879314e 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -185,14 +185,19 @@ private async Task GetDataToStream(string filePath, RepeatedField prop foreach (var record in csv.GetRecords()) { - var fullRecord = new List(); + var fullRecord = new List(); + bool isInvalidRecord = false; foreach (KeyValuePair col in record) { - fullRecord.Add(col.Value.ToString()); + string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); + var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); + if (convertedToType == null) + isInvalidRecord = true; + fullRecord.Add(convertedToType); } var data = JsonSerializer.Serialize(fullRecord); - await responseStream.WriteAsync(new PublishRecord { Data = data, Invalid = false }); + await responseStream.WriteAsync(new PublishRecord { Data = data, Invalid = isInvalidRecord }); } } @@ -203,5 +208,26 @@ private async Task GetDataToStream(string filePath, RepeatedField prop } } + public static string NameToTypeConvert(string name) + { + switch (name) + { + case "integer": + return "System.Int32"; + case "number": + return "System.Decimal"; + case "datetime": + return "System.DateTime"; + case "boolean": + return "System.Boolean"; + case "string": + return "System.String"; + default: + return "System.String"; + } + + + } + } } From 3436e03e057f5a8c47cfbcce4c3ade4a5fe0b78d Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 19:51:18 -0400 Subject: [PATCH 15/22] WiP --- .../NaveegoGrpcPlugin/Program.cs | 2 +- .../Services/PluginService.cs | 72 +++++++++++++++++-- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index 713f0ec..f54cb1a 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -21,7 +21,7 @@ public static int Main(string[] args) .MinimumLevel.Information() .MinimumLevel.Override("Microsoft", LogEventLevel.Information) .Enrich.FromLogContext() - //.WriteTo.Console() + // .WriteTo.Console() .WriteTo.File("PluginLog.txt", rollingInterval: RollingInterval.Day) .CreateLogger(); diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 879314e..5c7a0f4 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -10,6 +10,7 @@ using CsvHelper; using Google.Protobuf.Collections; using System.Text.Json; +using System.ComponentModel; namespace NaveegoGrpcPlugin { @@ -17,6 +18,7 @@ public class PluginService : Plugin.Plugin.PluginBase { private readonly ILogger _logger; private List discoveredSchemas; + private string errorMsg; private readonly char delimiter; public PluginService(ILogger logger) { @@ -190,13 +192,24 @@ private async Task GetDataToStream(string filePath, RepeatedField prop foreach (KeyValuePair col in record) { string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); - var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); - if (convertedToType == null) - isInvalidRecord = true; - fullRecord.Add(convertedToType); - } + var canIt = TypeDescriptor.GetConverter(Type.GetType(typeName)); + if (CanConvert(col.Value, Type.GetType(typeName))) + { + var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); + if (convertedToType == null) + isInvalidRecord = true; + fullRecord.Add(convertedToType); + } + else + { + fullRecord.Add(col.Value.ToString()); + } + + + } var data = JsonSerializer.Serialize(fullRecord); + //var newPublishRecord = PrepareRecordForPublish(record, props); await responseStream.WriteAsync(new PublishRecord { Data = data, Invalid = isInvalidRecord }); } @@ -208,6 +221,53 @@ private async Task GetDataToStream(string filePath, RepeatedField prop } } + public PublishRecord PrepareRecordForPublish(dynamic record, RepeatedField props) + { + string data = string.Empty; + bool isInvalidRecord = false; + errorMsg = string.Empty; + var fullRecord = new List(); + + foreach (KeyValuePair col in record) + { + string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); + if (CanConvert(col.Value, Type.GetType(typeName))) + { + var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); + fullRecord.Add(convertedToType); + } + else + { + isInvalidRecord = true; + } + } + + if (isInvalidRecord) + { + fullRecord = null; + } + + data = JsonSerializer.Serialize(fullRecord); + + return new PublishRecord { Data = data, Invalid = isInvalidRecord, Error = errorMsg }; + } + + public bool CanConvert(object objToCast, Type type) + { + try + { + Convert.ChangeType(objToCast, type); + return true; + } + catch (Exception ex) + { + errorMsg = ex.Message; + _logger.LogWarning(ex, "Could not convert to desired data type."); + return false; + } + + } + public static string NameToTypeConvert(string name) { switch (name) @@ -225,8 +285,6 @@ public static string NameToTypeConvert(string name) default: return "System.String"; } - - } } From 9b3ad1e687743bc722308941d3ae63fed2619a97 Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 20:09:40 -0400 Subject: [PATCH 16/22] moved record publish to its own function --- .../NaveegoGrpcPlugin/Program.cs | 2 ++ .../Services/PluginService.cs | 33 ++++--------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index f54cb1a..853f766 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -14,6 +14,7 @@ namespace NaveegoGrpcPlugin { public class Program { + public static int Main(string[] args) { @@ -50,6 +51,7 @@ public static int Main(string[] args) private static void ProcessExitHandler(object sender, EventArgs e) { Console.WriteLine("Shutting down"); + } // Additional configuration is required to successfully run gRPC on macOS. diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 5c7a0f4..5186d00 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -187,37 +187,15 @@ private async Task GetDataToStream(string filePath, RepeatedField prop foreach (var record in csv.GetRecords()) { - var fullRecord = new List(); - bool isInvalidRecord = false; - foreach (KeyValuePair col in record) - { - string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); - - var canIt = TypeDescriptor.GetConverter(Type.GetType(typeName)); - if (CanConvert(col.Value, Type.GetType(typeName))) - { - var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); - if (convertedToType == null) - isInvalidRecord = true; - fullRecord.Add(convertedToType); - } - else - { - fullRecord.Add(col.Value.ToString()); - } - - - } - var data = JsonSerializer.Serialize(fullRecord); - //var newPublishRecord = PrepareRecordForPublish(record, props); - await responseStream.WriteAsync(new PublishRecord { Data = data, Invalid = isInvalidRecord }); + var newPublishRecord = PrepareRecordForPublish(record, props); + await responseStream.WriteAsync(newPublishRecord); } } } - catch (Exception e) + catch (Exception ex) { - _logger.LogError(e, "Error opening csv"); + _logger.LogError(ex, "Error opening csv"); } } @@ -238,13 +216,14 @@ public PublishRecord PrepareRecordForPublish(dynamic record, RepeatedField Date: Sat, 28 Sep 2019 20:42:57 -0400 Subject: [PATCH 17/22] fixed bug not detecting boolean correctly --- NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index cab72b2..88edb85 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -83,8 +83,7 @@ private void VoteForDatetime(string value) private void VoteForBoolean(string value) { bool booleanCheck; - bool.TryParse(value, out booleanCheck); - if (booleanCheck) + if (bool.TryParse(value, out booleanCheck)) { TypeVotes[typeof(bool)] = TypeVotes[typeof(bool)] + 1; typeFound = true; From 232e07dd4e85c04e4b590d47a3ace5de0db84dfe Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 20:43:06 -0400 Subject: [PATCH 18/22] clean up --- .../Services/PluginService.cs | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 5186d00..64ebc91 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -64,9 +64,9 @@ private void LookForFiles(string fileGlob) } } - catch (Exception e) + catch (Exception ex) { - _logger.LogError(e, "Error looking for files in glob"); + _logger.LogError(ex, "Error looking for files in glob"); } } @@ -87,12 +87,10 @@ private void InvestigateFile(string filePath) discoveredSchemas.Add(CreateSchema(filePath, props)); } } - catch (Exception e) + catch (Exception ex) { - _logger.LogError(e, "Error opening csv"); + _logger.LogError(ex, "Error opening csv"); } - - } private Schema CheckForExistingSchema(List props) @@ -125,16 +123,10 @@ private Schema CreateSchema(string fileName, List props) private List CreateProperties(string filePath) { - string[] headers; List types = new List(); using (var reader = new StreamReader(filePath)) using (var csv = new CsvReader(reader)) { - csv.Read(); - csv.ReadHeader(); - - headers = csv.Context.HeaderRecord; - foreach (var record in csv.GetRecords()) { @@ -221,11 +213,6 @@ public PublishRecord PrepareRecordForPublish(dynamic record, RepeatedField Date: Sat, 28 Sep 2019 21:14:18 -0400 Subject: [PATCH 19/22] refactored create properties --- .../Services/PluginService.cs | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 64ebc91..674845f 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -122,6 +122,29 @@ private Schema CreateSchema(string fileName, List props) } private List CreateProperties(string filePath) + { + List scannedColumns = ScanForTypes(filePath); + + List newProps = new List(); + foreach (var column in scannedColumns) + { + int max = 0; + string typeName = string.Empty; + foreach (var candidate in column.TypeVotes) + { + if (candidate.Value > max) + { + max = candidate.Value; + typeName = column.TypeNameConvert(candidate.Key); + } + } + newProps.Add(new Property { Name = column.ColumnName, Type = typeName }); + } + + return newProps; + } + + private static List ScanForTypes(string filePath) { List types = new List(); using (var reader = new StreamReader(filePath)) @@ -129,7 +152,7 @@ private List CreateProperties(string filePath) { foreach (var record in csv.GetRecords()) { - + foreach (KeyValuePair col in record) { var colName = col.Key.ToString(); @@ -143,32 +166,15 @@ private List CreateProperties(string filePath) { types.Add(new Types(colName, field)); } - - } - } - } - - List newProps = new List(); - foreach (var type in types) - { - int max = 0; - string typeName = string.Empty; - foreach (var myType in type.TypeVotes) - { - if (myType.Value > max) - { - max = myType.Value; - typeName = type.TypeNameConvert(myType.Key); } + } - newProps.Add(new Property { Name = type.ColumnName, Type = typeName }); } - return newProps; + return types; } - private async Task GetDataToStream(string filePath, RepeatedField props, IServerStreamWriter responseStream) { try From 58d0b341a581296de540effa3e8308abb548e17a Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sat, 28 Sep 2019 21:14:48 -0400 Subject: [PATCH 20/22] added RFC 3339 compliance --- .../Services/PluginService.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 674845f..7aae77f 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -10,7 +10,7 @@ using CsvHelper; using Google.Protobuf.Collections; using System.Text.Json; -using System.ComponentModel; +using System.Globalization; namespace NaveegoGrpcPlugin { @@ -210,7 +210,15 @@ public PublishRecord PrepareRecordForPublish(dynamic record, RepeatedField Date: Sun, 29 Sep 2019 17:02:38 -0400 Subject: [PATCH 21/22] better logging and cleanup --- .../NaveegoGrpcPlugin/Classes/Types.cs | 2 - .../Services/PluginService.cs | 88 ++++++++++++------- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs index 88edb85..6fc0fe7 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Classes/Types.cs @@ -8,9 +8,7 @@ namespace NaveegoGrpcPlugin public class Types { public string ColumnName { get; set; } - public Dictionary TypeVotes { get; private set; } - private bool typeFound; public Types(string columnName, string value) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs index 7aae77f..545a7ce 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Services/PluginService.cs @@ -29,13 +29,16 @@ public PluginService(ILogger logger) public override Task Discover(DiscoverRequest request, ServerCallContext context) { + _logger.LogInformation("Received request to discover schemas."); var examine = request.Settings.FileGlob; LookForFiles(examine); return Task.FromResult(new DiscoverResponse { Schemas = { discoveredSchemas } }); + } public override async Task Publish(PublishRequest request, IServerStreamWriter responseStream, ServerCallContext context) { + _logger.LogInformation("Received request to publish records."); var filePaths = request.Schema.Settings.Split(delimiter); var props = request.Schema.Properties; foreach (var file in filePaths) @@ -89,7 +92,7 @@ private void InvestigateFile(string filePath) } catch (Exception ex) { - _logger.LogError(ex, "Error opening csv"); + _logger.LogError(ex, "Error investigating file."); } } @@ -126,6 +129,7 @@ private List CreateProperties(string filePath) List scannedColumns = ScanForTypes(filePath); List newProps = new List(); + foreach (var column in scannedColumns) { int max = 0; @@ -144,33 +148,42 @@ private List CreateProperties(string filePath) return newProps; } - private static List ScanForTypes(string filePath) + private List ScanForTypes(string filePath) { List types = new List(); - using (var reader = new StreamReader(filePath)) - using (var csv = new CsvReader(reader)) + try { - foreach (var record in csv.GetRecords()) - { + _logger.LogInformation("Scanning for types on {file}", filePath); - foreach (KeyValuePair col in record) + using (var reader = new StreamReader(filePath)) + using (var csv = new CsvReader(reader)) + { + foreach (var record in csv.GetRecords()) { - var colName = col.Key.ToString(); - var field = col.Value.ToString(); - if (types.Where(w => w.ColumnName == colName).Any()) - { - var colType = types.Where(w => w.ColumnName == colName).First(); - colType.DetectTypes(field); - } - else + + foreach (KeyValuePair col in record) { - types.Add(new Types(colName, field)); + var colName = col.Key.ToString(); + var field = col.Value.ToString(); + if (types.Where(w => w.ColumnName == colName).Any()) + { + var colType = types.Where(w => w.ColumnName == colName).First(); + colType.DetectTypes(field); + } + else + { + types.Add(new Types(colName, field)); + } + } } - } } + catch (Exception ex) + { + _logger.LogError(ex, "Error while scanning CSVs."); + } return types; } @@ -179,6 +192,7 @@ private async Task GetDataToStream(string filePath, RepeatedField prop { try { + _logger.LogInformation("Publishing records for {file}", filePath); using (var reader = new StreamReader(filePath)) using (var csv = new CsvReader(reader)) { @@ -193,7 +207,7 @@ private async Task GetDataToStream(string filePath, RepeatedField prop } catch (Exception ex) { - _logger.LogError(ex, "Error opening csv"); + _logger.LogError(ex, "Error publishing records."); } } @@ -204,30 +218,38 @@ public PublishRecord PrepareRecordForPublish(dynamic record, RepeatedField(); - foreach (KeyValuePair col in record) + try { - string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); - if (CanConvert(col.Value, Type.GetType(typeName))) + foreach (KeyValuePair col in record) { - var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); - if(convertedToType.GetType() == typeof(DateTime)) + string typeName = NameToTypeConvert(props.Where(w => w.Name == col.Key.ToString()).Select(s => s.Type).FirstOrDefault()); + if (CanConvert(col.Value, Type.GetType(typeName))) { - fullRecord.Add(ToRfc3339String((DateTime)convertedToType)); + var convertedToType = Convert.ChangeType(col.Value, Type.GetType(typeName)); + if(convertedToType.GetType() == typeof(DateTime)) + { + fullRecord.Add(ToRfc3339String((DateTime)convertedToType)); + } + else + { + fullRecord.Add(convertedToType); + } + } else { - fullRecord.Add(convertedToType); + fullRecord.Add(null); + isInvalidRecord = true; } - } - else - { - fullRecord.Add(null); - isInvalidRecord = true; - } - } - data = JsonSerializer.Serialize(fullRecord); + data = JsonSerializer.Serialize(fullRecord); + + } + catch (Exception ex) + { + _logger.LogError(ex, "Error preparing to publish record."); + } return new PublishRecord { Data = data, Invalid = isInvalidRecord, Error = errorMsg }; } From 83ee9349f0593911f1ac658d05c4f7a93b17d4aa Mon Sep 17 00:00:00 2001 From: Rob Newling Date: Sun, 29 Sep 2019 17:26:46 -0400 Subject: [PATCH 22/22] remove unneeded comments --- NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs | 3 --- NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs | 4 ---- 2 files changed, 7 deletions(-) diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs index 853f766..8551477 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Program.cs @@ -22,7 +22,6 @@ public static int Main(string[] args) .MinimumLevel.Information() .MinimumLevel.Override("Microsoft", LogEventLevel.Information) .Enrich.FromLogContext() - // .WriteTo.Console() .WriteTo.File("PluginLog.txt", rollingInterval: RollingInterval.Day) .CreateLogger(); @@ -54,8 +53,6 @@ private static void ProcessExitHandler(object sender, EventArgs e) } - // Additional configuration is required to successfully run gRPC on macOS. - // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 public static IHostBuilder CreateHostBuilder(string[] args, IConfiguration configuration) => Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => diff --git a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs index d78be90..e48315a 100644 --- a/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs +++ b/NaveegoGrpcPlugin/NaveegoGrpcPlugin/Startup.cs @@ -19,14 +19,11 @@ public Startup(IConfiguration configuration) { Configuration = configuration; } - // This method gets called by the runtime. Use this method to add services to the container. - // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 public void ConfigureServices(IServiceCollection services) { services.AddGrpc(); } - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime applicationLifetime) { if (env.IsDevelopment()) @@ -49,7 +46,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApp applicationLifetime.ApplicationStarted.Register(ShowPort); - } private void ShowPort()