Modules: Robust loader and plugin system (#249)

* Start work on a less buggy, more flexible module system

* IPlatformInfo: Use Stone 7.0.0 schema

* ExtensibilityLoader: Isolate CoreService through a ServiceContainer requiring IPluginContainer to specify exactly which services are required.

* StoneProvider: Refactor StoneProvider into a support module

* Loader: Refactor some names

* StoreProviders: In order to prevent databases from being recreated, scope database service creation to a privileged composable to register them as first-class services.

* PluginManager: Begin to refactor PluginManager into a much higher level API

* Plugin: Introduce the Provision-Plugin Plugin Manager

* Plugins: Allow non-provisioned, standalone plugins that do not have a provisioned content folder, can not access module references, and can be loaded without requesting a provision from the plugin manager.

* Cleanup: Move Loader into Framework

* AssemblyModuleLoader: Catch and rethrow the TypeLoadException

* Logger: Setup output on first use.

* Plugin: Make composable

* IModule: Add ContentsDirectory as a standard directory.

* Snowball: Delete obsolete Snowball packaging

* Tooling: Add a module packager to publish plugins as a module.

* Snowflake: Various cleanups

* AssemblyModulePacker: Multi-target 1.0 and 1.1, remove dependency on CliWrap

* Support: Add Pack CLI tool to all support assemblies

* Dependencies: Consolidate and update dependencies

* Runtime: Update to build against .NET Standard to remove preliminary dependencies.

* KeyedImageCache: Refactor out into a service to avoid a dependency on System.Drawing in the framework.

* AssemblyModulePacker: Support netcoreapp2.0

* Build: Fix build on newest VS Preview

* Logging: Move logging implementation into Snowflake.Framework.Services

* TargetFramework: Retarget to netcoreapp2.0 to shrink bundle size

* AssemblyModulePacker: Target only netcoreapp2.0

* Utility: Refactor out persistence implementations to reduce bundle size of projects that import Snowflake.Framework

Also includes Force.Crc32 into source code.

* Travis: Build on 2.0.0-preview1-005977

* Tests: Fix test build

* Tests: Use Snowflake.Persistence

* Shiragame: Refactor Shiragame out into a support service

* Cleanup

* ShiragameProvider: fix module metadata

* Library: Use a tuple instead of an anonymous object

* Library: Add some DLR casts to work around funky behaviour on Linux

* Appveyor: Build on Visual Studio 2016 Preview image

* Tests: Fix xunit warnings

* Plugin: Implement the configuration API.

* Extensibility: Implement the configuration API

* PluginProvision: Expose access to a configuration store to retrieve plugin configuration.

* ICoreService: Rename to IServiceContainer

* Force.Crc32: Add tests and license declaration

* Tests: Add a bunch of tests

* Tests: Write lots of loader tests

* Tests: Fix tests that violate the different-assembly policy

* StructuredFilename: Add tests

* Tests: Test plugin

* Test: Test JsonPluginProperties

.travis.yml

@@ -2,7 +2,7 @@ language: csharp
 sudo: required
 solution: Snowflake.sln
 mono: none
-dotnet: 1.0.1
+dotnet: 2.0.0-preview1-005977
 dist: trusty 
 install:
   - dotnet restore src/Snowflake.sln

appveyor.yml

@@ -1,5 +1,5 @@
 version: 0.2.{build}
-os: Visual Studio 2017
+os: Visual Studio 2017 Preview
 branches:
   # whitelist
   only:

src/Snowflake.Bootstrap.Windows/Snowflake.Bootstrap.Windows.csproj

@@ -1,16 +1,22 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp1.1</TargetFramework>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
   </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Microsoft.NETCore.App" Version="2.0.0-preview1-002111-00" />
+  </ItemGroup>
+
 
   <ItemGroup>
+    <ProjectReference Include="..\Snowflake.Framework\Snowflake.Framework.csproj" />
     <ProjectReference Include="..\Snowflake.Plugin.Emulators.RetroArch\Snowflake.Plugin.Emulators.RetroArch.csproj" />
     <ProjectReference Include="..\Snowflake.Framework.Primitives\Snowflake.Framework.Primitives.csproj" />
     <ProjectReference Include="..\Snowflake.Framework.Services\Snowflake.Framework.Services.csproj" />
     <ProjectReference Include="..\Snowflake.Framework.Utility\Snowflake.Framework.Utility.csproj" />
-    <ProjectReference Include="..\Snowflake.Framework\Snowflake.Framework.csproj" />
   </ItemGroup>
 
+
 </Project>

src/Snowflake.Bootstrap.Windows/SnowflakeShell.cs

@@ -17,12 +17,15 @@
 using Snowflake.Services;
 using Snowflake.Utility;
 using Snowflake.Plugin.Emulators.RetroArch.Adapters.Bsnes;
+using Snowflake.Romfile;
+using Snowflake.Loader;
+using Snowflake.Services.AssemblyLoader;
 
 namespace Snowflake.Shell.Windows
 {
     internal class SnowflakeShell
     {
-        private ICoreService loadedCore;
+        private IServiceContainer loadedCore;
         private readonly string appDataDirectory = PathUtility.GetSnowflakeDataPath();
         internal SnowflakeShell()
         {
@@ -32,17 +35,20 @@ internal SnowflakeShell()
         public void StartCore()
         {
 
-            this.loadedCore = new CoreService(this.appDataDirectory);
+            this.loadedCore = new ServiceContainer(this.appDataDirectory);
+            var loader = this.loadedCore.Get<IModuleEnumerator>();
+            var composer = new AssemblyComposer(this.loadedCore, loader);
+            composer.Compose();
             //this.loadedCore.Get<IEmulatorAssembliesManager>()?.LoadEmulatorAssemblies();
-            this.loadedCore.Get<IPluginManager>()?.Initialize();
+            //this.loadedCore.Get<IPluginManager>()?.Initialize();
            /* this.loadedCore.Get<IServerManager>().RegisterServer("ThemeServer", new ThemeServer(Path.Combine(this.loadedCore.AppDataDirectory, "themes")));
             foreach (string serverName in this.loadedCore.Get<IServerManager>().RegisteredServers)
             {
                 this.loadedCore.Get<IServerManager>()?.StartServer(serverName);
                 var serverStartEvent = new ServerStartEventArgs(this.loadedCore, serverName);
                 SnowflakeEventManager.EventSource.RaiseEvent(serverStartEvent); //todo Move event registration to SnowflakeEVentManager
             }*/
-            var im = this.loadedCore.Get<IPluginManager>().Get<IInputEnumerator>()["InputEnumerator-Keyboard"];
+            /*var im = this.loadedCore.Get<IPluginManager>().Get<IInputEnumerator>()["InputEnumerator-Keyboard"];
             var ep = new EmulatedPort(1, im.ControllerLayout,
                 im.GetConnectedDevices().First(), MappedControllerElementCollection.GetDefaultMappings(im.ControllerLayout, this.loadedCore.Get<IStoneProvider>().Controllers["NES_CONTROLLER"]));
             var gr = new GameRecord(this.loadedCore.Get<IStoneProvider>().Platforms["NINTENDO_SNES"], "test",
@@ -51,7 +57,7 @@ public void StartCore()
             
             var lmfao = raadapter.Instantiate(gr, gr.Files[0], 0, new List<IEmulatedPort> { ep});
             lmfao.Create();
-            lmfao.Start();
+            lmfao.Start();*/
         }
 
         public void StartShell() {

src/Snowflake.Framework.Persistence/Snowflake.Framework.Persistence.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.SQLite" Version="2.0.0-preview1-final" />
+    <PackageReference Include="Dapper" Version="1.50.3-beta1" />
+    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Snowflake.Framework.Primitives\Snowflake.Framework.Primitives.csproj" />
+  </ItemGroup>
+
+</Project>

src/Snowflake.Framework.Utility/SqliteDatabase.cs → src/Snowflake.Framework.Persistence/SqliteDatabase.cs

@@ -7,22 +7,23 @@
 using System.Text;
 using Microsoft.Data.Sqlite;
 using Dapper;
+using Snowflake.Persistence;
 
-namespace Snowflake.Utility
+namespace Snowflake.Persistence
 {
-    public class SqliteDatabase
+    public class SqliteDatabase : ISqlDatabase
     {
-        public string FileName { get; }
+        public string DatabaseName { get; }
         private readonly string dbConnectionString;
 
         public SqliteDatabase(string fileName)
         {
-            this.FileName = fileName;
-            if(!File.Exists(this.FileName)) File.Create(this.FileName).Dispose();
-            this.dbConnectionString = $"Data Source={this.FileName};";
+            this.DatabaseName = fileName;
+            if(!File.Exists(this.DatabaseName)) File.Create(this.DatabaseName).Dispose();
+            this.dbConnectionString = $"Data Source={this.DatabaseName};";
         }
 
-        public SqliteConnection GetConnection()
+        public IDbConnection GetConnection()
         {
             var connection = new SqliteConnection(this.dbConnectionString);
             connection.Open();
@@ -46,10 +47,10 @@ public SqliteConnection GetConnection()
         /// <typeparam name="T">The type the query will return</typeparam>
         /// <param name="queryFunction">A function to query the database using the opened connection</param>
         /// <returns>The requested data.</returns>
-        public T Query<T>(Func<DbConnection, T> queryFunction)
+        public T Query<T>(Func<IDbConnection, T> queryFunction)
         {
             T record;
-            using (SqliteConnection dbConnection = this.GetConnection())
+            using (var dbConnection = this.GetConnection())
             {
                 dbConnection.Open();
                 record = queryFunction(dbConnection);
@@ -63,9 +64,9 @@ record = queryFunction(dbConnection);
         /// The connection will be safely closed after the operation.
         /// </summary>
         /// <param name="queryFunction">A function to query the database using the opened connection</param>
-        public void Execute(Action<DbConnection> queryFunction)
+        public void Execute(Action<IDbConnection> queryFunction)
         {
-            using (SqliteConnection dbConnection = this.GetConnection())
+            using (var dbConnection = this.GetConnection())
             {
                 dbConnection.Open();
                 queryFunction(dbConnection);

src/Snowflake.Framework.Utility/SqliteKeyValueStore.cs → src/Snowflake.Framework.Persistence/SqliteKeyValueStore.cs

@@ -7,7 +7,7 @@
 using System.Threading.Tasks;
 using Newtonsoft.Json;
 
-namespace Snowflake.Utility
+namespace Snowflake.Persistence
 {
     //todo use simple execute api
 

src/Snowflake.Framework.Utility/SqliteMemoryDatabase.cs → src/Snowflake.Framework.Persistence/SqliteMemoryDatabase.cs

@@ -8,7 +8,7 @@
 using Dapper;
 using Microsoft.Data.Sqlite;
 
-namespace Snowflake.Utility
+namespace Snowflake.Persistence
 {
     public class SqliteMemoryDatabase
     {

src/Snowflake.Framework.Primitives/Caching/IKeyedImageCache.cs

@@ -4,7 +4,8 @@
 using System.Text;
 using System.Threading.Tasks;
 using Snowflake.Records.File;
-using System.Drawing;
+using System.IO;
+
 namespace Snowflake.Caching
 {
     /// <summary>
@@ -16,21 +17,21 @@ public interface IKeyedImageCache
         /// Adds an image to the image cache, generating resized image/jpeg versions of the files, and
         /// returning a list of file records.
         /// </summary>
-        /// <param name="image">The image to add</param>
+        /// <param name="imageStream">The image to add as a stream</param>
         /// <param name="recordGuid">A record to link to</param>
         /// <param name="imageType">The type of image See <see cref="ImageTypes"/> for recognized image types</param>
         /// <returns>The generated file records with appropriate metadata linking it to the cache folder</returns>
-        IList<IFileRecord> Add(Image image, Guid recordGuid, string imageType);
+        IList<IFileRecord> Add(Stream imageStream, Guid recordGuid, string imageType);
 
         /// <summary>
         /// Adds an image to the image cache, generating resized image/jpeg versions of the files, and
         /// returning a list of file records.
         /// </summary>
-        /// <param name="image">The image to add</param>
+        /// <param name="imageStream">The image to add as a stream</param>
         /// <param name="recordGuid">A record to link to</param>
         /// <param name="imageType">The type of image See <see cref="ImageTypes"/> for recognized image types</param>
         /// <param name="dateTime">A date time to link it to.</param>
         /// <returns>The generated file records with appropriate metadata linking it to the cache folder</returns>
-        IList<IFileRecord> Add(Image image, Guid recordGuid, string imageType, DateTime dateTime);
+        IList<IFileRecord> Add(Stream imageStream, Guid recordGuid, string imageType, DateTime dateTime);
     }
 }

src/Snowflake.Framework.Primitives/Configuration/IConfigurationOption.cs

@@ -18,20 +18,24 @@ public interface IConfigurationOption
         /// A description of this configuration option
         /// </summary>
         string Description { get; }
+
         /// <summary>
         /// Whether or not this option is a simple option (displayed in "Simple" configuration mode)
         /// </summary>
         bool Simple { get; }
+
         /// <summary>
         /// Whether or not this option is private (not ever displayed to the user)
         /// </summary>
         bool Private { get; }
+
         /// <summary>
         /// A 'flag' property is never serialized into the configuration option, and is instead used to cause
         /// side effects to the configuration during emulator instance creation by the emulator handler.
         /// If a flag affects the configuration, it should be placed in the same section it modifies.
         /// </summary>
         bool Flag { get; }
+
         /// <summary>
         /// The maximum value allowable for a number value
         /// </summary>

src/Snowflake.Framework.Primitives/Emulator/IBiosManager.cs

@@ -9,7 +9,7 @@ namespace Snowflake.Emulator
 {
     /// <summary>
     /// Manages BIOS files for multiple platforms.
-    /// 
+    /// todo: Validate MD5?
     /// </summary>
     public interface IBiosManager
     {

src/Snowflake.Framework.Primitives/Emulator/IEmulatorAdapter.cs

@@ -29,7 +29,7 @@ public interface IEmulatorAdapter : IPlugin
         /// </para>
         /// <para>
         /// The general convention for these files are
-        /// InputApi.DEVICE_NAME
+        /// common/InputMapping/DEVICE_NAME
         /// </para>
         /// </summary>
         IEnumerable<IInputMapping> InputMappings { get; }

src/Snowflake.Framework.Primitives/Extensibility/Configuration/IPluginConfigurationStore.cs

@@ -0,0 +1,37 @@
+using Snowflake.Configuration;
+using System;
+
+namespace Snowflake.Extensibility.Configuration
+{
+    /// <summary>
+    /// Represents a store that can save and retrieve an aribtrary configuration 
+    /// collection representing a single emulator configuration file, associated with a game record
+    /// </summary>
+    /// <remarks>
+    /// To "delete" a configuration, just overwrite the existing values with a default instance
+    /// </remarks>
+    public interface IPluginConfigurationStore
+    {
+        /// <summary>
+        /// Retrieves the configuration collection associated with this game record.
+        /// This method is guaranteed to return a usable instance of the configuration collection. 
+        /// If a prior configuration has not been set, it should return a default instance with all
+        /// properties initialized.
+        /// </summary>
+        /// <typeparam name="T">The type of configuration collection</typeparam>
+        IConfigurationSection<T> Get<T>() where T: class, IConfigurationSection<T>;
+
+        /// <summary>
+        /// Saves and persists a configuration collection to the store.
+        /// </summary>
+        /// <param name="configuration">The configuration to save to the store</param>
+        void Set<T>(IConfigurationSection<T> configuration) where T: class, IConfigurationSection<T>;
+
+        /// <summary>
+        /// Updates a single <em>existing</em> configuration value, this will error if the GUID is not found in
+        /// the database.
+        /// </summary>
+        /// <param name="value">The configuration value with valid GUID and updated data</param>
+        void Set(IConfigurationValue value);
+    }
+}