From bbc118d9c07eb5df598726038009c7e15bea0cfb Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 15:32:43 -0700 Subject: [PATCH 01/21] Add possibility to exclude some grain types in assembly loading --- src/Orleans/Configuration/NodeConfiguration.cs | 6 +++++- .../GrainTypeManager/SiloAssemblyLoader.cs | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Orleans/Configuration/NodeConfiguration.cs b/src/Orleans/Configuration/NodeConfiguration.cs index d3ae789dfb..66a29a8e52 100644 --- a/src/Orleans/Configuration/NodeConfiguration.cs +++ b/src/Orleans/Configuration/NodeConfiguration.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Net.Sockets; using System.Text; @@ -212,6 +213,8 @@ public string TraceFilePattern public Dictionary AdditionalAssemblyDirectories { get; set; } + public List ExcludedGrainTypes { get; set; } + public string SiloShutdownEventName { get; set; } internal const string DEFAULT_NODE_NAME = "default"; @@ -270,6 +273,7 @@ public NodeConfiguration() UseNagleAlgorithm = false; AdditionalAssemblyDirectories = new Dictionary(); + ExcludedGrainTypes = new List(); } public NodeConfiguration(NodeConfiguration other) @@ -320,6 +324,7 @@ public NodeConfiguration(NodeConfiguration other) StartupTypeName = other.StartupTypeName; AdditionalAssemblyDirectories = other.AdditionalAssemblyDirectories; + ExcludedGrainTypes = other.ExcludedGrainTypes.ToList(); } public override string ToString() @@ -487,7 +492,6 @@ internal void Load(XmlElement root) case "AdditionalAssemblyDirectories": ConfigUtilities.ParseAdditionalAssemblyDirectories(AdditionalAssemblyDirectories, child); break; - } } } diff --git a/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs b/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs index f35c582997..591012d1f6 100644 --- a/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs +++ b/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs @@ -13,17 +13,21 @@ namespace Orleans.Runtime { internal class SiloAssemblyLoader { + private readonly List excludedGrains; private readonly LoggerImpl logger = LogManager.GetLogger("AssemblyLoader.Silo"); private List discoveredAssemblyLocations; private Dictionary directories; public SiloAssemblyLoader(NodeConfiguration nodeConfig) - : this(nodeConfig.AdditionalAssemblyDirectories) + : this(nodeConfig.AdditionalAssemblyDirectories, nodeConfig.ExcludedGrainTypes) { } - public SiloAssemblyLoader(IDictionary additionalDirectories) + public SiloAssemblyLoader(IDictionary additionalDirectories, IEnumerable excludedGrains = null) { + this.excludedGrains = excludedGrains != null + ? new List(excludedGrains) + : new List(); var exeRoot = Path.GetDirectoryName(typeof(SiloAssemblyLoader).GetTypeInfo().Assembly.Location); var appRoot = Path.Combine(exeRoot, "Applications"); var cwd = Directory.GetCurrentDirectory(); @@ -80,6 +84,9 @@ private void LoadApplicationAssemblies() foreach (var grainType in grainTypes) { var className = TypeUtils.GetFullName(grainType); + if (excludedGrains.Contains(className)) + continue; + if (result.ContainsKey(className)) throw new InvalidOperationException( string.Format("Precondition violated: GetLoadedGrainTypes should not return a duplicate type ({0})", className)); From f793f2f4b5c05b7de5d1b0b7f7fd98af704b513d Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Mon, 12 Sep 2016 14:23:05 -0700 Subject: [PATCH 02/21] Add test for excluded grains --- .../HeterogeneousTests.cs | 45 +++++++++++++++++++ test/Tester/Tester.csproj | 1 + 2 files changed, 46 insertions(+) create mode 100644 test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs diff --git a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs new file mode 100644 index 0000000000..20f73e9dc6 --- /dev/null +++ b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs @@ -0,0 +1,45 @@ +using System; +using System.Linq; +using System.Threading; +using Orleans; +using Orleans.Runtime; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests.GrainInterfaces; +using UnitTests.Grains; +using Xunit; + +namespace Tester.HeterogeneousSilosTests +{ + public class HeterogeneousTests : OrleansTestingBase, IDisposable + { + private TestCluster cluster; + + private void SetupAndDeployCluster(params Type[] blackListedTypes) + { + var typesName = blackListedTypes.Select(t => t.FullName).ToList(); + var options = new TestClusterOptions(1); + options.ClusterConfiguration.Overrides[Silo.PrimarySiloName].ExcludedGrainTypes = typesName; + cluster = new TestCluster(options); + cluster.Deploy(); + } + + public void Dispose() + { + cluster?.StopAllSilos(); + } + + [Fact] + public void GrainExcludedTest() + { + SetupAndDeployCluster(typeof(TestGrain)); + + // Should fail + var exception = Assert.Throws(() => GrainFactory.GetGrain(0)); + Assert.Contains("Cannot find an implementation class for grain interface", exception.Message); + + // Should not fail + GrainFactory.GetGrain(0); + } + } +} diff --git a/test/Tester/Tester.csproj b/test/Tester/Tester.csproj index b2df34b8fc..fc2bef456f 100644 --- a/test/Tester/Tester.csproj +++ b/test/Tester/Tester.csproj @@ -48,6 +48,7 @@ + From 568cf00a71fd2445cad541800d3e85cad93688ec Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 16:10:54 -0700 Subject: [PATCH 03/21] Add method to merge two GrainInterfaceMap --- src/Orleans/Runtime/GrainInterfaceMap.cs | 41 +++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/Orleans/Runtime/GrainInterfaceMap.cs b/src/Orleans/Runtime/GrainInterfaceMap.cs index 2395780411..a57ccb4347 100644 --- a/src/Orleans/Runtime/GrainInterfaceMap.cs +++ b/src/Orleans/Runtime/GrainInterfaceMap.cs @@ -67,7 +67,6 @@ public override string ToString() private readonly Dictionary table; private readonly HashSet unordered; - [NonSerialized] private readonly Dictionary implementationIndex; [NonSerialized] // Client shouldn't need this @@ -91,6 +90,38 @@ public GrainInterfaceMap(bool localTestMode, PlacementStrategy defaultPlacementS loadedGrainAsemblies = new HashSet(); } + internal void AddMap(GrainInterfaceMap map) + { + foreach (var kvp in map.typeToInterfaceData) + { + if (!typeToInterfaceData.ContainsKey(kvp.Key)) + { + typeToInterfaceData.Add(kvp.Key, kvp.Value); + } + } + + foreach (var kvp in map.table) + { + if (!table.ContainsKey(kvp.Key)) + { + table.Add(kvp.Key, kvp.Value); + } + } + + foreach (var grainClassTypeCode in map.unordered) + { + unordered.Add(grainClassTypeCode); + } + + foreach (var kvp in map.implementationIndex) + { + if (!implementationIndex.ContainsKey(kvp.Key)) + { + implementationIndex.Add(kvp.Key, kvp.Value); + } + } + } + internal void AddEntry(int interfaceId, Type iface, int grainTypeCode, string grainInterface, string grainClass, string assembly, bool isGenericGrainClass, PlacementStrategy placement, MultiClusterRegistrationStrategy registrationStrategy, bool primaryImplementation = false) { @@ -173,6 +204,14 @@ internal bool ContainsGrainInterface(int interfaceId) } } + internal bool ContainsGrainImplementation(int typeCode) + { + lock (this) + { + return implementationIndex.ContainsKey(typeCode); + } + } + internal bool TryGetTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy registrationStrategy, string genericArguments = null) { lock (this) From 4bdc77a67bb928b01beb08e0627806ec0ebf0bc5 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 17:19:16 -0700 Subject: [PATCH 04/21] GrainTypeManager store the cluster GrainInterfaceMap and is able to rebuild it --- .../GrainTypeManager/GrainTypeManager.cs | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index 466693920a..7985a9532f 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net; @@ -13,18 +14,25 @@ namespace Orleans.Runtime internal class GrainTypeManager { private IDictionary grainTypes; + private IDictionary grainInterfaceMapsBySilo; private readonly Logger logger = LogManager.GetLogger("GrainTypeManager"); private readonly GrainInterfaceMap grainInterfaceMap; private readonly Dictionary invokers = new Dictionary(); private readonly SiloAssemblyLoader loader; private static readonly object lockable = new object(); + private readonly PlacementStrategy defaultPlacementStrategy; - private readonly PlacementStrategy defaultPlacementStrategy; + internal IDictionary GrainInterfaceMapsBySilo + { + get { return new Dictionary(grainInterfaceMapsBySilo); } + } public static GrainTypeManager Instance { get; private set; } public IEnumerable> GrainClassTypeData { get { return grainTypes; } } + public GrainInterfaceMap ClusterGrainInterfaceMap { get; private set; } + public static void Stop() { Instance = null; @@ -40,6 +48,8 @@ public GrainTypeManager(bool localTestMode, SiloAssemblyLoader loader, DefaultPl this.defaultPlacementStrategy = defaultPlacementStrategy.PlacementStrategy; this.loader = loader; grainInterfaceMap = new GrainInterfaceMap(localTestMode, this.defaultPlacementStrategy); + ClusterGrainInterfaceMap = grainInterfaceMap; + grainInterfaceMapsBySilo = new Dictionary(); lock (lockable) { if (Instance != null) @@ -134,10 +144,25 @@ internal bool TryGetPrimaryImplementation(string grainInterface, out string grai internal void GetTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy activationStrategy, string genericArguments = null) { - if (!grainInterfaceMap.TryGetTypeInfo(typeCode, out grainClass, out placement, out activationStrategy, genericArguments)) + if (!ClusterGrainInterfaceMap.TryGetTypeInfo(typeCode, out grainClass, out placement, out activationStrategy, genericArguments)) throw new OrleansException(String.Format("Unexpected: Cannot find an implementation class for grain interface {0}", typeCode)); } + internal void SetInterfaceMapsBySilo(IDictionary value) + { + grainInterfaceMapsBySilo = value; + RebuildFullGrainInterfaceMap(); + } + + internal IList GetSupportedSilos(int typeCode) + { + // TODO more efficient way... + return (from interfaceMap in grainInterfaceMapsBySilo + where interfaceMap.Value.ContainsGrainImplementation(typeCode) + select interfaceMap.Key) + .ToList(); + } + private void InitializeGrainClassData(SiloAssemblyLoader loader, bool strict) { grainTypes = loader.GetGrainClassTypes(strict); @@ -241,6 +266,17 @@ internal IGrainMethodInvoker GetInvoker(int interfaceId, string genericGrainType interfaceName, interfaceId)); } + private void RebuildFullGrainInterfaceMap() + { + var newClusterGrainInterfaceMap = new GrainInterfaceMap(false, defaultPlacementStrategy); + newClusterGrainInterfaceMap.AddMap(grainInterfaceMap); + foreach (var map in grainInterfaceMapsBySilo.Values) + { + newClusterGrainInterfaceMap.AddMap(map); + } + ClusterGrainInterfaceMap = newClusterGrainInterfaceMap; + } + private class InvokerData { private readonly Type baseInvokerType; From 3aab7546326f6ba7418daa96f4cd2ae3af754162 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Fri, 4 Nov 2016 14:13:10 -0700 Subject: [PATCH 05/21] Better implementation of GrainTypeManager.GetSupportedSilos --- src/Orleans/Runtime/GrainInterfaceMap.cs | 9 ++++++-- .../GrainTypeManager/GrainTypeManager.cs | 23 +++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Orleans/Runtime/GrainInterfaceMap.cs b/src/Orleans/Runtime/GrainInterfaceMap.cs index a57ccb4347..bf23cec350 100644 --- a/src/Orleans/Runtime/GrainInterfaceMap.cs +++ b/src/Orleans/Runtime/GrainInterfaceMap.cs @@ -74,18 +74,23 @@ public override string ToString() private readonly bool localTestMode; private readonly HashSet loadedGrainAsemblies; + + private readonly PlacementStrategy defaultPlacementStrategy; - private readonly PlacementStrategy defaultPlacementStrategy; + internal IList SupportedGrainTypes + { + get { return implementationIndex.Keys.ToList(); } + } public GrainInterfaceMap(bool localTestMode, PlacementStrategy defaultPlacementStrategy) { - this.defaultPlacementStrategy = defaultPlacementStrategy; table = new Dictionary(); typeToInterfaceData = new Dictionary(); primaryImplementations = new Dictionary(); implementationIndex = new Dictionary(); unordered = new HashSet(); this.localTestMode = localTestMode; + this.defaultPlacementStrategy = defaultPlacementStrategy; if(localTestMode) // if we are running in test mode, we'll build a list of loaded grain assemblies to help with troubleshooting deployment issue loadedGrainAsemblies = new HashSet(); } diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index 7985a9532f..f681964cc2 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Net; using System.Reflection; @@ -15,6 +16,7 @@ internal class GrainTypeManager { private IDictionary grainTypes; private IDictionary grainInterfaceMapsBySilo; + private Dictionary> supportedSilosByTypeCode; private readonly Logger logger = LogManager.GetLogger("GrainTypeManager"); private readonly GrainInterfaceMap grainInterfaceMap; private readonly Dictionary invokers = new Dictionary(); @@ -156,11 +158,7 @@ internal void SetInterfaceMapsBySilo(IDictionary internal IList GetSupportedSilos(int typeCode) { - // TODO more efficient way... - return (from interfaceMap in grainInterfaceMapsBySilo - where interfaceMap.Value.ContainsGrainImplementation(typeCode) - select interfaceMap.Key) - .ToList(); + return supportedSilosByTypeCode[typeCode]; } private void InitializeGrainClassData(SiloAssemblyLoader loader, bool strict) @@ -269,12 +267,23 @@ internal IGrainMethodInvoker GetInvoker(int interfaceId, string genericGrainType private void RebuildFullGrainInterfaceMap() { var newClusterGrainInterfaceMap = new GrainInterfaceMap(false, defaultPlacementStrategy); + var newSupportedSilosByTypeCode = new Dictionary>(); newClusterGrainInterfaceMap.AddMap(grainInterfaceMap); - foreach (var map in grainInterfaceMapsBySilo.Values) + foreach (var kvp in grainInterfaceMapsBySilo) { - newClusterGrainInterfaceMap.AddMap(map); + newClusterGrainInterfaceMap.AddMap(kvp.Value); + foreach (var grainType in kvp.Value.SupportedGrainTypes) + { + IList supportedSilos; + if (!newSupportedSilosByTypeCode.TryGetValue(grainType, out supportedSilos)) + { + newSupportedSilosByTypeCode[grainType] = supportedSilos = new List(); + } + supportedSilos.Add(kvp.Key); + } } ClusterGrainInterfaceMap = newClusterGrainInterfaceMap; + supportedSilosByTypeCode = newSupportedSilosByTypeCode; } private class InvokerData From 777331378e59912342e80c7da26f6ba3e65579ef Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 16:29:48 -0700 Subject: [PATCH 06/21] Add/rename method to return the TypeCodeMap for the whole cluster and an other to explicitely return the local TypeCodeMap for the silo in TypeManager --- src/Orleans/Messaging/ProxiedMessageCenter.cs | 6 +++--- src/Orleans/Runtime/ITypeManager.cs | 17 +++++++++++++++-- .../GrainTypeManager/GrainTypeManager.cs | 2 +- .../GrainTypeManager/TypeManager.cs | 9 +++++++-- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/Orleans/Messaging/ProxiedMessageCenter.cs b/src/Orleans/Messaging/ProxiedMessageCenter.cs index 842f446d0d..39d04c7079 100644 --- a/src/Orleans/Messaging/ProxiedMessageCenter.cs +++ b/src/Orleans/Messaging/ProxiedMessageCenter.cs @@ -278,7 +278,7 @@ private void RejectOrResend(Message msg) public Task GetTypeCodeMap(GrainFactory grainFactory) { var silo = GetLiveGatewaySiloAddress(); - return GetTypeManager(silo, grainFactory).GetTypeCodeMap(silo); + return GetTypeManager(silo, grainFactory).GetClusterTypeCodeMap(); } public Task GetImplicitStreamSubscriberTable(GrainFactory grainFactory) @@ -397,9 +397,9 @@ public int ReceiveQueueLength #endregion - private ITypeManager GetTypeManager(SiloAddress destination, GrainFactory grainFactory) + private IClusterTypeManager GetTypeManager(SiloAddress destination, GrainFactory grainFactory) { - return grainFactory.GetSystemTarget(Constants.TypeManagerId, destination); + return grainFactory.GetSystemTarget(Constants.TypeManagerId, destination); } private SiloAddress GetLiveGatewaySiloAddress() diff --git a/src/Orleans/Runtime/ITypeManager.cs b/src/Orleans/Runtime/ITypeManager.cs index c60fed9982..ce126b9e69 100644 --- a/src/Orleans/Runtime/ITypeManager.cs +++ b/src/Orleans/Runtime/ITypeManager.cs @@ -6,10 +6,23 @@ namespace Orleans.Runtime /// /// Client gateway interface for obtaining the grain interface/type map. /// - internal interface ITypeManager : ISystemTarget + internal interface IClusterTypeManager : ISystemTarget { - Task GetTypeCodeMap(SiloAddress silo); + /// + /// Acquires grain interface map for all grain types supported across the entire cluster + /// + /// + Task GetClusterTypeCodeMap(); Task GetImplicitStreamSubscriberTable(SiloAddress silo); } + + internal interface ISiloTypeManager : ISystemTarget + { + /// + /// Acquires grain interface map for all grain types supported by hosted silo. + /// + /// + Task GetSiloTypeCodeMap(); + } } diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index f681964cc2..90b9367c81 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -219,7 +219,7 @@ public bool TryGetData(string name, out GrainTypeData result) return grainTypes.TryGetValue(name, out result); } - internal IGrainTypeResolver GetTypeCodeMap() + internal GrainInterfaceMap GetTypeCodeMap() { // the map is immutable at this point return grainInterfaceMap; diff --git a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs index 758b4483d7..f658366640 100644 --- a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs @@ -4,7 +4,7 @@ namespace Orleans.Runtime { - internal class TypeManager : SystemTarget, ITypeManager + internal class TypeManager : SystemTarget, IClusterTypeManager, ISiloTypeManager { private readonly GrainTypeManager grainTypeManager; @@ -15,7 +15,12 @@ internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager) } - public Task GetTypeCodeMap(SiloAddress silo) + public Task GetClusterTypeCodeMap() + { + return Task.FromResult(grainTypeManager.ClusterGrainInterfaceMap); + } + + public Task GetSiloTypeCodeMap() { return Task.FromResult(grainTypeManager.GetTypeCodeMap()); } From a259b319262f26026ccc96d099f79d9e5c522477 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 17:30:38 -0700 Subject: [PATCH 07/21] TypeManager refresh the cluster GrainInterfaceMapTimer every 500ms --- src/Orleans/Logging/ErrorCodes.cs | 2 + .../GrainTypeManager/TypeManager.cs | 85 ++++++++++++++++++- src/OrleansRuntime/Silo/Silo.cs | 11 ++- 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/Orleans/Logging/ErrorCodes.cs b/src/Orleans/Logging/ErrorCodes.cs index 9a6ec47def..2d923ed0a8 100644 --- a/src/Orleans/Logging/ErrorCodes.cs +++ b/src/Orleans/Logging/ErrorCodes.cs @@ -1093,6 +1093,8 @@ internal enum ErrorCode GlobalSingleInstance_MaintainerException = GlobalSingleInstanceBase + 3, GlobalSingleInstance_MultipleOwners = GlobalSingleInstanceBase + 4, + TypeManagerBase = Runtime + 4200, + TypeManager_GetSiloGrainInterfaceMapError = TypeManagerBase + 1, } } // ReSharper restore InconsistentNaming diff --git a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs index f658366640..6889518bcb 100644 --- a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs @@ -1,17 +1,38 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Orleans.Runtime.Providers; +using Orleans.Runtime.Scheduler; namespace Orleans.Runtime { internal class TypeManager : SystemTarget, IClusterTypeManager, ISiloTypeManager { + private readonly Logger logger = LogManager.GetLogger("TypeManager"); private readonly GrainTypeManager grainTypeManager; + private readonly ISiloStatusOracle statusOracle; + private readonly OrleansTaskScheduler scheduler; + private readonly AsyncTaskSafeTimer refreshClusterGrainInterfaceMapTimer; - internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager) + internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISiloStatusOracle oracle, OrleansTaskScheduler scheduler) : base(Constants.TypeManagerId, myAddr) { + if (grainTypeManager == null) + throw new ArgumentNullException(nameof(grainTypeManager)); + if (oracle == null) + throw new ArgumentNullException(nameof(oracle)); + if (scheduler == null) + throw new ArgumentNullException(nameof(scheduler)); + this.grainTypeManager = grainTypeManager; + statusOracle = oracle; + this.scheduler = scheduler; + this.refreshClusterGrainInterfaceMapTimer = new AsyncTaskSafeTimer( + OnRefreshClusterMapTimer, + null, + TimeSpan.Zero, // Force to do it once right now + TimeSpan.FromMilliseconds(500)); // TODO Config } @@ -34,6 +55,68 @@ public Task GetImplicitStreamSubscriberTa } return Task.FromResult(table); } + + private async Task OnRefreshClusterMapTimer(object _) + { + + logger.Info("OnRefreshClusterMapTimer: refresh start"); + var activeSilos = statusOracle.GetApproximateSiloStatuses(onlyActive: true); + var knownSilosClusterGrainInterfaceMap = grainTypeManager.GrainInterfaceMapsBySilo; + + // Build the new map. Always start by himself + var newSilosClusterGrainInterfaceMap = new Dictionary + { + {this.Silo, grainTypeManager.GetTypeCodeMap()} + }; + var getGrainInterfaceMapTasks = new List>>(); + + + foreach (var siloAddress in activeSilos.Keys) + { + if (siloAddress.Equals(this.Silo)) continue; + + GrainInterfaceMap value; + if (knownSilosClusterGrainInterfaceMap.TryGetValue(siloAddress, out value)) + { + logger.Verbose3($"OnRefreshClusterMapTimer: value already found locally for {siloAddress}"); + newSilosClusterGrainInterfaceMap[siloAddress] = value; + } + else + { + // Value not found, let's get it + logger.Verbose3($"OnRefreshClusterMapTimer: value not found locally for {siloAddress}"); + getGrainInterfaceMapTasks.Add(GetTargetSiloGrainInterfaceMap(siloAddress)); + } + } + + if (getGrainInterfaceMapTasks.Any()) + { + foreach (var keyValuePair in await Task.WhenAll(getGrainInterfaceMapTasks)) + { + if (keyValuePair.Value != null) + newSilosClusterGrainInterfaceMap.Add(keyValuePair.Key, keyValuePair.Value); + } + } + + grainTypeManager.SetInterfaceMapsBySilo(newSilosClusterGrainInterfaceMap); + } + + private async Task> GetTargetSiloGrainInterfaceMap(SiloAddress siloAddress) + { + try + { + var remoteTypeManager = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget(Constants.TypeManagerId, siloAddress); + var siloTypeCodeMap = await scheduler.QueueTask(() => remoteTypeManager.GetSiloTypeCodeMap(), SchedulingContext); + return new KeyValuePair(siloAddress, siloTypeCodeMap); + } + catch (Exception ex) + { + // Will be retried on the next timer hit + logger.Error(ErrorCode.TypeManager_GetSiloGrainInterfaceMapError, $"Exception when trying to get GrainInterfaceMap for silos {siloAddress}", ex); + + return new KeyValuePair(siloAddress, null); + } + } } } diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 339d5fd497..fb33e38773 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -60,7 +60,8 @@ public enum SiloType private readonly IncomingMessageAgent incomingSystemAgent; private readonly IncomingMessageAgent incomingPingAgent; private readonly Logger logger; - private readonly GrainTypeManager typeManager; + private readonly GrainTypeManager grainTypeManager; + private TypeManager typeManager; private readonly ManualResetEvent siloTerminatedEvent; private readonly SiloStatisticsManager siloStatistics; private readonly MembershipFactory membershipFactory; @@ -93,6 +94,7 @@ public enum SiloType internal GlobalConfiguration GlobalConfig => this.initializationParams.GlobalConfig; internal NodeConfiguration LocalConfig => this.initializationParams.NodeConfig; internal OrleansTaskScheduler LocalScheduler { get { return scheduler; } } + internal GrainTypeManager LocalGrainTypeManager { get { return grainTypeManager; } } internal ILocalGrainDirectory LocalGrainDirectory { get { return localGrainDirectory; } } internal ISiloStatusOracle LocalSiloStatusOracle { get { return membershipOracle; } } internal IMultiClusterOracle LocalMultiClusterOracle { get { return multiClusterOracle; } } @@ -304,7 +306,7 @@ internal Silo(SiloInitializationParameters initializationParams) throw; } - typeManager = Services.GetRequiredService(); + grainTypeManager = Services.GetRequiredService(); // Performance metrics siloStatistics = Services.GetRequiredService(); @@ -381,7 +383,8 @@ private void CreateSystemTargets() logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager"); this.RegisterSystemTarget(this.Services.GetRequiredService()); - this.RegisterSystemTarget(new TypeManager(this.SiloAddress, this.typeManager)); + typeManager = new TypeManager(SiloAddress, LocalGrainTypeManager, membershipOracle, LocalScheduler); + RegisterSystemTarget(typeManager); logger.Verbose("Creating {0} System Target", "MembershipOracle"); if (this.membershipOracle is SystemTarget) @@ -474,7 +477,7 @@ private void DoStart() ConfigureThreadPoolAndServicePointSettings(); // This has to start first so that the directory system target factory gets loaded before we start the router. - typeManager.Start(); + grainTypeManager.Start(); runtimeClient.Start(); // The order of these 4 is pretty much arbitrary. From adf6ea1272a6ff20c211a716be3a861b302b411d Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 16:50:59 -0700 Subject: [PATCH 08/21] TypeManager implements ISiloStatusListener and refresh cluster only if needed --- .../GrainTypeManager/TypeManager.cs | 18 ++++++++++++++++-- src/OrleansRuntime/Silo/Silo.cs | 2 ++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs index 6889518bcb..d2ca635f90 100644 --- a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs @@ -7,12 +7,13 @@ namespace Orleans.Runtime { - internal class TypeManager : SystemTarget, IClusterTypeManager, ISiloTypeManager + internal class TypeManager : SystemTarget, IClusterTypeManager, ISiloTypeManager, ISiloStatusListener { private readonly Logger logger = LogManager.GetLogger("TypeManager"); private readonly GrainTypeManager grainTypeManager; private readonly ISiloStatusOracle statusOracle; private readonly OrleansTaskScheduler scheduler; + private bool hasToRefreshClusterGrainInterfaceMap; private readonly AsyncTaskSafeTimer refreshClusterGrainInterfaceMapTimer; internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISiloStatusOracle oracle, OrleansTaskScheduler scheduler) @@ -28,6 +29,7 @@ internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISil this.grainTypeManager = grainTypeManager; statusOracle = oracle; this.scheduler = scheduler; + hasToRefreshClusterGrainInterfaceMap = true; this.refreshClusterGrainInterfaceMapTimer = new AsyncTaskSafeTimer( OnRefreshClusterMapTimer, null, @@ -56,8 +58,20 @@ public Task GetImplicitStreamSubscriberTa return Task.FromResult(table); } + public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status) + { + hasToRefreshClusterGrainInterfaceMap = true; + } + private async Task OnRefreshClusterMapTimer(object _) { + // Check if we have to refresh + if (!hasToRefreshClusterGrainInterfaceMap) + { + logger.Verbose3("OnRefreshClusterMapTimer: no refresh required"); + return; + } + hasToRefreshClusterGrainInterfaceMap = false; logger.Info("OnRefreshClusterMapTimer: refresh start"); var activeSilos = statusOracle.GetApproximateSiloStatuses(onlyActive: true); @@ -113,7 +127,7 @@ private async Task OnRefreshClusterMapTimer(object _) { // Will be retried on the next timer hit logger.Error(ErrorCode.TypeManager_GetSiloGrainInterfaceMapError, $"Exception when trying to get GrainInterfaceMap for silos {siloAddress}", ex); - + hasToRefreshClusterGrainInterfaceMap = true; return new KeyValuePair(siloAddress, null); } } diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index fb33e38773..49f18577c8 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -419,6 +419,8 @@ private void InjectDependencies() // consistentRingProvider is not a system target per say, but it behaves like the localGrainDirectory, so it is here LocalSiloStatusOracle.SubscribeToSiloStatusEvents((ISiloStatusListener)RingProvider); + LocalSiloStatusOracle.SubscribeToSiloStatusEvents(typeManager); + LocalSiloStatusOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService()); if (!GlobalConfig.ReminderServiceType.Equals(GlobalConfiguration.ReminderServiceProviderType.Disabled)) From 83057de48bb360a87a09a711f490e1d78338ace7 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 17:56:32 -0700 Subject: [PATCH 09/21] Add a method in IPlacementContext to return a list of compatible silos given a GrainId --- src/OrleansRuntime/Catalog/Catalog.cs | 10 ++++++++++ src/OrleansRuntime/Placement/IPlacementContext.cs | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/OrleansRuntime/Catalog/Catalog.cs b/src/OrleansRuntime/Catalog/Catalog.cs index 3157aa60ef..f5a617af8d 100644 --- a/src/OrleansRuntime/Catalog/Catalog.cs +++ b/src/OrleansRuntime/Catalog/Catalog.cs @@ -119,6 +119,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont public GrainTypeManager GrainTypeManager { get; private set; } + public SiloAddress LocalSilo { get; private set; } internal ISiloStatusOracle SiloStatusOracle { get; set; } internal readonly ActivationCollector ActivationCollector; @@ -200,6 +201,15 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont /// public Dispatcher Dispatcher { get; } + public IList GetCompatibleSiloList(GrainId grain) + { + var typeCode = grain.GetTypeCode(); + var compatibleSilos = GrainTypeManager.GetSupportedSilos(typeCode).Intersect(AllActiveSilos).ToList(); + if (compatibleSilos.Count == 0) + throw new OrleansException($"TypeCode ${typeCode} not supported in the cluster"); + return compatibleSilos; + } + internal void SetStorageManager(IStorageProviderManager storageManager) { storageProviderManager = storageManager; diff --git a/src/OrleansRuntime/Placement/IPlacementContext.cs b/src/OrleansRuntime/Placement/IPlacementContext.cs index f6133f4856..1ae8dd160f 100644 --- a/src/OrleansRuntime/Placement/IPlacementContext.cs +++ b/src/OrleansRuntime/Placement/IPlacementContext.cs @@ -24,6 +24,8 @@ internal interface IPlacementContext List AllActiveSilos { get; } + IList GetCompatibleSiloList(GrainId grain); + SiloAddress LocalSilo { get; } SiloStatus LocalSiloStatus { get; } @@ -70,6 +72,16 @@ public static string GetGrainTypeName(this IPlacementContext @this, int typeCode return grainClass; } + public static string GetGrainTypeNameAndSupportedSilos(this IPlacementContext @this, GrainId grainId, out IList supportedSiloAddresses, string genericArguments = null) + { + string grainClass; + PlacementStrategy unused; + MultiClusterRegistrationStrategy unusedActivationStrategy; + @this.GetGrainTypeInfo(grainId.GetTypeCode(), out grainClass, out unused, out unusedActivationStrategy, genericArguments); + supportedSiloAddresses = @this.GetCompatibleSiloList(grainId); + return grainClass; + } + public static string GetGrainTypeName(this IPlacementContext @this, GrainId grainId, string genericArguments = null) { return @this.GetGrainTypeName(grainId.GetTypeCode(), genericArguments); From 77e80d83bfdf4852418c83289a36a1b18f0c1e54 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 18:00:44 -0700 Subject: [PATCH 10/21] Update PlacementDirectors to use only compatible silos --- .../Placement/ActivationCountPlacementDirector.cs | 4 +++- src/OrleansRuntime/Placement/IPlacementContext.cs | 2 -- src/OrleansRuntime/Placement/PreferLocalPlacementDirector.cs | 4 ++-- src/OrleansRuntime/Placement/RandomPlacementDirector.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/OrleansRuntime/Placement/ActivationCountPlacementDirector.cs b/src/OrleansRuntime/Placement/ActivationCountPlacementDirector.cs index ececdd2c65..57de373970 100644 --- a/src/OrleansRuntime/Placement/ActivationCountPlacementDirector.cs +++ b/src/OrleansRuntime/Placement/ActivationCountPlacementDirector.cs @@ -89,11 +89,13 @@ private Task MakePlacement(PlacementStrategy strategy, GrainId public Task SelectSiloPowerOfK(PlacementStrategy strategy, GrainId grain, IPlacementContext context) { - // Exclude overloaded silos + var compatibleSilos = context.GetCompatibleSiloList(grain); + // Exclude overloaded and non-compatible silos var relevantSilos = new List(); foreach (CachedLocalStat current in localCache.Values) { if (IsSiloOverloaded(current.SiloStats)) continue; + if (!compatibleSilos.Contains(current.Address)) continue; relevantSilos.Add(current); } diff --git a/src/OrleansRuntime/Placement/IPlacementContext.cs b/src/OrleansRuntime/Placement/IPlacementContext.cs index 1ae8dd160f..107010b46f 100644 --- a/src/OrleansRuntime/Placement/IPlacementContext.cs +++ b/src/OrleansRuntime/Placement/IPlacementContext.cs @@ -22,8 +22,6 @@ internal interface IPlacementContext bool LocalLookup(GrainId grain, out List addresses); - List AllActiveSilos { get; } - IList GetCompatibleSiloList(GrainId grain); SiloAddress LocalSilo { get; } diff --git a/src/OrleansRuntime/Placement/PreferLocalPlacementDirector.cs b/src/OrleansRuntime/Placement/PreferLocalPlacementDirector.cs index a4b87cfe0b..9c80dfd674 100644 --- a/src/OrleansRuntime/Placement/PreferLocalPlacementDirector.cs +++ b/src/OrleansRuntime/Placement/PreferLocalPlacementDirector.cs @@ -15,8 +15,8 @@ internal class PreferLocalPlacementDirector : RandomPlacementDirector, IPlacemen public override Task OnAddActivation(PlacementStrategy strategy, GrainId grain, IPlacementContext context) { - // if local silo is not active, revert to random placement - if (context.LocalSiloStatus != SiloStatus.Active) + // if local silo is not active or does not support this type of grain, revert to random placement + if (context.LocalSiloStatus != SiloStatus.Active || !context.GetCompatibleSiloList(grain).Contains(context.LocalSilo)) return base.OnAddActivation(strategy, grain, context); var grainType = context.GetGrainTypeName(grain); diff --git a/src/OrleansRuntime/Placement/RandomPlacementDirector.cs b/src/OrleansRuntime/Placement/RandomPlacementDirector.cs index 9d5153382b..48573b70fb 100644 --- a/src/OrleansRuntime/Placement/RandomPlacementDirector.cs +++ b/src/OrleansRuntime/Placement/RandomPlacementDirector.cs @@ -41,7 +41,7 @@ protected PlacementResult ChooseRandomActivation(List places, PlacementStrategy strategy, GrainId grain, IPlacementContext context) { var grainType = context.GetGrainTypeName(grain); - var allSilos = context.AllActiveSilos; + var allSilos = context.GetCompatibleSiloList(grain); return Task.FromResult( PlacementResult.SpecifyCreation(allSilos[random.Next(allSilos.Count)], strategy, grainType)); } From 6c5133f965d1fc39d581e926cf981f9b2723d990 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 27 Oct 2016 18:03:11 -0700 Subject: [PATCH 11/21] Add test for heterogeneous silos --- .../HeterogeneousTests.cs | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs index 20f73e9dc6..1d3ee92e76 100644 --- a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs +++ b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Orleans; using Orleans.Runtime; using Orleans.TestingHost; @@ -15,10 +16,12 @@ public class HeterogeneousTests : OrleansTestingBase, IDisposable { private TestCluster cluster; - private void SetupAndDeployCluster(params Type[] blackListedTypes) + private void SetupAndDeployCluster(string defaultPlacementStrategy, params Type[] blackListedTypes) { + cluster?.StopAllSilos(); var typesName = blackListedTypes.Select(t => t.FullName).ToList(); var options = new TestClusterOptions(1); + options.ClusterConfiguration.Globals.DefaultPlacementStrategy = defaultPlacementStrategy; options.ClusterConfiguration.Overrides[Silo.PrimarySiloName].ExcludedGrainTypes = typesName; cluster = new TestCluster(options); cluster.Deploy(); @@ -27,12 +30,13 @@ private void SetupAndDeployCluster(params Type[] blackListedTypes) public void Dispose() { cluster?.StopAllSilos(); + cluster = null; } [Fact] public void GrainExcludedTest() { - SetupAndDeployCluster(typeof(TestGrain)); + SetupAndDeployCluster("RandomPlacement", typeof(TestGrain)); // Should fail var exception = Assert.Throws(() => GrainFactory.GetGrain(0)); @@ -41,5 +45,54 @@ public void GrainExcludedTest() // Should not fail GrainFactory.GetGrain(0); } + + + [Fact] + public async Task MergeGrainResolverTests() + { + await MergeGrainResolverTestsImpl("RandomPlacement", typeof(TestGrain)); + await MergeGrainResolverTestsImpl("PreferLocalPlacement", typeof(TestGrain)); + await MergeGrainResolverTestsImpl("ActivationCountBasedPlacement", typeof(TestGrain)); + } + + private async Task MergeGrainResolverTestsImpl(string defaultPlacementStrategy, params Type[] blackListedTypes) + { + SetupAndDeployCluster(defaultPlacementStrategy, blackListedTypes); + + // Should fail + var exception = Assert.Throws(() => GrainFactory.GetGrain(0)); + Assert.Contains("Cannot find an implementation class for grain interface", exception.Message); + + // Start a new silo with TestGrain + cluster.StartAdditionalSilo(); + Thread.Sleep(1000); + + // Disconnect/Reconnect the client + GrainClient.Uninitialize(); + cluster.InitializeClient(); + + for (var i = 0; i < 5; i++) + { + // Success + var g = GrainFactory.GetGrain(i); + await g.SetLabel("Hello world"); + } + + // Stop the latest silos + cluster.StopSecondarySilos(); + Thread.Sleep(1000); + + var grain = GrainFactory.GetGrain(0); + var orleansException = await Assert.ThrowsAsync(() => grain.SetLabel("Hello world")); + Assert.Contains("Cannot find an implementation class for grain interface", orleansException.Message); + + // Disconnect/Reconnect the client + GrainClient.Uninitialize(); + cluster.InitializeClient(); + + // Should fail + exception = Assert.Throws(() => GrainFactory.GetGrain(0)); + Assert.Contains("Cannot find an implementation class for grain interface", exception.Message); + } } } From 7cca65da194a59327361f9b82153e27ed636ae72 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 17 Nov 2016 15:52:56 -0800 Subject: [PATCH 12/21] Add config for TypeMapRefreshTimeout value --- src/Orleans/Configuration/GlobalConfiguration.cs | 6 ++++++ src/OrleansRuntime/GrainTypeManager/TypeManager.cs | 4 ++-- src/OrleansRuntime/Silo/Silo.cs | 5 +++-- test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs | 2 ++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Orleans/Configuration/GlobalConfiguration.cs b/src/Orleans/Configuration/GlobalConfiguration.cs index 365f4f7905..bc2258f984 100644 --- a/src/Orleans/Configuration/GlobalConfiguration.cs +++ b/src/Orleans/Configuration/GlobalConfiguration.cs @@ -170,6 +170,10 @@ public bool PrimaryNodeIsRequired /// The number of seconds to attempt to join a cluster of silos before giving up. /// public TimeSpan MaxJoinAttemptTime { get; set; } + /// + /// The number of seconds to refresh the cluster grain interface map + /// + public TimeSpan TypeMapRefreshTimeout { get; set; } internal ConfigValue ExpectedClusterSizeConfigValue { get; set; } /// /// The expected size of a cluster. Need not be very accurate, can be an overestimate. @@ -470,6 +474,7 @@ internal bool UseAzureSystemStore private static readonly TimeSpan DEFAULT_LIVENESS_DEATH_VOTE_EXPIRATION_TIMEOUT = TimeSpan.FromSeconds(120); private static readonly TimeSpan DEFAULT_LIVENESS_I_AM_ALIVE_TABLE_PUBLISH_TIMEOUT = TimeSpan.FromMinutes(5); private static readonly TimeSpan DEFAULT_LIVENESS_MAX_JOIN_ATTEMPT_TIME = TimeSpan.FromMinutes(5); // 5 min + private static readonly TimeSpan DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME = TimeSpan.FromMinutes(1); private const int DEFAULT_LIVENESS_NUM_MISSED_PROBES_LIMIT = 3; private const int DEFAULT_LIVENESS_NUM_PROBED_SILOS = 3; private const int DEFAULT_LIVENESS_NUM_VOTES_FOR_DEATH_DECLARATION = 2; @@ -522,6 +527,7 @@ internal GlobalConfiguration() UseLivenessGossip = DEFAULT_LIVENESS_USE_LIVENESS_GOSSIP; ValidateInitialConnectivity = DEFAULT_VALIDATE_INITIAL_CONNECTIVITY; MaxJoinAttemptTime = DEFAULT_LIVENESS_MAX_JOIN_ATTEMPT_TIME; + TypeMapRefreshTimeout = DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME; MaxMultiClusterGateways = DEFAULT_MAX_MULTICLUSTER_GATEWAYS; BackgroundGossipInterval = DEFAULT_BACKGROUND_GOSSIP_INTERVAL; UseGlobalSingleInstanceByDefault = DEFAULT_USE_GLOBAL_SINGLE_INSTANCE; diff --git a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs index d2ca635f90..b168c1c3e7 100644 --- a/src/OrleansRuntime/GrainTypeManager/TypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/TypeManager.cs @@ -16,7 +16,7 @@ internal class TypeManager : SystemTarget, IClusterTypeManager, ISiloTypeManager private bool hasToRefreshClusterGrainInterfaceMap; private readonly AsyncTaskSafeTimer refreshClusterGrainInterfaceMapTimer; - internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISiloStatusOracle oracle, OrleansTaskScheduler scheduler) + internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISiloStatusOracle oracle, OrleansTaskScheduler scheduler, TimeSpan refreshClusterMapTimeout) : base(Constants.TypeManagerId, myAddr) { if (grainTypeManager == null) @@ -34,7 +34,7 @@ internal TypeManager(SiloAddress myAddr, GrainTypeManager grainTypeManager, ISil OnRefreshClusterMapTimer, null, TimeSpan.Zero, // Force to do it once right now - TimeSpan.FromMilliseconds(500)); // TODO Config + refreshClusterMapTimeout); } diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 49f18577c8..d417121090 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -382,9 +382,10 @@ private void CreateSystemTargets() RegisterSystemTarget(LocalGrainDirectory.RemoteClusterGrainDirectory); logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager"); + this.RegisterSystemTarget(this.Services.GetRequiredService()); - typeManager = new TypeManager(SiloAddress, LocalGrainTypeManager, membershipOracle, LocalScheduler); - RegisterSystemTarget(typeManager); + typeManager = new TypeManager(SiloAddress, this.grainTypeManager, membershipOracle, LocalScheduler, GlobalConfig.TypeMapRefreshTimeout); + this.RegisterSystemTarget(typeManager); logger.Verbose("Creating {0} System Target", "MembershipOracle"); if (this.membershipOracle is SystemTarget) diff --git a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs index 1d3ee92e76..e44161eff9 100644 --- a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs +++ b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs @@ -15,12 +15,14 @@ namespace Tester.HeterogeneousSilosTests public class HeterogeneousTests : OrleansTestingBase, IDisposable { private TestCluster cluster; + private readonly TimeSpan refreshTimeout = TimeSpan.FromMilliseconds(200); private void SetupAndDeployCluster(string defaultPlacementStrategy, params Type[] blackListedTypes) { cluster?.StopAllSilos(); var typesName = blackListedTypes.Select(t => t.FullName).ToList(); var options = new TestClusterOptions(1); + options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = refreshTimeout; options.ClusterConfiguration.Globals.DefaultPlacementStrategy = defaultPlacementStrategy; options.ClusterConfiguration.Overrides[Silo.PrimarySiloName].ExcludedGrainTypes = typesName; cluster = new TestCluster(options); From bde9f2f319f9f8f277f3b8eec8a90ccca22bafcd Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 17 Nov 2016 16:03:26 -0800 Subject: [PATCH 13/21] Reduce waiting time in MergeGrainResolverTests --- test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs index e44161eff9..0bc25f1f53 100644 --- a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs +++ b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs @@ -61,13 +61,15 @@ private async Task MergeGrainResolverTestsImpl(string defaultPlacementStrategy, { SetupAndDeployCluster(defaultPlacementStrategy, blackListedTypes); + var delayTimeout = refreshTimeout.Add(refreshTimeout); + // Should fail var exception = Assert.Throws(() => GrainFactory.GetGrain(0)); Assert.Contains("Cannot find an implementation class for grain interface", exception.Message); // Start a new silo with TestGrain cluster.StartAdditionalSilo(); - Thread.Sleep(1000); + await Task.Delay(delayTimeout); // Disconnect/Reconnect the client GrainClient.Uninitialize(); @@ -82,7 +84,7 @@ private async Task MergeGrainResolverTestsImpl(string defaultPlacementStrategy, // Stop the latest silos cluster.StopSecondarySilos(); - Thread.Sleep(1000); + await Task.Delay(delayTimeout); var grain = GrainFactory.GetGrain(0); var orleansException = await Assert.ThrowsAsync(() => grain.SetLabel("Hello world")); From d619e3eb72dd9582873a33c5c84b8e07cba84e05 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Wed, 23 Nov 2016 10:56:56 -0800 Subject: [PATCH 14/21] Fix ExceptionPropagationTests.ExceptionPropagationGrainToGrainMessageBodyDeserializationFailure --- test/Tester/ExceptionPropagationTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Tester/ExceptionPropagationTests.cs b/test/Tester/ExceptionPropagationTests.cs index b75d3287e0..a8b5bf388c 100644 --- a/test/Tester/ExceptionPropagationTests.cs +++ b/test/Tester/ExceptionPropagationTests.cs @@ -33,6 +33,7 @@ protected override TestCluster CreateTestCluster() var options = new TestClusterOptions(2); options.ClientConfiguration.SerializationProviders.Add(typeof(OneWaySerializer).GetTypeInfo()); options.ClusterConfiguration.Globals.SerializationProviders.Add(typeof(OneWaySerializer).GetTypeInfo()); + options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(200); return new TestCluster(options); } } From 8b054d938af8c6cebb777e0705910cde3b6a0cf9 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 1 Dec 2016 10:40:31 -0800 Subject: [PATCH 15/21] Fix SilosStopTests --- test/Tester/MembershipTests/SilosStopTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Tester/MembershipTests/SilosStopTests.cs b/test/Tester/MembershipTests/SilosStopTests.cs index 0702928409..c65b30ea2e 100644 --- a/test/Tester/MembershipTests/SilosStopTests.cs +++ b/test/Tester/MembershipTests/SilosStopTests.cs @@ -18,6 +18,7 @@ public override TestCluster CreateTestCluster() options.ClusterConfiguration.Globals.DefaultPlacementStrategy = "ActivationCountBasedPlacement"; options.ClusterConfiguration.Globals.NumMissedProbesLimit = 1; options.ClusterConfiguration.Globals.NumVotesForDeathDeclaration = 1; + options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(100); // use only Primary as the gateway options.ClientConfiguration.Gateways = options.ClientConfiguration.Gateways.Take(1).ToList(); From 25cd773ca47377980bdef95373a659893d25c4f4 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Thu, 1 Dec 2016 10:41:52 -0800 Subject: [PATCH 16/21] Fix ElasticPlacementTests --- test/TesterInternal/General/ElasticPlacementTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/TesterInternal/General/ElasticPlacementTest.cs b/test/TesterInternal/General/ElasticPlacementTest.cs index 9f1b2ed026..3532be60d4 100644 --- a/test/TesterInternal/General/ElasticPlacementTest.cs +++ b/test/TesterInternal/General/ElasticPlacementTest.cs @@ -32,6 +32,7 @@ public override TestCluster CreateTestCluster() options.ClusterConfiguration.Globals.LivenessType = GlobalConfiguration.LivenessProviderType.AzureTable; options.ClientConfiguration.GatewayProvider = ClientConfiguration.GatewayProviderType.AzureTable; + options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(100); return new TestCluster(options); } From fc940b69233dadf6bf76a5758d5707a14f2a0aa5 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Tue, 6 Dec 2016 11:54:50 -0800 Subject: [PATCH 17/21] Use IReadOnlyDictionary for GrainInterfaceMapsBySilo in GrainTypeManager to avoid allocation --- src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index 90b9367c81..3da39e5772 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -15,7 +15,7 @@ namespace Orleans.Runtime internal class GrainTypeManager { private IDictionary grainTypes; - private IDictionary grainInterfaceMapsBySilo; + private IReadOnlyDictionary grainInterfaceMapsBySilo; private Dictionary> supportedSilosByTypeCode; private readonly Logger logger = LogManager.GetLogger("GrainTypeManager"); private readonly GrainInterfaceMap grainInterfaceMap; @@ -24,9 +24,9 @@ internal class GrainTypeManager private static readonly object lockable = new object(); private readonly PlacementStrategy defaultPlacementStrategy; - internal IDictionary GrainInterfaceMapsBySilo + internal IReadOnlyDictionary GrainInterfaceMapsBySilo { - get { return new Dictionary(grainInterfaceMapsBySilo); } + get { return grainInterfaceMapsBySilo; } } public static GrainTypeManager Instance { get; private set; } @@ -150,7 +150,7 @@ internal void GetTypeInfo(int typeCode, out string grainClass, out PlacementStra throw new OrleansException(String.Format("Unexpected: Cannot find an implementation class for grain interface {0}", typeCode)); } - internal void SetInterfaceMapsBySilo(IDictionary value) + internal void SetInterfaceMapsBySilo(IReadOnlyDictionary value) { grainInterfaceMapsBySilo = value; RebuildFullGrainInterfaceMap(); From 8dd74221e7c8cd67a111a89e93dd230e31444d85 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Tue, 6 Dec 2016 13:43:37 -0800 Subject: [PATCH 18/21] Rename TypeMapRefreshTimeout to TypeMapRefreshInterval for better clarity --- src/Orleans/Configuration/GlobalConfiguration.cs | 4 ++-- src/OrleansRuntime/Silo/Silo.cs | 2 +- test/Tester/ExceptionPropagationTests.cs | 2 +- test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs | 6 +++--- test/Tester/MembershipTests/SilosStopTests.cs | 2 +- test/TesterInternal/General/ElasticPlacementTest.cs | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Orleans/Configuration/GlobalConfiguration.cs b/src/Orleans/Configuration/GlobalConfiguration.cs index bc2258f984..d576e011c4 100644 --- a/src/Orleans/Configuration/GlobalConfiguration.cs +++ b/src/Orleans/Configuration/GlobalConfiguration.cs @@ -173,7 +173,7 @@ public bool PrimaryNodeIsRequired /// /// The number of seconds to refresh the cluster grain interface map /// - public TimeSpan TypeMapRefreshTimeout { get; set; } + public TimeSpan TypeMapRefreshInterval { get; set; } internal ConfigValue ExpectedClusterSizeConfigValue { get; set; } /// /// The expected size of a cluster. Need not be very accurate, can be an overestimate. @@ -527,7 +527,7 @@ internal GlobalConfiguration() UseLivenessGossip = DEFAULT_LIVENESS_USE_LIVENESS_GOSSIP; ValidateInitialConnectivity = DEFAULT_VALIDATE_INITIAL_CONNECTIVITY; MaxJoinAttemptTime = DEFAULT_LIVENESS_MAX_JOIN_ATTEMPT_TIME; - TypeMapRefreshTimeout = DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME; + TypeMapRefreshInterval = DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME; MaxMultiClusterGateways = DEFAULT_MAX_MULTICLUSTER_GATEWAYS; BackgroundGossipInterval = DEFAULT_BACKGROUND_GOSSIP_INTERVAL; UseGlobalSingleInstanceByDefault = DEFAULT_USE_GLOBAL_SINGLE_INSTANCE; diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index d417121090..7a092f6159 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -384,7 +384,7 @@ private void CreateSystemTargets() logger.Verbose("Creating {0} System Target", "ClientObserverRegistrar + TypeManager"); this.RegisterSystemTarget(this.Services.GetRequiredService()); - typeManager = new TypeManager(SiloAddress, this.grainTypeManager, membershipOracle, LocalScheduler, GlobalConfig.TypeMapRefreshTimeout); + typeManager = new TypeManager(SiloAddress, this.grainTypeManager, membershipOracle, LocalScheduler, GlobalConfig.TypeMapRefreshInterval); this.RegisterSystemTarget(typeManager); logger.Verbose("Creating {0} System Target", "MembershipOracle"); diff --git a/test/Tester/ExceptionPropagationTests.cs b/test/Tester/ExceptionPropagationTests.cs index a8b5bf388c..1fb9995ab9 100644 --- a/test/Tester/ExceptionPropagationTests.cs +++ b/test/Tester/ExceptionPropagationTests.cs @@ -33,7 +33,7 @@ protected override TestCluster CreateTestCluster() var options = new TestClusterOptions(2); options.ClientConfiguration.SerializationProviders.Add(typeof(OneWaySerializer).GetTypeInfo()); options.ClusterConfiguration.Globals.SerializationProviders.Add(typeof(OneWaySerializer).GetTypeInfo()); - options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(200); + options.ClusterConfiguration.Globals.TypeMapRefreshInterval = TimeSpan.FromMilliseconds(200); return new TestCluster(options); } } diff --git a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs index 0bc25f1f53..063a06d233 100644 --- a/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs +++ b/test/Tester/HeterogeneousSilosTests/HeterogeneousTests.cs @@ -15,14 +15,14 @@ namespace Tester.HeterogeneousSilosTests public class HeterogeneousTests : OrleansTestingBase, IDisposable { private TestCluster cluster; - private readonly TimeSpan refreshTimeout = TimeSpan.FromMilliseconds(200); + private readonly TimeSpan refreshInterval = TimeSpan.FromMilliseconds(200); private void SetupAndDeployCluster(string defaultPlacementStrategy, params Type[] blackListedTypes) { cluster?.StopAllSilos(); var typesName = blackListedTypes.Select(t => t.FullName).ToList(); var options = new TestClusterOptions(1); - options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = refreshTimeout; + options.ClusterConfiguration.Globals.TypeMapRefreshInterval = refreshInterval; options.ClusterConfiguration.Globals.DefaultPlacementStrategy = defaultPlacementStrategy; options.ClusterConfiguration.Overrides[Silo.PrimarySiloName].ExcludedGrainTypes = typesName; cluster = new TestCluster(options); @@ -61,7 +61,7 @@ private async Task MergeGrainResolverTestsImpl(string defaultPlacementStrategy, { SetupAndDeployCluster(defaultPlacementStrategy, blackListedTypes); - var delayTimeout = refreshTimeout.Add(refreshTimeout); + var delayTimeout = refreshInterval.Add(refreshInterval); // Should fail var exception = Assert.Throws(() => GrainFactory.GetGrain(0)); diff --git a/test/Tester/MembershipTests/SilosStopTests.cs b/test/Tester/MembershipTests/SilosStopTests.cs index c65b30ea2e..f89c31520c 100644 --- a/test/Tester/MembershipTests/SilosStopTests.cs +++ b/test/Tester/MembershipTests/SilosStopTests.cs @@ -18,7 +18,7 @@ public override TestCluster CreateTestCluster() options.ClusterConfiguration.Globals.DefaultPlacementStrategy = "ActivationCountBasedPlacement"; options.ClusterConfiguration.Globals.NumMissedProbesLimit = 1; options.ClusterConfiguration.Globals.NumVotesForDeathDeclaration = 1; - options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(100); + options.ClusterConfiguration.Globals.TypeMapRefreshInterval = TimeSpan.FromMilliseconds(100); // use only Primary as the gateway options.ClientConfiguration.Gateways = options.ClientConfiguration.Gateways.Take(1).ToList(); diff --git a/test/TesterInternal/General/ElasticPlacementTest.cs b/test/TesterInternal/General/ElasticPlacementTest.cs index 3532be60d4..621f05acc1 100644 --- a/test/TesterInternal/General/ElasticPlacementTest.cs +++ b/test/TesterInternal/General/ElasticPlacementTest.cs @@ -32,7 +32,7 @@ public override TestCluster CreateTestCluster() options.ClusterConfiguration.Globals.LivenessType = GlobalConfiguration.LivenessProviderType.AzureTable; options.ClientConfiguration.GatewayProvider = ClientConfiguration.GatewayProviderType.AzureTable; - options.ClusterConfiguration.Globals.TypeMapRefreshTimeout = TimeSpan.FromMilliseconds(100); + options.ClusterConfiguration.Globals.TypeMapRefreshInterval = TimeSpan.FromMilliseconds(100); return new TestCluster(options); } From 44e117b49782e57100632eb762c2bb1f5585f020 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Tue, 6 Dec 2016 15:54:09 -0800 Subject: [PATCH 19/21] Cleaning unused method in IPlacementContext --- src/OrleansRuntime/Placement/IPlacementContext.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/OrleansRuntime/Placement/IPlacementContext.cs b/src/OrleansRuntime/Placement/IPlacementContext.cs index 107010b46f..ea819f956e 100644 --- a/src/OrleansRuntime/Placement/IPlacementContext.cs +++ b/src/OrleansRuntime/Placement/IPlacementContext.cs @@ -70,16 +70,6 @@ public static string GetGrainTypeName(this IPlacementContext @this, int typeCode return grainClass; } - public static string GetGrainTypeNameAndSupportedSilos(this IPlacementContext @this, GrainId grainId, out IList supportedSiloAddresses, string genericArguments = null) - { - string grainClass; - PlacementStrategy unused; - MultiClusterRegistrationStrategy unusedActivationStrategy; - @this.GetGrainTypeInfo(grainId.GetTypeCode(), out grainClass, out unused, out unusedActivationStrategy, genericArguments); - supportedSiloAddresses = @this.GetCompatibleSiloList(grainId); - return grainClass; - } - public static string GetGrainTypeName(this IPlacementContext @this, GrainId grainId, string genericArguments = null) { return @this.GetGrainTypeName(grainId.GetTypeCode(), genericArguments); From 2c5f893addf6360c374d259db696b5c1c93dab6c Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Tue, 6 Dec 2016 15:54:42 -0800 Subject: [PATCH 20/21] Revert "Use IReadOnlyDictionary for GrainInterfaceMapsBySilo in GrainTypeManager to avoid allocation" This reverts commit fc940b69233dadf6bf76a5758d5707a14f2a0aa5. --- src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index 3da39e5772..90b9367c81 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -15,7 +15,7 @@ namespace Orleans.Runtime internal class GrainTypeManager { private IDictionary grainTypes; - private IReadOnlyDictionary grainInterfaceMapsBySilo; + private IDictionary grainInterfaceMapsBySilo; private Dictionary> supportedSilosByTypeCode; private readonly Logger logger = LogManager.GetLogger("GrainTypeManager"); private readonly GrainInterfaceMap grainInterfaceMap; @@ -24,9 +24,9 @@ internal class GrainTypeManager private static readonly object lockable = new object(); private readonly PlacementStrategy defaultPlacementStrategy; - internal IReadOnlyDictionary GrainInterfaceMapsBySilo + internal IDictionary GrainInterfaceMapsBySilo { - get { return grainInterfaceMapsBySilo; } + get { return new Dictionary(grainInterfaceMapsBySilo); } } public static GrainTypeManager Instance { get; private set; } @@ -150,7 +150,7 @@ internal void GetTypeInfo(int typeCode, out string grainClass, out PlacementStra throw new OrleansException(String.Format("Unexpected: Cannot find an implementation class for grain interface {0}", typeCode)); } - internal void SetInterfaceMapsBySilo(IReadOnlyDictionary value) + internal void SetInterfaceMapsBySilo(IDictionary value) { grainInterfaceMapsBySilo = value; RebuildFullGrainInterfaceMap(); From 5da8979dd288de1d42d3ebfadb2821fba007bc45 Mon Sep 17 00:00:00 2001 From: Benjamin Petit Date: Tue, 6 Dec 2016 16:04:06 -0800 Subject: [PATCH 21/21] Use IReadOnlyDictionary for GrainInterfaceMapsBySilo in GrainTypeManager to avoid returning a copy --- src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs index 90b9367c81..df00e45bc8 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs @@ -15,7 +15,7 @@ namespace Orleans.Runtime internal class GrainTypeManager { private IDictionary grainTypes; - private IDictionary grainInterfaceMapsBySilo; + private Dictionary grainInterfaceMapsBySilo; private Dictionary> supportedSilosByTypeCode; private readonly Logger logger = LogManager.GetLogger("GrainTypeManager"); private readonly GrainInterfaceMap grainInterfaceMap; @@ -24,9 +24,9 @@ internal class GrainTypeManager private static readonly object lockable = new object(); private readonly PlacementStrategy defaultPlacementStrategy; - internal IDictionary GrainInterfaceMapsBySilo + internal IReadOnlyDictionary GrainInterfaceMapsBySilo { - get { return new Dictionary(grainInterfaceMapsBySilo); } + get { return grainInterfaceMapsBySilo; } } public static GrainTypeManager Instance { get; private set; } @@ -150,7 +150,7 @@ internal void GetTypeInfo(int typeCode, out string grainClass, out PlacementStra throw new OrleansException(String.Format("Unexpected: Cannot find an implementation class for grain interface {0}", typeCode)); } - internal void SetInterfaceMapsBySilo(IDictionary value) + internal void SetInterfaceMapsBySilo(Dictionary value) { grainInterfaceMapsBySilo = value; RebuildFullGrainInterfaceMap();