Heterogenous silos support

src/Orleans/Configuration/GlobalConfiguration.cs

@@ -170,6 +170,10 @@ public bool PrimaryNodeIsRequired
         /// The number of seconds to attempt to join a cluster of silos before giving up.
         /// </summary>
         public TimeSpan MaxJoinAttemptTime { get; set; }
+        /// <summary>
+        /// The number of seconds to refresh the cluster grain interface map
+        /// </summary>
+        public TimeSpan TypeMapRefreshInterval { get; set; }
         internal ConfigValue<int> ExpectedClusterSizeConfigValue { get; set; }
         /// <summary>
         /// The expected size of a cluster. Need not be very accurate, can be an overestimate.
@@ -470,6 +474,7 @@ internal bool UseAzureSystemStore
         private static readonly TimeSpan DEFAULT_LIVENESS_DEATH_VOTE_EXPIRATION_TIMEOUT = TimeSpan.FromSeconds(120);
         private static readonly TimeSpan DEFAULT_LIVENESS_I_AM_ALIVE_TABLE_PUBLISH_TIMEOUT = TimeSpan.FromMinutes(5);
         private static readonly TimeSpan DEFAULT_LIVENESS_MAX_JOIN_ATTEMPT_TIME = TimeSpan.FromMinutes(5); // 5 min
+        private static readonly TimeSpan DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME = TimeSpan.FromMinutes(1);
         private const int DEFAULT_LIVENESS_NUM_MISSED_PROBES_LIMIT = 3;
         private const int DEFAULT_LIVENESS_NUM_PROBED_SILOS = 3;
         private const int DEFAULT_LIVENESS_NUM_VOTES_FOR_DEATH_DECLARATION = 2;
@@ -522,6 +527,7 @@ internal GlobalConfiguration()
             UseLivenessGossip = DEFAULT_LIVENESS_USE_LIVENESS_GOSSIP;
             ValidateInitialConnectivity = DEFAULT_VALIDATE_INITIAL_CONNECTIVITY;
             MaxJoinAttemptTime = DEFAULT_LIVENESS_MAX_JOIN_ATTEMPT_TIME;
+            TypeMapRefreshInterval = DEFAULT_REFRESH_CLUSTER_INTERFACEMAP_TIME;
             MaxMultiClusterGateways = DEFAULT_MAX_MULTICLUSTER_GATEWAYS;
             BackgroundGossipInterval = DEFAULT_BACKGROUND_GOSSIP_INTERVAL;
             UseGlobalSingleInstanceByDefault = DEFAULT_USE_GLOBAL_SINGLE_INSTANCE;

src/Orleans/Configuration/NodeConfiguration.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Sockets;
 using System.Text;
@@ -212,6 +213,8 @@ public string TraceFilePattern
 
         public Dictionary<string, SearchOption> AdditionalAssemblyDirectories { get; set; }
 
+        public List<string> ExcludedGrainTypes { get; set; }
+
         public string SiloShutdownEventName { get; set; }
 
         internal const string DEFAULT_NODE_NAME = "default";
@@ -270,6 +273,7 @@ public NodeConfiguration()
             UseNagleAlgorithm = false;
 
             AdditionalAssemblyDirectories = new Dictionary<string, SearchOption>();
+            ExcludedGrainTypes = new List<string>();
         }
 
         public NodeConfiguration(NodeConfiguration other)
@@ -320,6 +324,7 @@ public NodeConfiguration(NodeConfiguration other)
 
             StartupTypeName = other.StartupTypeName;
             AdditionalAssemblyDirectories = other.AdditionalAssemblyDirectories;
+            ExcludedGrainTypes = other.ExcludedGrainTypes.ToList();
         }
 
         public override string ToString()
@@ -487,7 +492,6 @@ internal void Load(XmlElement root)
                     case "AdditionalAssemblyDirectories":
                         ConfigUtilities.ParseAdditionalAssemblyDirectories(AdditionalAssemblyDirectories, child);
                         break;
-
                 }
             }
         }

src/Orleans/Logging/ErrorCodes.cs

@@ -1093,6 +1093,8 @@ internal enum ErrorCode
         GlobalSingleInstance_MaintainerException = GlobalSingleInstanceBase + 3,
         GlobalSingleInstance_MultipleOwners = GlobalSingleInstanceBase + 4,
 
+        TypeManagerBase = Runtime + 4200,
+        TypeManager_GetSiloGrainInterfaceMapError = TypeManagerBase + 1,
     }
 }
 // ReSharper restore InconsistentNaming

src/Orleans/Messaging/ProxiedMessageCenter.cs

@@ -278,7 +278,7 @@ private void RejectOrResend(Message msg)
         public Task<IGrainTypeResolver> GetTypeCodeMap(GrainFactory grainFactory)
         {
             var silo = GetLiveGatewaySiloAddress();
-            return GetTypeManager(silo, grainFactory).GetTypeCodeMap(silo);
+            return GetTypeManager(silo, grainFactory).GetClusterTypeCodeMap();
         }
 
         public Task<Streams.ImplicitStreamSubscriberTable> GetImplicitStreamSubscriberTable(GrainFactory grainFactory)
@@ -397,9 +397,9 @@ public int ReceiveQueueLength
 
         #endregion
 
-        private ITypeManager GetTypeManager(SiloAddress destination, GrainFactory grainFactory)
+        private IClusterTypeManager GetTypeManager(SiloAddress destination, GrainFactory grainFactory)
         {
-            return grainFactory.GetSystemTarget<ITypeManager>(Constants.TypeManagerId, destination);
+            return grainFactory.GetSystemTarget<IClusterTypeManager>(Constants.TypeManagerId, destination);
         }
 
         private SiloAddress GetLiveGatewaySiloAddress()

src/Orleans/Runtime/GrainInterfaceMap.cs

@@ -67,30 +67,66 @@ public override string ToString()
         private readonly Dictionary<int, GrainInterfaceData> table;
         private readonly HashSet<int> unordered;
 
-        [NonSerialized]
         private readonly Dictionary<int, GrainClassData> implementationIndex;
 
         [NonSerialized] // Client shouldn't need this
         private readonly Dictionary<string, string> primaryImplementations;
 
         private readonly bool localTestMode;
         private readonly HashSet<string> loadedGrainAsemblies;
+
+		private readonly PlacementStrategy defaultPlacementStrategy;
 
-        private readonly PlacementStrategy defaultPlacementStrategy;
+        internal IList<int> SupportedGrainTypes
+        {
+            get { return implementationIndex.Keys.ToList(); }
+        }
 
         public GrainInterfaceMap(bool localTestMode, PlacementStrategy defaultPlacementStrategy)
         {
-            this.defaultPlacementStrategy = defaultPlacementStrategy;
             table = new Dictionary<int, GrainInterfaceData>();
             typeToInterfaceData = new Dictionary<string, GrainInterfaceData>();
             primaryImplementations = new Dictionary<string, string>();
             implementationIndex = new Dictionary<int, GrainClassData>();
             unordered = new HashSet<int>();
             this.localTestMode = localTestMode;
+            this.defaultPlacementStrategy = defaultPlacementStrategy;
             if(localTestMode) // if we are running in test mode, we'll build a list of loaded grain assemblies to help with troubleshooting deployment issue
                 loadedGrainAsemblies = new HashSet<string>();
         }
 
+        internal void AddMap(GrainInterfaceMap map)
+        {
+            foreach (var kvp in map.typeToInterfaceData)
+            {
+                if (!typeToInterfaceData.ContainsKey(kvp.Key))
+                {
+                    typeToInterfaceData.Add(kvp.Key, kvp.Value);
+                }
+            }
+
+            foreach (var kvp in map.table)
+            {
+                if (!table.ContainsKey(kvp.Key))
+                {
+                    table.Add(kvp.Key, kvp.Value);
+                }
+            }
+
+            foreach (var grainClassTypeCode in map.unordered)
+            {
+                unordered.Add(grainClassTypeCode);
+            }
+
+            foreach (var kvp in map.implementationIndex)
+            {
+                if (!implementationIndex.ContainsKey(kvp.Key))
+                {
+                    implementationIndex.Add(kvp.Key, kvp.Value);
+                }
+            }
+        }
+
         internal void AddEntry(int interfaceId, Type iface, int grainTypeCode, string grainInterface, string grainClass, string assembly, 
                                 bool isGenericGrainClass, PlacementStrategy placement, MultiClusterRegistrationStrategy registrationStrategy, bool primaryImplementation = false)
         {
@@ -173,6 +209,14 @@ internal bool ContainsGrainInterface(int interfaceId)
             }
         }
 
+        internal bool ContainsGrainImplementation(int typeCode)
+        {
+            lock (this)
+            {
+                return implementationIndex.ContainsKey(typeCode);
+            }
+        }
+
         internal bool TryGetTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy registrationStrategy, string genericArguments = null)
         {
             lock (this)

src/Orleans/Runtime/ITypeManager.cs

@@ -6,10 +6,23 @@ namespace Orleans.Runtime
     /// <summary>
     /// Client gateway interface for obtaining the grain interface/type map.
     /// </summary>
-    internal interface ITypeManager : ISystemTarget
+    internal interface IClusterTypeManager : ISystemTarget
     {
-        Task<IGrainTypeResolver> GetTypeCodeMap(SiloAddress silo);
+        /// <summary>
+        /// Acquires grain interface map for all grain types supported across the entire cluster
+        /// </summary>
+        /// <returns></returns>
+        Task<IGrainTypeResolver> GetClusterTypeCodeMap();
 
         Task<Streams.ImplicitStreamSubscriberTable> GetImplicitStreamSubscriberTable(SiloAddress silo);
     }
+
+    internal interface ISiloTypeManager : ISystemTarget
+    {
+        /// <summary>
+        /// Acquires grain interface map for all grain types supported by hosted silo.
+        /// </summary>
+        /// <returns></returns>
+        Task<GrainInterfaceMap> GetSiloTypeCodeMap();
+    }
 }

src/OrleansRuntime/Catalog/Catalog.cs

@@ -119,6 +119,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont
 
 
         public GrainTypeManager GrainTypeManager { get; private set; }
+
         public SiloAddress LocalSilo { get; private set; }
         internal ISiloStatusOracle SiloStatusOracle { get; set; }
         internal readonly ActivationCollector ActivationCollector;
@@ -200,6 +201,15 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont
         /// </summary>
         public Dispatcher Dispatcher { get; }
 
+        public IList<SiloAddress> GetCompatibleSiloList(GrainId grain)
+        {
+            var typeCode = grain.GetTypeCode();
+            var compatibleSilos = GrainTypeManager.GetSupportedSilos(typeCode).Intersect(AllActiveSilos).ToList();
+            if (compatibleSilos.Count == 0)
+                throw new OrleansException($"TypeCode ${typeCode} not supported in the cluster");
+            return compatibleSilos;
+        }
+
         internal void SetStorageManager(IStorageProviderManager storageManager)
         {
             storageProviderManager = storageManager;

src/OrleansRuntime/GrainTypeManager/GrainTypeManager.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Linq;
 using System.Net;
 using System.Reflection;
@@ -13,18 +15,26 @@ namespace Orleans.Runtime
     internal class GrainTypeManager
     {
         private IDictionary<string, GrainTypeData> grainTypes;
+        private Dictionary<SiloAddress, GrainInterfaceMap> grainInterfaceMapsBySilo;
+        private Dictionary<int, IList<SiloAddress>> supportedSilosByTypeCode;
         private readonly Logger logger = LogManager.GetLogger("GrainTypeManager");
         private readonly GrainInterfaceMap grainInterfaceMap;
         private readonly Dictionary<int, InvokerData> invokers = new Dictionary<int, InvokerData>();
         private readonly SiloAssemblyLoader loader;
         private static readonly object lockable = new object();
+		private readonly PlacementStrategy defaultPlacementStrategy;
 
-        private readonly PlacementStrategy defaultPlacementStrategy;
+        internal IReadOnlyDictionary<SiloAddress, GrainInterfaceMap> GrainInterfaceMapsBySilo
+        {
+            get { return grainInterfaceMapsBySilo; }
+        }
 
         public static GrainTypeManager Instance { get; private set; }
 
         public IEnumerable<KeyValuePair<string, GrainTypeData>> GrainClassTypeData { get { return grainTypes; } }
 
+        public GrainInterfaceMap ClusterGrainInterfaceMap { get; private set; }
+
         public static void Stop()
         {
             Instance = null;
@@ -40,6 +50,8 @@ public GrainTypeManager(bool localTestMode, SiloAssemblyLoader loader, DefaultPl
             this.defaultPlacementStrategy = defaultPlacementStrategy.PlacementStrategy;
             this.loader = loader;
             grainInterfaceMap = new GrainInterfaceMap(localTestMode, this.defaultPlacementStrategy);
+            ClusterGrainInterfaceMap = grainInterfaceMap;
+            grainInterfaceMapsBySilo = new Dictionary<SiloAddress, GrainInterfaceMap>();
             lock (lockable)
             {
                 if (Instance != null)
@@ -134,10 +146,21 @@ internal bool TryGetPrimaryImplementation(string grainInterface, out string grai
 
         internal void GetTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy activationStrategy, string genericArguments = null)
         {
-            if (!grainInterfaceMap.TryGetTypeInfo(typeCode, out grainClass, out placement, out activationStrategy, genericArguments))
+            if (!ClusterGrainInterfaceMap.TryGetTypeInfo(typeCode, out grainClass, out placement, out activationStrategy, genericArguments))
                 throw new OrleansException(String.Format("Unexpected: Cannot find an implementation class for grain interface {0}", typeCode));
         }
 
+        internal void SetInterfaceMapsBySilo(Dictionary<SiloAddress, GrainInterfaceMap> value)
+        {
+            grainInterfaceMapsBySilo = value;
+            RebuildFullGrainInterfaceMap();
+        }
+
+        internal IList<SiloAddress> GetSupportedSilos(int typeCode)
+        {
+            return supportedSilosByTypeCode[typeCode];
+        }
+
         private void InitializeGrainClassData(SiloAssemblyLoader loader, bool strict)
         {
             grainTypes = loader.GetGrainClassTypes(strict);
@@ -196,7 +219,7 @@ public bool TryGetData(string name, out GrainTypeData result)
             return grainTypes.TryGetValue(name, out result);
         }
 
-        internal IGrainTypeResolver GetTypeCodeMap()
+        internal GrainInterfaceMap GetTypeCodeMap()
         {
             // the map is immutable at this point
             return grainInterfaceMap;
@@ -241,6 +264,28 @@ internal IGrainMethodInvoker GetInvoker(int interfaceId, string genericGrainType
                 interfaceName, interfaceId));
         }
 
+        private void RebuildFullGrainInterfaceMap()
+        {
+            var newClusterGrainInterfaceMap = new GrainInterfaceMap(false, defaultPlacementStrategy);
+            var newSupportedSilosByTypeCode = new Dictionary<int, IList<SiloAddress>>();
+            newClusterGrainInterfaceMap.AddMap(grainInterfaceMap);
+            foreach (var kvp in grainInterfaceMapsBySilo)
+            {
+                newClusterGrainInterfaceMap.AddMap(kvp.Value);
+                foreach (var grainType in kvp.Value.SupportedGrainTypes)
+                {
+                    IList<SiloAddress> supportedSilos;
+                    if (!newSupportedSilosByTypeCode.TryGetValue(grainType, out supportedSilos))
+                    {
+                        newSupportedSilosByTypeCode[grainType] = supportedSilos = new List<SiloAddress>();
+                    }
+                    supportedSilos.Add(kvp.Key);
+                }
+            }
+            ClusterGrainInterfaceMap = newClusterGrainInterfaceMap;
+            supportedSilosByTypeCode = newSupportedSilosByTypeCode;
+        }
+
         private class InvokerData
         {
             private readonly Type baseInvokerType;

src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs

@@ -13,17 +13,21 @@ namespace Orleans.Runtime
 {
     internal class SiloAssemblyLoader
     {
+        private readonly List<string> excludedGrains;
         private readonly LoggerImpl logger = LogManager.GetLogger("AssemblyLoader.Silo");
         private List<string> discoveredAssemblyLocations;
         private Dictionary<string, SearchOption> directories;
 
         public SiloAssemblyLoader(NodeConfiguration nodeConfig)
-            : this(nodeConfig.AdditionalAssemblyDirectories)
+            : this(nodeConfig.AdditionalAssemblyDirectories, nodeConfig.ExcludedGrainTypes)
         {
         }
 
-        public SiloAssemblyLoader(IDictionary<string, SearchOption> additionalDirectories)
+        public SiloAssemblyLoader(IDictionary<string, SearchOption> additionalDirectories, IEnumerable<string> excludedGrains = null)
         {
+            this.excludedGrains = excludedGrains != null
+                ? new List<string>(excludedGrains)
+                : new List<string>();
             var exeRoot = Path.GetDirectoryName(typeof(SiloAssemblyLoader).GetTypeInfo().Assembly.Location);
             var appRoot = Path.Combine(exeRoot, "Applications");
             var cwd = Directory.GetCurrentDirectory();
@@ -80,6 +84,9 @@ private void LoadApplicationAssemblies()
             foreach (var grainType in grainTypes)
             {
                 var className = TypeUtils.GetFullName(grainType);
+                if (excludedGrains.Contains(className))
+                    continue;
+
                 if (result.ContainsKey(className))
                     throw new InvalidOperationException(
                         string.Format("Precondition violated: GetLoadedGrainTypes should not return a duplicate type ({0})", className));