From 657ca9bfe48d60aade4f0f6f8a0d2ea42ac76313 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 2 Dec 2016 09:43:44 -0800 Subject: [PATCH 01/14] revised #1854, for naming, packaging, and log-consistency provider selection --- src/Orleans/Async/BatchWorker.cs | 56 +- src/Orleans/Async/TaskExtensions.cs | 4 + .../Configuration/GlobalConfiguration.cs | 31 + .../Configuration/OrleansConfiguration.xsd | 12 + .../Configuration/ProviderConfiguration.cs | 1 + src/Orleans/Core/GrainAttributes.cs | 33 +- .../ClusterLocalRegistration.cs | 8 +- .../GlobalSingleInstanceRegistration.cs | 8 +- .../MultiClusterRegistrationStrategy.cs | 32 +- .../LogConsistency/ConnectionIssues.cs | 68 ++ .../IConnectionIssueListener.cs | 27 + .../ILogConsistencyDiagnostics.cs | 43 + .../LogConsistency/ILogConsistencyProvider.cs | 47 + .../LogConsistency/ILogConsistentGrain.cs | 46 + src/Orleans/LogConsistency/ILogViewAdaptor.cs | 113 +++ .../LogConsistency/ILogViewAdaptorHost.cs | 35 + .../LogConsistency/IProtocolServices.cs | 138 +++ src/Orleans/Logging/ErrorCodes.cs | 8 + .../MultiCluster/IProtocolParticipant.cs | 49 + src/Orleans/Orleans.csproj | 10 + src/Orleans/Providers/IOrleansProvider.cs | 2 + src/Orleans/Providers/IProviderRuntime.cs | 9 + src/Orleans/Runtime/Constants.cs | 6 +- .../IProtocolGateway.cs | 22 + .../ICustomStorageInterface.cs | 31 + .../LogConsistencyProvider.cs | 87 ++ .../LogViewAdaptor.cs | 377 ++++++++ src/OrleansEventSourcing/JournaledGrain.cs | 332 ++++++- .../JournaledGrainState.cs | 65 -- .../OrleansEventSourcing.csproj | 13 +- .../Protocols/ConnectionIssues.cs | 82 ++ .../Protocols/NotificationMessage.cs | 48 + .../Protocols/NotificationTracker.cs | 226 +++++ .../Protocols/PrimaryBasedLogViewAdaptor.cs | 895 ++++++++++++++++++ .../Protocols/RecordedConnectionIssue.cs | 96 ++ .../DefaultAdaptorFactory.cs | 30 + .../GrainStateWithMetaData.cs | 160 ++++ .../LogConsistencyProvider.cs | 93 ++ .../VersionedStateStorage/LogViewAdaptor.cs | 360 +++++++ src/OrleansRuntime/Catalog/Catalog.cs | 108 ++- src/OrleansRuntime/Catalog/GrainCreator.cs | 51 +- .../GrainTypeManager/GrainTypeData.cs | 2 + .../GrainTypeManager/SiloAssemblyLoader.cs | 4 +- .../ILogConsistencyProviderManager.cs | 19 + .../LogConsistencyProviderManager.cs | 106 +++ .../LogConsistency/ProtocolGateway.cs | 30 + .../LogConsistency/ProtocolServices.cs | 256 +++++ .../IMultiClusterOracle.cs | 18 + .../MultiClusterNetwork/MultiClusterOracle.cs | 15 + .../MultiClusterOracleData.cs | 62 +- src/OrleansRuntime/OrleansRuntime.csproj | 4 + src/OrleansRuntime/Silo/Silo.cs | 17 + src/OrleansTestingHost/AppDomainSiloHost.cs | 18 +- .../LogConsistencyProviderConfiguration.cs | 52 + .../OrleansTestingHost.csproj | 1 + .../IJournaledPersonGrain.cs | 4 + .../ILogConsistentGrain.cs | 94 ++ .../TestGrainInterfaces.csproj | 1 + .../EventSourcing/JournaledPersonGrain.cs | 126 ++- .../EventSourcing/JournaledPerson_Events.cs | 10 +- test/TestGrains/EventSourcing/PersonState.cs | 7 +- test/TestGrains/LogConsistentGrain.cs | 184 ++++ .../LogConsistentGrainVariations.cs | 119 +++ test/TestGrains/TestGrains.csproj | 3 + .../EventSourcingTests/JournaledGrainTests.cs | 81 ++ test/Tester/Tester.csproj | 1 + .../BasicLogConsistentGrainTests.cs | 125 +++ .../GeoClusterTests/BasicMultiClusterTest.cs | 56 +- .../GlobalSingleInstanceClusterTests.cs | 4 +- .../LogConsistencyTestFixture.cs | 496 ++++++++++ .../LogConsistencyTestsFourClusters.cs | 58 ++ .../LogConsistencyTestsTwoClusters.cs | 59 ++ .../MultiClusterNetworkTests.cs | 8 +- .../MultiClusterRegistrationTests.cs | 12 +- .../GeoClusterTests/TestingClusterHost.cs | 48 +- test/TesterInternal/TesterInternal.csproj | 8 + 76 files changed, 5783 insertions(+), 187 deletions(-) create mode 100644 src/Orleans/LogConsistency/ConnectionIssues.cs create mode 100644 src/Orleans/LogConsistency/IConnectionIssueListener.cs create mode 100644 src/Orleans/LogConsistency/ILogConsistencyDiagnostics.cs create mode 100644 src/Orleans/LogConsistency/ILogConsistencyProvider.cs create mode 100644 src/Orleans/LogConsistency/ILogConsistentGrain.cs create mode 100644 src/Orleans/LogConsistency/ILogViewAdaptor.cs create mode 100644 src/Orleans/LogConsistency/ILogViewAdaptorHost.cs create mode 100644 src/Orleans/LogConsistency/IProtocolServices.cs create mode 100644 src/Orleans/MultiCluster/IProtocolParticipant.cs create mode 100644 src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs create mode 100644 src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs create mode 100644 src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs create mode 100644 src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs delete mode 100644 src/OrleansEventSourcing/JournaledGrainState.cs create mode 100644 src/OrleansEventSourcing/Protocols/ConnectionIssues.cs create mode 100644 src/OrleansEventSourcing/Protocols/NotificationMessage.cs create mode 100644 src/OrleansEventSourcing/Protocols/NotificationTracker.cs create mode 100644 src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs create mode 100644 src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs create mode 100644 src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs create mode 100644 src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs create mode 100644 src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs create mode 100644 src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs create mode 100644 src/OrleansRuntime/LogConsistency/ILogConsistencyProviderManager.cs create mode 100644 src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs create mode 100644 src/OrleansRuntime/LogConsistency/ProtocolGateway.cs create mode 100644 src/OrleansRuntime/LogConsistency/ProtocolServices.cs create mode 100644 src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs create mode 100644 test/TestGrainInterfaces/ILogConsistentGrain.cs create mode 100644 test/TestGrains/LogConsistentGrain.cs create mode 100644 test/TestGrains/LogConsistentGrainVariations.cs create mode 100644 test/Tester/EventSourcingTests/JournaledGrainTests.cs create mode 100644 test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs create mode 100644 test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs create mode 100644 test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs create mode 100644 test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs diff --git a/src/Orleans/Async/BatchWorker.cs b/src/Orleans/Async/BatchWorker.cs index bd08cc8982..28f9a5fe63 100644 --- a/src/Orleans/Async/BatchWorker.cs +++ b/src/Orleans/Async/BatchWorker.cs @@ -16,12 +16,14 @@ public abstract class BatchWorker /// Implement this member in derived classes to define what constitutes a work cycle protected abstract Task Work(); + protected object lockable = new object(); + /// /// Notify the worker that there is more work. /// public void Notify() { - lock (this) + lock (lockable) { if (currentWorkCycle != null) { @@ -62,7 +64,7 @@ private void CheckForMoreWork() TaskCompletionSource signal = null; Task taskToSignal = null; - lock (this) + lock (lockable) { if (moreWork) { @@ -109,7 +111,7 @@ public async Task WaitForCurrentWorkToBeServiced() Task waitfortask = null; // figure out exactly what we need to wait for - lock (this) + lock (lockable) { if (!moreWork) // just wait for current work cycle @@ -132,5 +134,53 @@ public async Task WaitForCurrentWorkToBeServiced() else if (waitfortask != null) await waitfortask; } + + /// + /// Notify the worker that there is more work, and wait for the current work cycle, and also the next work cycle if there is currently unserviced work. + /// + public async Task NotifyAndWaitForWorkToBeServiced() + { + Task waitForTaskTask = null; + Task waitForTask = null; + + lock (lockable) + { + if (currentWorkCycle != null) + { + moreWork = true; + if (nextWorkCyclePromise == null) + nextWorkCyclePromise = new TaskCompletionSource(); + waitForTaskTask = nextWorkCyclePromise.Task; + } + else + { + Start(); + waitForTask = currentWorkCycle; + } + } + + if (waitForTaskTask != null) + await await waitForTaskTask; + + else if (waitForTask != null) + await waitForTask; + } + } + + /// A convenient variant of a batch worker + /// that allows the work function to be passed as a constructor argument + public class BatchWorkerFromDelegate : BatchWorker + { + public BatchWorkerFromDelegate(Func work) + { + this.work = work; + } + + private Func work; + + protected override Task Work() + { + return work(); + } } } \ No newline at end of file diff --git a/src/Orleans/Async/TaskExtensions.cs b/src/Orleans/Async/TaskExtensions.cs index 5b135b57f4..78f1798089 100644 --- a/src/Orleans/Async/TaskExtensions.cs +++ b/src/Orleans/Async/TaskExtensions.cs @@ -344,6 +344,10 @@ internal static T GetResult(this Task task) { return task.GetAwaiter().GetResult(); } + internal static void GetResult(this Task task) + { + task.GetAwaiter().GetResult(); + } } } diff --git a/src/Orleans/Configuration/GlobalConfiguration.cs b/src/Orleans/Configuration/GlobalConfiguration.cs index 365f4f7905..e282e0fa6c 100644 --- a/src/Orleans/Configuration/GlobalConfiguration.cs +++ b/src/Orleans/Configuration/GlobalConfiguration.cs @@ -10,6 +10,7 @@ using Orleans.Providers; using Orleans.Storage; using Orleans.Streams; +using Orleans.LogConsistency; namespace Orleans.Runtime.Configuration { @@ -1070,6 +1071,36 @@ public void RegisterStorageProvider(string providerTypeFullName, string provider ProviderConfigurationUtility.RegisterProvider(ProviderConfigurations, ProviderCategoryConfiguration.STORAGE_PROVIDER_CATEGORY_NAME, providerTypeFullName, providerName, properties); } + /// + /// Registers a given log-consistency provider. + /// + /// Full name of the log-consistency provider type + /// Name of the log-consistency provider + /// Properties that will be passed to the log-consistency provider upon initialization + public void RegisterLogConsistencyProvider(string providerTypeFullName, string providerName, IDictionary properties = null) + { + ProviderConfigurationUtility.RegisterProvider(ProviderConfigurations, ProviderCategoryConfiguration.LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME, providerTypeFullName, providerName, properties); + } + + + /// + /// Registers a given type of where is a log-consistency provider + /// + /// Non-abstract type which implements a log-consistency storage interface + /// Name of the log-consistency provider + /// Properties that will be passed to log-consistency provider upon initialization + public void RegisterLogConsistencyProvider(string providerName, IDictionary properties = null) where T : ILogConsistencyProvider + { + Type providerType = typeof(T); + if (providerType.IsAbstract || + providerType.IsGenericType || + !typeof(ILogConsistencyProvider).IsAssignableFrom(providerType)) + throw new ArgumentException("Expected non-generic, non-abstract type which implements ILogConsistencyProvider interface", "typeof(T)"); + + ProviderConfigurationUtility.RegisterProvider(ProviderConfigurations, ProviderCategoryConfiguration.LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME, providerType.FullName, providerName, properties); + } + + /// /// Retrieves an existing provider configuration /// diff --git a/src/Orleans/Configuration/OrleansConfiguration.xsd b/src/Orleans/Configuration/OrleansConfiguration.xsd index e8121c107a..c65dc3193b 100644 --- a/src/Orleans/Configuration/OrleansConfiguration.xsd +++ b/src/Orleans/Configuration/OrleansConfiguration.xsd @@ -820,6 +820,18 @@ + + + + The optional LogConsistencyProviders element contains configuration for log-consistency provider instances. + + + + + + + + diff --git a/src/Orleans/Configuration/ProviderConfiguration.cs b/src/Orleans/Configuration/ProviderConfiguration.cs index d7c3ad5127..d2dab9f7d9 100644 --- a/src/Orleans/Configuration/ProviderConfiguration.cs +++ b/src/Orleans/Configuration/ProviderConfiguration.cs @@ -197,6 +197,7 @@ public class ProviderCategoryConfiguration public const string BOOTSTRAP_PROVIDER_CATEGORY_NAME = "Bootstrap"; public const string STORAGE_PROVIDER_CATEGORY_NAME = "Storage"; public const string STREAM_PROVIDER_CATEGORY_NAME = "Stream"; + public const string LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME = "LogConsistency"; public string Name { get; set; } public IDictionary Providers { get; set; } diff --git a/src/Orleans/Core/GrainAttributes.cs b/src/Orleans/Core/GrainAttributes.cs index 458c430cac..429d48c806 100644 --- a/src/Orleans/Core/GrainAttributes.cs +++ b/src/Orleans/Core/GrainAttributes.cs @@ -2,6 +2,8 @@ using System.Linq; using System.Reflection; using Orleans.GrainDirectory; +using Orleans.Runtime; + namespace Orleans { namespace Concurrency @@ -274,15 +276,42 @@ namespace Providers [AttributeUsage(AttributeTargets.Class)] public sealed class StorageProviderAttribute : Attribute { + /// + /// The name of the provider to be used for persisting of grain state + /// + public string ProviderName { get; set; } + public StorageProviderAttribute() { - ProviderName = Runtime.Constants.DEFAULT_STORAGE_PROVIDER_NAME; + ProviderName = Runtime.Constants.DEFAULT_STORAGE_PROVIDER_NAME; } + } + + /// + /// The [Orleans.Providers.LogConsistencyProvider] attribute is used to define which consistency provider to use for grains using the log-view state abstraction. + /// + /// Specifying [Orleans.Providers.LogConsistencyProvider] property is recommended for all grains that derive + /// from ILogConsistentGrain, such as JournaledGrain. + /// If no [Orleans.Providers.LogConsistencyProvider] attribute is specified, then the runtime tries to locate + /// one as follows. First, it looks for a + /// "Default" provider in the configuration file, then it checks if the grain type defines a default. + /// If a consistency provider cannot be located for this grain, then the grain will fail to load into the Silo. + /// + /// + [AttributeUsage(AttributeTargets.Class)] + public sealed class LogConsistencyProviderAttribute : Attribute + { /// - /// The name of the storage provider to ne used for persisting state for this grain. + /// The name of the provider to be used for consistency /// public string ProviderName { get; set; } + + public LogConsistencyProviderAttribute() + { + ProviderName = Runtime.Constants.DEFAULT_LOG_CONSISTENCY_PROVIDER_NAME; + } } + } [AttributeUsage(AttributeTargets.Interface)] diff --git a/src/Orleans/GrainDirectory/ClusterLocalRegistration.cs b/src/Orleans/GrainDirectory/ClusterLocalRegistration.cs index 6507127bff..0855bb4837 100644 --- a/src/Orleans/GrainDirectory/ClusterLocalRegistration.cs +++ b/src/Orleans/GrainDirectory/ClusterLocalRegistration.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using Orleans.MultiCluster; namespace Orleans.GrainDirectory { @@ -34,9 +36,11 @@ public override int GetHashCode() return GetType().GetHashCode(); } - internal override bool IsSingleInstance() + public override IEnumerable GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId) { - return true; + foreach (var clusterId in mcConfig.Clusters) + if (clusterId != myClusterId) + yield return clusterId; } } } diff --git a/src/Orleans/GrainDirectory/GlobalSingleInstanceRegistration.cs b/src/Orleans/GrainDirectory/GlobalSingleInstanceRegistration.cs index ed214c7097..211b76e688 100644 --- a/src/Orleans/GrainDirectory/GlobalSingleInstanceRegistration.cs +++ b/src/Orleans/GrainDirectory/GlobalSingleInstanceRegistration.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using Orleans.MultiCluster; namespace Orleans.GrainDirectory { @@ -41,9 +43,11 @@ public override int GetHashCode() return GetType().GetHashCode(); } - internal override bool IsSingleInstance() + private static List emptyList = new List(); + + public override IEnumerable GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId) { - return true; + return emptyList; // there is only one instance, so no remote instances } } } diff --git a/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs b/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs index 474a6ecfa6..9bc59c0c16 100644 --- a/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs +++ b/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs @@ -1,14 +1,32 @@ using System; using Orleans.Runtime.Configuration; +using Orleans.MultiCluster; +using System.Collections.Generic; namespace Orleans.GrainDirectory { + + /// + /// Interface for multi-cluster registration strategies. Used by protocols that coordinate multiple instances. + /// + public interface IMultiClusterRegistrationStrategy { + + /// + /// Determines which remote clusters have instances. + /// + /// The multi-cluster configuration + /// The cluster id of this cluster + /// + IEnumerable GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId); + + } + /// /// A superclass for all multi-cluster registration strategies. - /// Strategy objects are used as keys to select the proper registrar. + /// Strategy object which is used as keys to select the proper registrar. /// [Serializable] - internal abstract class MultiClusterRegistrationStrategy + internal abstract class MultiClusterRegistrationStrategy : IMultiClusterRegistrationStrategy { private static MultiClusterRegistrationStrategy defaultStrategy; @@ -33,6 +51,14 @@ internal static MultiClusterRegistrationStrategy GetDefault() return defaultStrategy; } - internal abstract bool IsSingleInstance(); + internal static MultiClusterRegistrationStrategy FromGrainType(Type graintype) + { + var attrs = graintype.GetCustomAttributes(typeof(RegistrationAttribute), true); + if (attrs.Length == 0) + return defaultStrategy; + return ((RegistrationAttribute)attrs[0]).RegistrationStrategy; + } + + public abstract IEnumerable GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId); } } diff --git a/src/Orleans/LogConsistency/ConnectionIssues.cs b/src/Orleans/LogConsistency/ConnectionIssues.cs new file mode 100644 index 0000000000..d093ff8cad --- /dev/null +++ b/src/Orleans/LogConsistency/ConnectionIssues.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.LogConsistency +{ + + /// + /// Represents information about connection issues encountered inside log consistency protocols. + /// It is used both inside the protocol to track retry loops, and is made visible to users + /// who want to monitor their log-view grains for communication issues. + /// + [Serializable] + public abstract class ConnectionIssue + { + /// + /// The UTC timestamp of the last time at which the issue was observed + /// + public DateTime TimeStamp { get; set; } + + /// + /// The UTC timestamp of the first time we observed this issue + /// + public DateTime TimeOfFirstFailure { get; set; } + + /// + /// The number of times we have observed this issue since the first failure + /// + public int NumberOfConsecutiveFailures { get; set; } + + /// + /// The delay we are waiting before the next retry + /// + public TimeSpan RetryDelay { get; set; } + + /// + /// Computes the retry delay based on the rest of the information. Is overridden by subclasses + /// that represent specific categories of issues. + /// + /// The previously used retry delay + /// + public abstract TimeSpan ComputeRetryDelay(TimeSpan? previous); + } + + + + /// + /// Represents information about notification failures encountered inside log consistency protocols. + /// + [Serializable] + public abstract class NotificationFailed : ConnectionIssue + { + /// + /// The clusterId of the remote cluster to which we had an issue when sending change notifications. + /// + public string RemoteClusterId { get; set; } + + /// + /// The exception we caught, or null if the problem was not caused by an exception. + /// + public Exception Exception { get; set; } + } + + + +} \ No newline at end of file diff --git a/src/Orleans/LogConsistency/IConnectionIssueListener.cs b/src/Orleans/LogConsistency/IConnectionIssueListener.cs new file mode 100644 index 0000000000..f65a287066 --- /dev/null +++ b/src/Orleans/LogConsistency/IConnectionIssueListener.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.LogConsistency +{ + /// + /// An interface that is implemented by log-consistent grains using virtual protected methods + /// that can be overridden by users, in order to monitor the connection issues. + /// + public interface IConnectionIssueListener + { + /// + /// Called when running into some sort of connection trouble. + /// The called code can modify the retry delay if desired, to change the default. + /// + void OnConnectionIssue(ConnectionIssue connectionIssue); + + /// + /// Called when a previously reported connection issue has been resolved. + /// + void OnConnectionIssueResolved(ConnectionIssue connectionIssue); + } + +} diff --git a/src/Orleans/LogConsistency/ILogConsistencyDiagnostics.cs b/src/Orleans/LogConsistency/ILogConsistencyDiagnostics.cs new file mode 100644 index 0000000000..080c09d883 --- /dev/null +++ b/src/Orleans/LogConsistency/ILogConsistencyDiagnostics.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.LogConsistency +{ + /// + /// Interface for diagnostics. + /// + public interface ILogConsistencyDiagnostics + { + + /// Gets a list of all currently unresolved connection issues. + IEnumerable UnresolvedConnectionIssues { get; } + + /// Turns on the statistics collection for this log-consistent grain. + void EnableStatsCollection(); + + /// Turns off the statistics collection for this log-consistent grain. + void DisableStatsCollection(); + + /// Gets the collected statistics for this log-consistent grain. + LogConsistencyStatistics GetStats(); + + } + + /// + /// A collection of statistics for grains using log-consistency. See + /// + public class LogConsistencyStatistics + { + /// + /// A map from event names to event counts + /// + public Dictionary EventCounters; + /// + /// A list of all measured stabilization latencies + /// + public List StabilizationLatenciesInMsecs; + } +} diff --git a/src/Orleans/LogConsistency/ILogConsistencyProvider.cs b/src/Orleans/LogConsistency/ILogConsistencyProvider.cs new file mode 100644 index 0000000000..9b847d32ea --- /dev/null +++ b/src/Orleans/LogConsistency/ILogConsistencyProvider.cs @@ -0,0 +1,47 @@ + +using System; +using System.Net; +using System.Runtime.Serialization; +using System.Threading.Tasks; +using System.Collections.Generic; +using Orleans.Providers; +using Orleans.Runtime; +using Orleans.Core; +using Orleans.Storage; + +namespace Orleans.LogConsistency +{ + /// + /// Interface to be implemented for a log consistency provider. + /// + public interface ILogConsistencyProvider : IProvider, ILogViewAdaptorFactory + { + /// Gets the TraceLogger used by this log-consistency provider. + Logger Log { get; } + + } + + + /// + /// Interface to be implemented for a log-view adaptor factory + /// + public interface ILogViewAdaptorFactory + { + /// Returns true if a storage provider is required for constructing adaptors. + bool UsesStorageProvider { get; } + + /// + /// Construct a to be installed in the given host grain. + /// + ILogViewAdaptor MakeLogViewAdaptor( + ILogViewAdaptorHost hostgrain, + TLogView initialstate, + string graintypename, + IStorageProvider storageProvider, + IProtocolServices services) + + where TLogView : class, new() + where TLogEntry : class; + + } +} diff --git a/src/Orleans/LogConsistency/ILogConsistentGrain.cs b/src/Orleans/LogConsistency/ILogConsistentGrain.cs new file mode 100644 index 0000000000..01431a03fe --- /dev/null +++ b/src/Orleans/LogConsistency/ILogConsistentGrain.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.Concurrency; +using Orleans.Runtime; +using Orleans.Storage; + +namespace Orleans.LogConsistency +{ + /// + /// This interface encapsulates functionality of grains that manage their state + /// based on log consistency, such as JournaledGrain. + /// It is the equivalent of for log-consistent grains. + /// + public interface ILogConsistentGrain + { + /// + /// called right after grain construction to install the log view adaptor + /// + /// The adaptor factory to use + /// The initial state of the view + /// The type name of the grain + /// The storage provider, if needed + /// Protocol services + void InstallAdaptor(ILogViewAdaptorFactory factory, object state, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services); + + /// + /// Gets the default adaptor factory to use, or null if there is no default + /// (in which case user MUST configure a consistency provider) + /// + ILogViewAdaptorFactory DefaultAdaptorFactory { get; } + } + + + /// + /// Base class for all grains that use log-consistency for managing the state. + /// It is the equivalent of for grains using log-consistency. + /// (SiloAssemblyLoader uses it to extract type) + /// + /// The type of the view + public class LogConsistentGrainBase : Grain + { + } +} diff --git a/src/Orleans/LogConsistency/ILogViewAdaptor.cs b/src/Orleans/LogConsistency/ILogViewAdaptor.cs new file mode 100644 index 0000000000..aadad9cf40 --- /dev/null +++ b/src/Orleans/LogConsistency/ILogViewAdaptor.cs @@ -0,0 +1,113 @@ +using Orleans.MultiCluster; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.LogConsistency +{ + /// + /// A log view adaptor is the storage interface for , whose state is defined as a log view. + /// + /// There is one adaptor per grain, which is installed by when the grain is activated. + /// + /// + /// Type for the log view + /// Type for the log entry + public interface ILogViewAdaptor : + ILogViewRead, + ILogViewUpdate, + ILogConsistencyDiagnostics + where TLogView : new() + { + /// Called during activation, right before the user grain activation code is run. + Task Activate(); + + /// Called during deactivation, right after the user grain deactivation code is run. + Task Deactivate(); + + /// Called when a grain receives a message from a remote instance. + Task OnProtocolMessageReceived(IProtocolMessage payload); + + /// Called after the silo receives a new multi-cluster configuration. + Task OnMultiClusterConfigurationChange(MultiClusterConfiguration next); + } + + /// + /// Interface for reading the log view. + /// + /// The type of the view (state of the grain). + /// The type of log entries. + public interface ILogViewRead + { + /// + /// Local, tentative view of the log (reflecting both confirmed and unconfirmed entries) + /// + TView TentativeView { get; } + + /// + /// Confirmed view of the log (reflecting only confirmed entries) + /// + TView ConfirmedView { get; } + + /// + /// The length of the confirmed prefix of the log + /// + int ConfirmedVersion { get; } + + /// + /// A list of the submitted entries that do not yet appear in the confirmed prefix. + /// + IEnumerable UnconfirmedSuffix { get; } + + } + + /// + /// Interface for updating the log. + /// + /// The type of log entries. + public interface ILogViewUpdate + { + /// + /// Submit a single log entry to be appended to the global log, + /// either at the current or at any later position. + /// + void Submit(TLogEntry entry); + + /// + /// Submit a range of log entries to be appended atomically to the global log, + /// either at the current or at any later position. + /// + void SubmitRange(IEnumerable entries); + + /// + /// Try to append a single log entry at the current position of the log. + /// + /// true if the entry was appended successfully, or false + /// if there was a concurrency conflict (i.e. some other entries were previously appended). + /// + Task TryAppend(TLogEntry entry); + + /// + /// Try to append a range of log entries atomically at the current position of the log. + /// + /// true if the entries were appended successfully, or false + /// if there was a concurrency conflict (i.e. some other entries were previously appended). + /// + Task TryAppendRange(IEnumerable entries); + + /// + /// Confirm all submitted entries. + ///Waits until all previously submitted entries appear in the confirmed prefix of the log. + /// + Task ConfirmSubmittedEntriesAsync(); + + /// + /// Confirm all submitted entries and get the latest log view. + ///Waits until all previously submitted entries appear in the confirmed prefix of the log, and forces a refresh of the confirmed prefix. + /// + /// + Task SynchronizeNowAsync(); + } +} diff --git a/src/Orleans/LogConsistency/ILogViewAdaptorHost.cs b/src/Orleans/LogConsistency/ILogViewAdaptorHost.cs new file mode 100644 index 0000000000..9d13c5fc17 --- /dev/null +++ b/src/Orleans/LogConsistency/ILogViewAdaptorHost.cs @@ -0,0 +1,35 @@ +using Orleans.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.LogConsistency +{ + /// + /// Interface implemented by all grains which use log-view consistency + /// It gives the log view adaptor access to grain-specific information and callbacks. + /// + /// type of the log view + /// type of log entries + public interface ILogViewAdaptorHost : IConnectionIssueListener + { + /// + /// Implementation of view transitions. + /// Any exceptions thrown will be caught and logged as a warning by . + /// + void UpdateView(TLogView view, TLogEntry entry); + + /// + /// Notifies the host grain about state changes. + /// Called by whenever the tentative or confirmed state changes. + /// Implementations may vary as to whether and how much they batch change notifications. + /// Any exceptions thrown will be caught and logged as a warning by . + /// + void OnViewChanged(bool tentative, bool confirmed); + + } + + +} diff --git a/src/Orleans/LogConsistency/IProtocolServices.cs b/src/Orleans/LogConsistency/IProtocolServices.cs new file mode 100644 index 0000000000..97c433dbd1 --- /dev/null +++ b/src/Orleans/LogConsistency/IProtocolServices.cs @@ -0,0 +1,138 @@ + +using System; +using System.Net; +using System.Runtime.Serialization; +using System.Threading.Tasks; +using System.Collections.Generic; +using Orleans.Providers; +using Orleans.Runtime; +using Orleans.MultiCluster; +using Orleans.GrainDirectory; + +namespace Orleans.LogConsistency +{ + /// + /// Functionality for use by log view adaptors that use custom consistency or replication protocols. + /// Abstracts communication between replicas of the log-consistent grain in different clusters. + /// + public interface IProtocolServices + { + /// + /// Send a message to a remote cluster. + /// + /// the message + /// the destination cluster id + /// + Task SendMessage(IProtocolMessage payload, string clusterId); + + + /// + /// The untyped reference for this grain. + /// + GrainReference GrainReference { get; } + + + /// + /// The multicluster registration strategy for this grain. + /// + IMultiClusterRegistrationStrategy RegistrationStrategy { get; } + + + /// + /// Whether this cluster is running in a multi-cluster network. + /// + /// + bool MultiClusterEnabled { get; } + + + /// + /// The id of this cluster. Returns "I" if no multi-cluster network is present. + /// + /// + string MyClusterId { get; } + + + /// + /// The current multicluster configuration of this silo + /// (as injected by the administrator) or null if none. + /// + MultiClusterConfiguration MultiClusterConfiguration { get; } + + /// + /// List of all clusters that currently appear to have at least one active + /// gateway reporting to the multi-cluster network. + /// There are no guarantees that this membership view is complete or consistent. + /// If there is no multi-cluster network, returns a list containing the single element "I". + /// + /// + IEnumerable ActiveClusters { get; } + + + void SubscribeToMultiClusterConfigurationChanges(); + + void UnsubscribeFromMultiClusterConfigurationChanges(); + + + #region Logging Functionality + + /// + /// Log an error that occurred in a log-consistency protocol. + /// + void ProtocolError(string msg, bool throwexception); + + /// + /// Log an exception that was caught in the log-consistency protocol. + /// + void CaughtException(string where, Exception e); + + /// + /// Log an exception that occurred in user code, for some callback + /// + /// The name of the callback + /// The context from which the callback was called + /// The caught exception + void CaughtUserCodeException(string callback, string where, Exception e); + + /// Output the specified message at Info log level. + void Info(string format, params object[] args); + /// Output the specified message at Verbose log level. + void Verbose(string format, params object[] args); + /// Output the specified message at Verbose2 log level. + void Verbose2(string format, params object[] args); + /// Output the specified message at Verbose3 log level. + void Verbose3(string format, params object[] args); + + #endregion + } + + + + /// + /// Exception thrown by protocol messaging layer. + /// + [Serializable] + public class ProtocolTransportException : OrleansException + { + public ProtocolTransportException() + { } + public ProtocolTransportException(string msg) + : base(msg) + { } + public ProtocolTransportException(string msg, Exception exc) + : base(msg, exc) + { } + protected ProtocolTransportException(SerializationInfo info, StreamingContext context) + : base(info, context) + { } + + public override string ToString() + { + if (InnerException != null) + return $"ProtocolTransportException: {InnerException}"; + else + return Message; + } + } + + +} diff --git a/src/Orleans/Logging/ErrorCodes.cs b/src/Orleans/Logging/ErrorCodes.cs index 9a6ec47def..d9c57b79b9 100644 --- a/src/Orleans/Logging/ErrorCodes.cs +++ b/src/Orleans/Logging/ErrorCodes.cs @@ -973,6 +973,8 @@ internal enum ErrorCode Provider_ProviderLoadedOk = ProviderManagerBase + 14, Provider_ProviderNotFound = ProviderManagerBase + 15, Provider_ProviderNotControllable = ProviderManagerBase + 16, + Provider_CatalogNoLogConsistencyProvider = ProviderManagerBase + 17, + Provider_CatalogLogConsistencyProviderAllocated = ProviderManagerBase + 18, AzureQueueBase = Runtime + 3200, AzureQueue_01 = AzureQueueBase + 1, @@ -1093,6 +1095,12 @@ internal enum ErrorCode GlobalSingleInstance_MaintainerException = GlobalSingleInstanceBase + 3, GlobalSingleInstance_MultipleOwners = GlobalSingleInstanceBase + 4, + LogConsistencyBase = Runtime + 4200, + LogConsistency_UserCodeException = LogConsistencyBase + 1, + LogConsistency_CaughtException = LogConsistencyBase + 2, + LogConsistency_ProtocolError = LogConsistencyBase + 3, + LogConsistency_ProtocolFatalError = LogConsistencyBase + 4, + } } // ReSharper restore InconsistentNaming diff --git a/src/Orleans/MultiCluster/IProtocolParticipant.cs b/src/Orleans/MultiCluster/IProtocolParticipant.cs new file mode 100644 index 0000000000..548eb81fdc --- /dev/null +++ b/src/Orleans/MultiCluster/IProtocolParticipant.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using Orleans.Concurrency; + +namespace Orleans.MultiCluster +{ + /// + /// Grain interface for grains that participate in multi-cluster-protocols. + /// + public interface IProtocolParticipant : IGrain + { + /// + /// Called when a message is received from another cluster. + /// This MUST interleave with other calls to avoid deadlocks. + /// + /// the protocol message to be delivered + /// + [AlwaysInterleave] + Task OnProtocolMessageReceived(IProtocolMessage payload); + + /// + /// Called when a configuration change notification is received. + /// + /// the next multi-cluster configuration + /// + [AlwaysInterleave] + Task OnMultiClusterConfigurationChange(MultiClusterConfiguration next); + + + /// + /// Called immediately before the user-level OnActivateAsync, on same scheduler. + /// + /// + Task ActivateProtocolParticipant(); + + /// + /// Called immediately after the user-level OnDeactivateAsync, on same scheduler. + /// + /// + Task DeactivateProtocolParticipant(); + } + + /// + /// interface to mark classes that represent protocol messages. + /// All such classes must be serializable. + /// + public interface IProtocolMessage + { + } +} diff --git a/src/Orleans/Orleans.csproj b/src/Orleans/Orleans.csproj index 78f82312f0..73b385e859 100644 --- a/src/Orleans/Orleans.csproj +++ b/src/Orleans/Orleans.csproj @@ -80,10 +80,13 @@ + + + @@ -103,6 +106,7 @@ + @@ -113,6 +117,9 @@ + + + @@ -126,7 +133,9 @@ + + @@ -157,6 +166,7 @@ + diff --git a/src/Orleans/Providers/IOrleansProvider.cs b/src/Orleans/Providers/IOrleansProvider.cs index b31b22207d..ba19683358 100644 --- a/src/Orleans/Providers/IOrleansProvider.cs +++ b/src/Orleans/Providers/IOrleansProvider.cs @@ -13,6 +13,8 @@ namespace Orleans.Providers /// /// /// + /// + public interface IProvider { /// The name of this provider instance, as given to it in the config. diff --git a/src/Orleans/Providers/IProviderRuntime.cs b/src/Orleans/Providers/IProviderRuntime.cs index b52f601ee8..c8d11ea62b 100644 --- a/src/Orleans/Providers/IProviderRuntime.cs +++ b/src/Orleans/Providers/IProviderRuntime.cs @@ -67,6 +67,15 @@ public interface IStorageProviderRuntime : IProviderRuntime // for now empty, later can add storage specific runtime capabilities. } + /// + /// Provider-facing interface for log consistency + /// + public interface ILogConsistencyProviderRuntime : IProviderRuntime + { + // for now empty, later can add provider specific runtime capabilities. + } + + /// /// Handles the invocation of the provided . /// diff --git a/src/Orleans/Runtime/Constants.cs b/src/Orleans/Runtime/Constants.cs index 3257197366..2f74d6cfe8 100644 --- a/src/Orleans/Runtime/Constants.cs +++ b/src/Orleans/Runtime/Constants.cs @@ -18,7 +18,9 @@ internal class Constants public const string ADO_INVARIANT_NAME = "AdoInvariant"; public const string DATA_CONNECTION_FOR_REMINDERS_STRING_NAME = "DataConnectionStringForReminders"; public const string ADO_INVARIANT_FOR_REMINDERS_NAME = "AdoInvariantForReminders"; - + + public const string DEFAULT_LOG_CONSISTENCY_PROVIDER_NAME = "Default"; + public const string ORLEANS_AZURE_UTILS_DLL = "OrleansAzureUtils"; public const string ORLEANS_SQL_UTILS_DLL = "OrleansSQLUtils"; @@ -42,6 +44,7 @@ internal class Constants public static readonly GrainId ClusterDirectoryServiceId = GrainId.GetSystemTargetGrainId(24); public static readonly GrainId StreamProviderManagerAgentSystemTargetId = GrainId.GetSystemTargetGrainId(25); public static readonly GrainId TestHooksSystemTargetId = GrainId.GetSystemTargetGrainId(26); + public static readonly GrainId ProtocolGatewayId = GrainId.GetSystemTargetGrainId(27); public const int PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE = 254; public const int PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE = 255; @@ -84,6 +87,7 @@ internal class Constants {MultiClusterOracleId,"MultiClusterOracle"}, {ReminderServiceId,"ReminderService"}, {TypeManagerId,"TypeManagerId"}, + {ProtocolGatewayId,"ProtocolGateway"}, {ProviderManagerSystemTargetId, "ProviderManagerSystemTarget"}, {DeploymentLoadPublisherSystemTargetId, "DeploymentLoadPublisherSystemTarget"}, }; diff --git a/src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs b/src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs new file mode 100644 index 0000000000..62b85c9c9b --- /dev/null +++ b/src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.MultiCluster; +using Orleans.Runtime; +using Orleans.Concurrency; + + +namespace Orleans.SystemTargetInterfaces +{ + /// + /// The protocol gateway is a relay that forwards incoming protocol messages from other clusters + /// to the appropriate grain in this cluster. + /// + internal interface IProtocolGateway : ISystemTarget + { + Task RelayMessage(GrainId id, IProtocolMessage payload); + } + +} diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs b/src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs new file mode 100644 index 0000000000..6e3b2f2dc7 --- /dev/null +++ b/src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.CustomVersionedStateStorage +{ + /// + /// The storage interface exposed by grains that want to use the CustomVersionedStateStorage log-consistency provider + /// The type for the state of the grain. + /// The type for delta objects that represent updates to the state. + /// + public interface ICustomStorageInterface + { + /// + /// Reads the current state and version from storage + /// (note that the state object may be mutated by the provider, so it must not be shared). + /// + /// the version number and a state object. + Task> ReadStateFromStorageAsync(); + + /// + /// Applies the given array of deltas to storage, if the version in storage matches the expected version. + /// Otherwise, does nothing. If successful, the version of storage increases by the number of deltas. + /// + /// true if the deltas were applied, false otherwise + Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int expectedversion); + } + +} diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs new file mode 100644 index 0000000000..c9700bc030 --- /dev/null +++ b/src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Runtime; +using Orleans.Providers; +using Orleans.Storage; + +namespace Orleans.EventSourcing.CustomVersionedStateStorage +{ + /// + /// A log-consistency provider that relies on grain-specific custom code for + /// loading states from storage, and writing deltas to storage. + /// Grains that wish to use this provider must implement the + /// interface, to define how state is read and how deltas are written. + /// If the provider attribute "PrimaryCluster" is supplied in the provider configuration, then only the specified cluster + /// accesses storage, and other clusters may not issue updates. + /// + public class LogConsistencyProvider : ILogConsistencyProvider + { + /// + public string Name { get; private set; } + + /// + public Logger Log { get; private set; } + + private static int counter; + private int id; + + /// + /// Specifies a clusterid of the primary cluster from which to access storage exclusively, null if + /// storage should be accessed direcly from all clusters. + /// + public string PrimaryCluster { get; private set; } + + /// + public bool UsesStorageProvider { get { return false; } } + + /// + /// Gets a unique name for this provider, suited for logging. + /// + protected virtual string GetLoggerName() + { + return string.Format("LogViews.{0}.{1}", GetType().Name, id); + } + + /// + /// Init function + /// + /// provider name + /// provider runtime, see + /// provider configuration + public Task Init(string name, IProviderRuntime providerRuntime, IProviderConfiguration config) + { + Name = name; + id = Interlocked.Increment(ref counter); + PrimaryCluster = config.GetProperty("PrimaryCluster", null); + + Log = providerRuntime.GetLogger(GetLoggerName()); + Log.Info("Init (Severity={0}) PrimaryCluster={1}", Log.SeverityLevel, + string.IsNullOrEmpty(PrimaryCluster) ? "(none specified)" : PrimaryCluster); + + return TaskDone.Done; + } + + /// + public Task Close() + { + return TaskDone.Done; + } + + /// + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, TView initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + where TView : class, new() + where TEntry : class + { + return new CustomStorageAdaptor(hostgrain, initialstate, services, PrimaryCluster); + } + + } + +} \ No newline at end of file diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs new file mode 100644 index 0000000000..8476ff342b --- /dev/null +++ b/src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs @@ -0,0 +1,377 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Runtime; +using Orleans.Storage; +using Orleans.MultiCluster; +using Orleans.EventSourcing.Protocols; + +namespace Orleans.EventSourcing.CustomVersionedStateStorage +{ + /// + /// A log consistency adaptor that uses the user-provided storage interface . + /// This interface must be implemented by any grain that uses this log view adaptor. + /// + /// log view type + /// log entry type + internal class CustomStorageAdaptor : PrimaryBasedLogViewAdaptor> + where TLogView : class, new() + where TLogEntry : class + { + /// + /// Initialize a new instance of CustomStorageAdaptor class + /// + public CustomStorageAdaptor(ILogViewAdaptorHost host, TLogView initialState, + IProtocolServices services, string primaryCluster) + : base(host, initialState, services) + { + if (!(host is ICustomStorageInterface)) + throw new BadProviderConfigException("Must implement ICustomStorageInterface for CustomStorageLogView provider"); + this.primaryCluster = primaryCluster; + } + + private string primaryCluster; + + private const int slowpollinterval = 10000; + + private TLogView cached; + private int version; + + /// + protected override TLogView LastConfirmedView() + { + return cached; + } + + /// + protected override int GetConfirmedVersion() + { + return version; + } + + /// + protected override void InitializeConfirmedView(TLogView initialstate) + { + cached = initialstate; + version = 0; + } + + /// + protected override bool SupportSubmissions + { + get + { + return MayAccessStorage(); + } + } + + private bool MayAccessStorage() + { + return (!Services.MultiClusterEnabled) + || string.IsNullOrEmpty(primaryCluster) + || primaryCluster == Services.MyClusterId; + } + + /// + protected override SubmissionEntry MakeSubmissionEntry(TLogEntry entry) + { + // no special tagging is required, thus we create a plain submission entry + return new SubmissionEntry() { Entry = entry }; + } + + [Serializable] + private class ReadRequest : IProtocolMessage + { + public int KnownVersion { get; set; } + } + [Serializable] + private class ReadResponse : IProtocolMessage + { + public int Version { get; set; } + + public ViewType Value { get; set; } + } + + /// + protected override Task OnMessageReceived(IProtocolMessage payload) + { + var request = (ReadRequest) payload; + + if (! MayAccessStorage()) + throw new ProtocolTransportException("message destined for primary cluster ended up elsewhere (inconsistent configurations?)"); + + var response = new ReadResponse() { Version = version }; + + // optimization: include value only if version is newer + if (version > request.KnownVersion) + response.Value = cached; + + return Task.FromResult(response); + } + + /// + protected override async Task ReadAsync() + { + enter_operation("ReadAsync"); + + while (true) + { + try + { + if (MayAccessStorage()) + { + // read from storage + var result = await ((ICustomStorageInterface)Host).ReadStateFromStorageAsync(); + version = result.Key; + cached = result.Value; + } + else + { + // read from primary cluster + var request = new ReadRequest() { KnownVersion = version }; + if (!Services.MultiClusterConfiguration.Clusters.Contains(primaryCluster)) + throw new ProtocolTransportException("the specified primary cluster is not in the multicluster configuration"); + var response =(ReadResponse) await Services.SendMessage(request, primaryCluster); + if (response.Version > request.KnownVersion) + { + version = response.Version; + cached = response.Value; + } + } + + Services.Verbose("read success v{0}", version); + + LastPrimaryIssue.Resolve(Host, Services); + + break; // successful + } + catch (Exception e) + { + // unwrap inner exception that was forwarded - helpful for debugging + if ((e as ProtocolTransportException)?.InnerException != null) + e = ((ProtocolTransportException)e).InnerException; + + LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + + await LastPrimaryIssue.DelayBeforeRetry(); + } + + exit_operation("ReadAsync"); + } + + /// + protected override async Task WriteAsync() + { + enter_operation("WriteAsync"); + + var updates = GetCurrentBatchOfUpdates().Select(submissionentry => submissionentry.Entry).ToList(); + bool writesuccessful = false; + bool transitionssuccessful = false; + + try + { + writesuccessful = await ((ICustomStorageInterface) Host).ApplyUpdatesToStorageAsync(updates, version); + + LastPrimaryIssue.Resolve(Host, Services); + } + catch (Exception e) + { + // unwrap inner exception that was forwarded - helpful for debugging + if ((e as ProtocolTransportException)?.InnerException != null) + e = ((ProtocolTransportException)e).InnerException; + + LastPrimaryIssue.Record(new UpdatePrimaryFailed() { Exception = e }, Host, Services); + } + + if (writesuccessful) + { + Services.Verbose("write ({0} updates) success v{1}", updates.Count, version + updates.Count); + + // now we update the cached state by applying the same updates + // in case we encounter any exceptions we will re-read the whole state from storage + try + { + foreach (var u in updates) + { + version++; + Host.UpdateView(this.cached, u); + } + + transitionssuccessful = true; + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(WriteAsync), e); + } + } + + if (!writesuccessful || !transitionssuccessful) + { + Services.Verbose("{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue); + + while (true) // be stubborn until we can re-read the state from storage + { + await LastPrimaryIssue.DelayBeforeRetry(); + + try + { + var result = await ((ICustomStorageInterface)Host).ReadStateFromStorageAsync(); + version = result.Key; + cached = result.Value; + + Services.Verbose("read success v{0}", version); + + LastPrimaryIssue.Resolve(Host, Services); + + break; + } + catch (Exception e) + { + // unwrap inner exception that was forwarded - helpful for debugging + if ((e as ProtocolTransportException)?.InnerException != null) + e = ((ProtocolTransportException)e).InnerException; + + LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + } + } + + // broadcast notifications to all other clusters + // TODO: send state instead of updates, if smaller + if (writesuccessful) + BroadcastNotification(new UpdateNotificationMessage() + { + Version = version, + Updates = updates, + }); + + exit_operation("WriteAsync"); + + return writesuccessful ? updates.Count : 0; + } + + /// + /// Describes a connection issue that occurred when updating the primary storage. + /// + [Serializable] + public class UpdatePrimaryFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"update primary failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// Describes a connection issue that occurred when reading from the primary storage. + /// + [Serializable] + public class ReadFromPrimaryFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"read from primary failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// A notification message that is sent to remote instances of this grain after the primary has been + /// updated, to let them know the latest version. Contains all the updates that were applied. + /// + [Serializable] + protected class UpdateNotificationMessage : INotificationMessage + { + /// + public int Version { get; set; } + + /// The list of updates that were applied. + public List Updates { get; set; } + + /// + /// A representation of this notification message suitable for tracing. + /// + public override string ToString() + { + return string.Format("v{0} ({1} updates)", Version, Updates.Count); + } + } + + private SortedList notifications = new SortedList(); + + /// + protected override void OnNotificationReceived(INotificationMessage payload) + { + var um = payload as UpdateNotificationMessage; + if (um != null) + notifications.Add(um.Version - um.Updates.Count, um); + else + base.OnNotificationReceived(payload); + } + + /// + protected override void ProcessNotifications() + { + + // discard notifications that are behind our already confirmed state + while (notifications.Count > 0 && notifications.ElementAt(0).Key < version) + { + Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + notifications.RemoveAt(0); + } + + // process notifications that reflect next global version + while (notifications.Count > 0 && notifications.ElementAt(0).Key == version) + { + var updatenotification = notifications.ElementAt(0).Value; + notifications.RemoveAt(0); + + // Apply all operations in pending + foreach (var u in updatenotification.Updates) + try + { + Host.UpdateView(cached, u); + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(ProcessNotifications), e); + } + + version = updatenotification.Version; + + Services.Verbose("notification success ({0} updates) v{1}", updatenotification.Updates.Count, version); + } + + Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + + base.ProcessNotifications(); + + } + + [Conditional("DEBUG")] + private void enter_operation(string name) + { + Services.Verbose2("/-- enter {0}", name); + } + + [Conditional("DEBUG")] + private void exit_operation(string name) + { + Services.Verbose2("\\-- exit {0}", name); + } + + } +} diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index ff3b79e53e..87a2019173 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -1,28 +1,340 @@ +using Orleans.Concurrency; +using Orleans.MultiCluster; +using Orleans.LogConsistency; using System; +using System.Linq; +using System.Collections.Generic; using System.Threading.Tasks; - +using Orleans.Storage; namespace Orleans.EventSourcing { /// - /// The base class for all grain classes that have event-sourced state. + /// A base class for log-consistent grains using standard event-sourcing terminology. + /// All operations are reentrancy-safe. + /// The type for the grain state, i.e. the aggregate view of the event log. + /// + public abstract class JournaledGrain : JournaledGrain + where TGrainState : class, new() + { } + + + /// + /// A base class for log-consistent grains using standard event-sourcing terminology. + /// All operations are reentrancy-safe. + /// The type for the grain state, i.e. the aggregate view of the event log. + /// The common base class for the events /// - public class JournaledGrain : Grain - where TGrainState : JournaledGrainState, new() + public abstract class JournaledGrain : + LogConsistentGrainBase, + ILogConsistentGrain, + IProtocolParticipant, + ILogViewAdaptorHost + where TGrainState : class, new() + where TEventBase: class { + protected JournaledGrain() { } + /// - /// Helper method for raising events, applying them to TGrainState and optionally committing to storage + /// Raise an event. /// /// Event to raise - /// Whether or not the event needs to be immediately committed to storage /// - protected Task RaiseStateEvent(TEvent @event, bool commit = true) - where TEvent : class + protected virtual void RaiseEvent(TEvent @event) + where TEvent : TEventBase + { + if (@event == null) throw new ArgumentNullException("event"); + + LogViewAdaptor.Submit(@event); + } + + /// + /// Raise multiple events, as an atomic sequence. + /// + /// Events to raise + /// + protected virtual void RaiseEvents(IEnumerable events) + where TEvent : TEventBase + { + if (events == null) throw new ArgumentNullException("events"); + + LogViewAdaptor.SubmitRange((IEnumerable) events); + } + + + /// + /// Raise an event conditionally. + /// Succeeds only if there are no conflicts, that is, no other events were raised in the meantime. + /// + /// Event to raise + /// true if successful, false if there was a conflict. + protected virtual Task RaiseConditionalEvent(TEvent @event) + where TEvent : TEventBase { if (@event == null) throw new ArgumentNullException("event"); - State.AddEvent(@event); - return commit ? WriteStateAsync() : TaskDone.Done; + return LogViewAdaptor.TryAppend(@event); } + + + /// + /// Raise multiple events, as an atomic sequence, conditionally. + /// Succeeds only if there are no conflicts, that is, no other events were raised in the meantime. + /// + /// Events to raise + /// true if successful, false if there was a conflict. + protected virtual Task RaiseConditionalEvents(IEnumerable events) + where TEvent : TEventBase + { + if (events == null) throw new ArgumentNullException("events"); + return LogViewAdaptor.TryAppendRange((IEnumerable) events); + } + + /// + /// The current state (includes both confirmed and unconfirmed events). + /// + protected TGrainState State + { + get { return this.LogViewAdaptor.TentativeView; } + } + + /// + /// The version of the state. + /// Always equal to the confirmed version plus the number of unconfirmed events. + /// + protected int Version + { + get { return this.LogViewAdaptor.ConfirmedVersion + this.LogViewAdaptor.UnconfirmedSuffix.Count(); } + } + + /// + /// Called whenever the current state may have changed due to local or remote events. + /// Override this to react to changes of the state. + /// + protected virtual void OnStateChanged() + { + } + + /// + /// The current confirmed state (includes only confirmed events). + /// + protected TGrainState ConfirmedState + { + get { return this.LogViewAdaptor.ConfirmedView; } + } + + /// + /// The version of the confirmed state. + /// Always equal to the number of confirmed events. + /// + protected int ConfirmedVersion + { + get { return this.LogViewAdaptor.ConfirmedVersion; } + } + + + + /// + /// Called after the confirmed state may have changed (i.e. the confirmed version number is larger). + /// Override this to react to changes of the confirmed state. + /// + protected virtual void OnConfirmedStateChanged() + { + // overridden by journaled grains that want to react to state changes + } + + + /// + /// Waits until all previously raised events have been confirmed. + /// + /// + protected Task ConfirmEvents() + { + return LogViewAdaptor.ConfirmSubmittedEntriesAsync(); + + } + + /// + /// Retrieves all events now, and confirms all previously raised events. + /// Effectively, this enforces synchronization with the global state. + /// + /// + protected Task FetchAllEventsNow() + { + return LogViewAdaptor.SynchronizeNowAsync(); + } + + + /// + /// Called when the underlying persistence or replication protocol is running into some sort of connection trouble. + /// Override this to monitor the health of the log-consistency protocol and/or + /// to customize retry delays. + /// Any exceptions thrown are caught and logged by the . + /// + /// The time to wait before retrying + protected virtual void OnConnectionIssue(ConnectionIssue issue) + { + } + + /// + /// Called when a previously reported connection issue has been resolved. + /// Override this to monitor the health of the log-consistency protocol. + /// Any exceptions thrown are caught and logged by the . + /// + protected virtual void OnConnectionIssueResolved(ConnectionIssue issue) + { + } + + + /// + protected IEnumerable UnresolvedConnectionIssues + { + get + { + return LogViewAdaptor.UnresolvedConnectionIssues; + } + } + + /// + protected void EnableStatsCollection() + { + LogViewAdaptor.EnableStatsCollection(); + } + + /// + protected void DisableStatsCollection() + { + LogViewAdaptor.DisableStatsCollection(); + } + + /// + protected LogConsistencyStatistics GetStats() + { + return LogViewAdaptor.GetStats(); + } + + + + /// + /// Defines how to apply events to the state. Unless it is overridden in the subclass, it calls + /// a dynamic "Apply" function on the state, with the event as a parameter. + /// All exceptions thrown by this method are caught and logged by the log view provider. + /// Override this to customize how to transition the state for a given event. + /// + /// + /// + protected virtual void TransitionState(TGrainState state, TEventBase @event) + { + dynamic s = state; + dynamic e = @event; + s.Apply(e); + } + + + + #region internal plumbing + + /// + /// Adaptor for log consistency protocol. + /// Is installed by the log-consistency provider. + /// + internal ILogViewAdaptor LogViewAdaptor { get; private set; } + + /// + /// Called right after grain is constructed, to install the adaptor. + /// The log-consistency provider contains a factory method that constructs the adaptor with chosen types for this grain + /// + void ILogConsistentGrain.InstallAdaptor(ILogViewAdaptorFactory factory, object initialState, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + { + // call the log consistency provider to construct the adaptor, passing the type argument + LogViewAdaptor = factory.MakeLogViewAdaptor(this, (TGrainState)initialState, graintypename, storageProvider, services); + } + + /// + /// If there is no log-consistency provider specified, store versioned state using default storage provider + /// + ILogViewAdaptorFactory ILogConsistentGrain.DefaultAdaptorFactory + { + get + { + return new VersionedStateStorage.DefaultAdaptorFactory(); + } + } + + /// + /// called by adaptor to update the view when entries are appended. + /// + /// log view + /// log entry + void ILogViewAdaptorHost.UpdateView(TGrainState view, TEventBase entry) + { + TransitionState(view, entry); + } + + /// + /// Notify log view adaptor of activation + /// + async Task IProtocolParticipant.ActivateProtocolParticipant() + { + await LogViewAdaptor.Activate(); + } + + /// + /// Notify log view adaptor of deactivation + /// + Task IProtocolParticipant.DeactivateProtocolParticipant() + { + return LogViewAdaptor.Deactivate(); + } + + /// + /// Receive a protocol message from other clusters, passed on to log view adaptor. + /// + [AlwaysInterleave] + Task IProtocolParticipant.OnProtocolMessageReceived(IProtocolMessage payload) + { + return LogViewAdaptor.OnProtocolMessageReceived(payload); + } + + /// + /// Receive a configuration change, pass on to log view adaptor. + /// + [AlwaysInterleave] + Task IProtocolParticipant.OnMultiClusterConfigurationChange(MultiCluster.MultiClusterConfiguration next) + { + return LogViewAdaptor.OnMultiClusterConfigurationChange(next); + } + + /// + /// called by adaptor on state change. + /// + void ILogViewAdaptorHost.OnViewChanged(bool tentative, bool confirmed) + { + if (tentative) + OnStateChanged(); + if (confirmed) + OnConfirmedStateChanged(); + } + + /// + /// called by adaptor on connection issues. + /// + void IConnectionIssueListener.OnConnectionIssue(ConnectionIssue connectionIssue) + { + OnConnectionIssue(connectionIssue); + } + + /// + /// called by adaptor when a connection issue is resolved. + /// + void IConnectionIssueListener.OnConnectionIssueResolved(ConnectionIssue connectionIssue) + { + OnConnectionIssueResolved(connectionIssue); + } + + + #endregion + } + } diff --git a/src/OrleansEventSourcing/JournaledGrainState.cs b/src/OrleansEventSourcing/JournaledGrainState.cs deleted file mode 100644 index e7840fa47c..0000000000 --- a/src/OrleansEventSourcing/JournaledGrainState.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; - - -namespace Orleans.EventSourcing -{ - /// - /// Base class for event-sourced grain state classes. - /// - public abstract class JournaledGrainState - where TGrainState : JournaledGrainState, new() - { - private List events = new List(); - protected TGrainState State; - - protected JournaledGrainState() - { - } - - public IEnumerable Events - { - get { return events.AsReadOnly(); } - } - - public int Version { get; private set; } - - public void AddEvent(TEvent @event) - where TEvent : class - { - events.Add(@event); - - StateTransition(@event); - - Version++; - } - - public void SetAll(TGrainState value) - { - State = value; - foreach (var @event in Events) - StateTransition(@event); - } - - private void StateTransition(TEvent @event) - where TEvent : class - { - dynamic me = this; - - try - { - me.Apply(@event); - } - catch(MissingMethodException) - { - OnMissingStateTransition(@event); - } - } - - protected virtual void OnMissingStateTransition(object @event) - { - // Log - } - } -} diff --git a/src/OrleansEventSourcing/OrleansEventSourcing.csproj b/src/OrleansEventSourcing/OrleansEventSourcing.csproj index b9211001b8..7f93c6b209 100644 --- a/src/OrleansEventSourcing/OrleansEventSourcing.csproj +++ b/src/OrleansEventSourcing/OrleansEventSourcing.csproj @@ -45,8 +45,19 @@ Properties\GlobalAssemblyInfo.cs + + + + + - + + + + + + + diff --git a/src/OrleansEventSourcing/Protocols/ConnectionIssues.cs b/src/OrleansEventSourcing/Protocols/ConnectionIssues.cs new file mode 100644 index 0000000000..dc936b5c3c --- /dev/null +++ b/src/OrleansEventSourcing/Protocols/ConnectionIssues.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.LogConsistency; + +namespace Orleans.EventSourcing.Protocols +{ + + /// + /// Describes a connection issue that occurred when sending update notifications to remote instances. + /// + [Serializable] + public class NotificationFailed : ConnectionIssue + { + /// The destination cluster which we could not reach successfully. + public string RemoteCluster { get; set; } + + /// The exception we caught when trying to send the notification message. + public Exception Exception { get; set; } + + /// + public override TimeSpan ComputeRetryDelay(TimeSpan? previous) + { + if (NumberOfConsecutiveFailures < 3) return TimeSpan.FromMilliseconds(1); + else if (NumberOfConsecutiveFailures < 1000) return TimeSpan.FromSeconds(30); + else return TimeSpan.FromMinutes(1); + } + } + + /// + /// Describes a connection issue that occurred when communicating with primary storage. + /// + [Serializable] + public class PrimaryOperationFailed : ConnectionIssue + { + /// + /// The exception that was caught when communicating with the primary. + /// + public Exception Exception { get; set; } + + /// + public override TimeSpan ComputeRetryDelay(TimeSpan? previous) + { + // after first fail do not backoff yet... keep it at zero + if (previous == null) + { + return TimeSpan.Zero; + } + + var backoff = previous.Value.TotalMilliseconds; + + if (random == null) + random = new Random(); + + // grows exponentially up to slowpoll interval + if (previous.Value.TotalMilliseconds < slowpollinterval) + backoff = (int)((backoff + random.Next(5, 15)) * 1.5); + + // during slowpoll, slightly randomize + if (backoff > slowpollinterval) + backoff = slowpollinterval + random.Next(1, 200); + + return TimeSpan.FromMilliseconds(backoff); + } + + + [ThreadStatic] + static Random random; + + private const int slowpollinterval = 10000; + } + + + + + + + + +} diff --git a/src/OrleansEventSourcing/Protocols/NotificationMessage.cs b/src/OrleansEventSourcing/Protocols/NotificationMessage.cs new file mode 100644 index 0000000000..83c91b94aa --- /dev/null +++ b/src/OrleansEventSourcing/Protocols/NotificationMessage.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.MultiCluster; + +namespace Orleans.EventSourcing.Protocols +{ + /// + /// Base class for notification messages that are sent by log view adaptors to other + /// clusters, after updating the log. All subclasses must be serializable. + /// + public interface INotificationMessage : IProtocolMessage + { + ///The version number. + int Version { get; } + + // a log-consistency provider can subclass this to add more information + // for example, the log entries that were appended, or the view + } + + /// A simple notification message containing only the version. + [Serializable] + public class VersionNotificationMessage : INotificationMessage + { + /// + public int Version { get; set; } + } + + + /// A notification message containing a batch of notification messages. + [Serializable] + public class BatchedNotificationMessage : INotificationMessage + { + /// The notification messages contained in this batch. + public List Notifications { get; set; } + + /// The version number - for a batch, this is the maximum version contained. + public int Version { + get + { + return Notifications.Aggregate(0, (v, m) => Math.Max(v, m.Version)); + } + } + } + +} diff --git a/src/OrleansEventSourcing/Protocols/NotificationTracker.cs b/src/OrleansEventSourcing/Protocols/NotificationTracker.cs new file mode 100644 index 0000000000..3e2119f600 --- /dev/null +++ b/src/OrleansEventSourcing/Protocols/NotificationTracker.cs @@ -0,0 +1,226 @@ +using Orleans.LogConsistency; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.Protocols +{ + + /// + /// Helper class for tracking notifications that a grain sends to other clusters after updating the log. + /// + internal class NotificationTracker + { + internal IProtocolServices services; + internal IConnectionIssueListener listener; + internal int maxNotificationBatchSize; + + private Dictionary sendWorkers; + + public NotificationTracker(IProtocolServices services, IEnumerable remoteInstances, int maxNotificationBatchSize, IConnectionIssueListener listener) + { + this.services = services; + this.listener = listener; + sendWorkers = new Dictionary(); + this.maxNotificationBatchSize = maxNotificationBatchSize; + + foreach (var x in remoteInstances) + { + services.Verbose("Now sending notifications to {0}", x); + sendWorkers.Add(x, new NotificationWorker(this, x)); + } + } + + public void BroadcastNotification(INotificationMessage msg, string exclude = null) + { + foreach (var kvp in sendWorkers) + { + if (kvp.Key != exclude) + { + var w = kvp.Value; + w.Enqueue(msg); + } + } + } + + /// + /// returns unresolved connection issues observed by the workers + /// + public IEnumerable UnresolvedConnectionIssues + { + get + { + return sendWorkers.Values.Select(sw => sw.LastConnectionIssue.Issue).Where(i => i != null); + } + } + + /// + /// Update the multicluster configuration (change who to send notifications to) + /// + public void UpdateNotificationTargets(IReadOnlyList remoteInstances) + { + var removed = sendWorkers.Keys.Except(remoteInstances); + foreach (var x in removed) + { + services.Verbose("No longer sending notifications to {0}", x); + sendWorkers[x].Done = true; + sendWorkers.Remove(x); + } + + var added = remoteInstances.Except(sendWorkers.Keys); + foreach (var x in added) + { + if (x != services.MyClusterId) + { + services.Verbose("Now sending notifications to {0}", x); + sendWorkers.Add(x, new NotificationWorker(this, x)); + } + } + } + + + public enum NotificationQueueState + { + Empty, + Single, + Batch, + VersionOnly + } + + /// + /// Asynchronous batch worker that sends notfications to a particular cluster. + /// + public class NotificationWorker : BatchWorker + { + private NotificationTracker tracker; + private string clusterId; + + /// + /// Queue messages + /// + public INotificationMessage QueuedMessage = null; + /// + /// Queue state + /// + public NotificationQueueState QueueState = NotificationQueueState.Empty; + /// + /// Last exception + /// + public RecordedConnectionIssue LastConnectionIssue; + /// + /// Number of consecutive failures + /// + public int NumConsecutiveFailures; + /// + /// Is current task done or not + /// + public bool Done; + + /// + /// Initialize a new instance of NotificationWorker class + /// + public NotificationWorker(NotificationTracker tracker, string clusterId) + { + this.tracker = tracker; + this.clusterId = clusterId; + } + + /// + /// Enqueue method + /// + /// The message to enqueue + public void Enqueue(INotificationMessage msg) + { + switch (QueueState) + { + case (NotificationQueueState.Empty): + { + QueuedMessage = msg; + QueueState = NotificationQueueState.Single; + break; + } + case (NotificationQueueState.Single): + { + var m = new List(); + m.Add(QueuedMessage); + m.Add(msg); + QueuedMessage = new BatchedNotificationMessage() { Notifications = m }; + QueueState = NotificationQueueState.Batch; + break; + } + case (NotificationQueueState.Batch): + { + var batchmsg = (BatchedNotificationMessage)QueuedMessage; + if (batchmsg.Notifications.Count < tracker.maxNotificationBatchSize) + { + batchmsg.Notifications.Add(msg); + break; + } + else + { + // keep only a version notification + QueuedMessage = new VersionNotificationMessage() { Version = msg.Version }; + QueueState = NotificationQueueState.VersionOnly; + break; + } + } + case (NotificationQueueState.VersionOnly): + { + ((VersionNotificationMessage)QueuedMessage).Version = msg.Version; + QueueState = NotificationQueueState.VersionOnly; + break; + } + } + Notify(); + } + + + + protected override async Task Work() + { + if (Done) return; // has been terminated - now garbage. + + // if we had issues sending last time, wait a bit before retrying + await LastConnectionIssue.DelayBeforeRetry(); + + // take all of current queue + var msg = QueuedMessage; + var state = QueueState; + + if (state == NotificationQueueState.Empty) + return; + + // queue is now empty (and may grow while this worker is doing awaits) + QueuedMessage = null; + QueueState = NotificationQueueState.Empty; + + // try to send it + try + { + await tracker.services.SendMessage(msg, clusterId); + + // notification was successful + tracker.services.Verbose("Sent notification to cluster {0}: {1}", clusterId, msg); + + LastConnectionIssue.Resolve(tracker.listener, tracker.services); + } + catch (Exception e) + { + tracker.services.Info("Could not send notification to cluster {0}: {1}", clusterId, e); + + LastConnectionIssue.Record( + new NotificationFailed() { RemoteCluster = clusterId, Exception = e }, + tracker.listener, tracker.services); + + // next time, send only version (this is an optimization that + // avoids the queueing and sending of lots of data when there are errors observed) + QueuedMessage = new VersionNotificationMessage() { Version = msg.Version }; + QueueState = NotificationQueueState.VersionOnly; + Notify(); + } + } + } + } +} diff --git a/src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs new file mode 100644 index 0000000000..6af513c01e --- /dev/null +++ b/src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs @@ -0,0 +1,895 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.LogConsistency; +using System.Diagnostics; +using Orleans.Serialization; +using Orleans.MultiCluster; + +namespace Orleans.EventSourcing.Protocols +{ + /// + /// A general template for constructing log view adaptors that are based on + /// a sequentially read and written primary. We use this to construct + /// a variety of different log-consistency providers, all following the same basic pattern + /// (read and write latest view from/to primary, and send notifications after writing). + /// + /// Note that the log itself is transient, i.e. not actually saved to storage - only the latest view and some + /// metadata (the log position, and write flags) is stored in the primary. + /// It is safe to interleave calls to this adaptor (using grain scheduler only, of course). + /// + /// + /// Subclasses override ReadAsync and WriteAsync to read from / write to primary. + /// Calls to the primary are serialized, i.e. never interleave. + /// + /// + /// The user-defined view of the log + /// The type of the log entries + /// The type of submission entries stored in pending queue + public abstract class PrimaryBasedLogViewAdaptor : ILogViewAdaptor + where TLogView : class, new() + where TLogEntry : class + where TSubmissionEntry : SubmissionEntry + { + + #region interface to subclasses that implement specific providers + + + /// + /// Set confirmed view the initial value (a view of the empty log) + /// + protected abstract void InitializeConfirmedView(TLogView initialstate); + + /// + /// Read cached global state. + /// + protected abstract TLogView LastConfirmedView(); + + /// + /// Read version of cached global state. + /// + protected abstract int GetConfirmedVersion(); + + /// + /// Read the latest primary state. Must block/retry until successful. + /// Should not throw exceptions, but record them in + /// + /// + protected abstract Task ReadAsync(); + + /// + /// Apply pending entries to the primary. Must block/retry until successful. + /// Should not throw exceptions, but record them in + /// + protected abstract Task WriteAsync(); + + /// + /// Create a submission entry for the submitted log entry. + /// Using a type parameter so we can add protocol-specific info to this class. + /// + /// + protected abstract TSubmissionEntry MakeSubmissionEntry(TLogEntry entry); + + /// + /// Whether this cluster supports submitting updates + /// + protected virtual bool SupportSubmissions { get { return true; } } + + /// + /// Handle protocol messages. + /// + protected virtual Task OnMessageReceived(IProtocolMessage payload) + { + // subclasses that define custom protocol messages must override this + throw new NotImplementedException(); + } + + /// + /// Handle notification messages. Override this to handle notification subtypes. + /// + protected virtual void OnNotificationReceived(INotificationMessage payload) + { + var msg = payload as VersionNotificationMessage; + if (msg != null) + { + if (msg.Version > lastVersionNotified) + lastVersionNotified = msg.Version; + return; + } + + var batchmsg = payload as BatchedNotificationMessage; + if (batchmsg != null) + { + foreach (var bm in batchmsg.Notifications) + OnNotificationReceived(bm); + return; + } + + // subclass should have handled this in override + throw new ProtocolTransportException(string.Format("message type {0} not handled by OnNotificationReceived", payload.GetType().FullName)); + } + + /// + /// The last version we have been notified of + /// + private int lastVersionNotified; + + /// + /// Process stored notifications during worker cycle. Override to handle notification subtypes. + /// + protected virtual void ProcessNotifications() + { + if (lastVersionNotified > this.GetConfirmedVersion()) + { + Services.Verbose("force refresh because of version notification v{0}", lastVersionNotified); + needRefresh = true; + } + } + + /// + /// Merge two notification messages, for batching. Override to handle notification subtypes. + /// + protected virtual INotificationMessage Merge(INotificationMessage earliermessage, INotificationMessage latermessage) + { + return new VersionNotificationMessage() + { + Version = latermessage.Version + }; + } + + /// + /// Called when configuration of the multicluster is changing. + /// + protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) + { + Configuration = next; + return TaskDone.Done; + } + + /// + /// The grain that is using this adaptor. + /// + protected ILogViewAdaptorHost Host { get; private set; } + + /// + /// The runtime services required for implementing notifications between grain instances in different cluster. + /// + protected IProtocolServices Services { get; private set; } + + /// + /// The current multi-cluster configuration for this grain instance. + /// + protected MultiClusterConfiguration Configuration { get; set; } + + /// + /// Tracks notifications sent. Created lazily since many copies will never need to send notifications. + /// + private NotificationTracker notificationTracker; + + private const int max_notification_batch_size = 10000; + + /// + /// Construct an instance, for the given parameters. + /// + protected PrimaryBasedLogViewAdaptor(ILogViewAdaptorHost host, + TLogView initialstate, IProtocolServices services) + { + Debug.Assert(host != null && services != null && initialstate != null); + this.Host = host; + this.Services = services; + InitializeConfirmedView(initialstate); + worker = new BatchWorkerFromDelegate(() => Work()); + } + + /// + public virtual async Task Activate() + { + Services.Verbose2("Activation Started"); + + Services.SubscribeToMultiClusterConfigurationChanges(); + + // initial load happens async + KickOffInitialRead().Ignore(); + + var latestconf = Services.MultiClusterConfiguration; + if (latestconf != null) + await OnMultiClusterConfigurationChange(latestconf); + + Services.Verbose2("Activation Complete"); + } + + private async Task KickOffInitialRead() + { + needInitialRead = true; + // kick off notification for initial read cycle with a bit of delay + // so that we don't do this several times if user does strong sync + await Task.Delay(TimeSpan.FromMilliseconds(10)); + Services.Verbose2("Notify ({0})", nameof(KickOffInitialRead)); + worker.Notify(); + } + + /// + public virtual async Task Deactivate() + { + Services.Verbose2("Deactivation Started"); + + while (!worker.IsIdle()) + { + await worker.WaitForCurrentWorkToBeServiced(); + } + + Services.UnsubscribeFromMultiClusterConfigurationChanges(); + + Services.Verbose2("Deactivation Complete"); + } + + + + #endregion + + // the currently submitted, unconfirmed entries. + private readonly List pending = new List(); + + + /// called at beginning of WriteAsync to the current tentative state + protected TLogView CopyTentativeState() + { + var state = TentativeView; + tentativeStateInternal = null; // to avoid aliasing + return state; + } + /// called at beginning of WriteAsync to the current batch of updates + protected TSubmissionEntry[] GetCurrentBatchOfUpdates() + { + return pending.ToArray(); // must use a copy + } + /// called at beginning of WriteAsync to get current number of pending updates + protected int GetNumberPendingUpdates() + { + return pending.Count; + } + + /// + /// Tentative State. Represents Stable State + effects of pending updates. + /// Computed lazily (null if not in use) + /// + private TLogView tentativeStateInternal; + + /// + /// A flag that indicates to the worker that the client wants to refresh the state + /// + private bool needRefresh; + + /// + /// A flag that indicates that we have not read global state at all yet, and should do so + /// + private bool needInitialRead; + + /// + /// Background worker which asynchronously sends operations to the leader + /// + private BatchWorker worker; + + + + + /// statistics gathering. Is null unless stats collection is turned on. + protected LogConsistencyStatistics stats = null; + + + /// For use by protocols. Determines if this cluster is part of the configured multicluster. + protected bool IsMyClusterJoined() + { + return (Configuration != null && Configuration.Clusters.Contains(Services.MyClusterId)); + } + + /// + /// Block until this cluster is joined to the multicluster. + /// + protected async Task EnsureClusterJoinedAsync() + { + while (!IsMyClusterJoined()) + { + Services.Verbose("Waiting for join"); + await Task.Delay(5000); + } + } + /// + /// Wait until this cluster has received a configuration that is at least as new as timestamp + /// + protected async Task GetCaughtUpWithConfigurationAsync(DateTime adminTimestamp) + { + while (Configuration == null || Configuration.AdminTimestamp < adminTimestamp) + { + Services.Verbose("Waiting for config {0}", adminTimestamp); + + await Task.Delay(5000); + } + } + + + + + + #region Interface + + /// + public void Submit(TLogEntry logEntry) + { + if (!SupportSubmissions) + throw new InvalidOperationException("provider does not support submissions on cluster " + Services.MyClusterId); + + if (stats != null) stats.EventCounters["SubmitCalled"]++; + + Services.Verbose2("Submit"); + + SubmitInternal(DateTime.UtcNow, logEntry); + + worker.Notify(); + } + + /// + public void SubmitRange(IEnumerable logEntries) + { + if (!SupportSubmissions) + throw new InvalidOperationException("Provider does not support submissions on cluster " + Services.MyClusterId); + + if (stats != null) stats.EventCounters["SubmitRangeCalled"]++; + + Services.Verbose2("SubmitRange"); + + var time = DateTime.UtcNow; + + foreach (var e in logEntries) + SubmitInternal(time, e); + + worker.Notify(); + } + + /// + public Task TryAppend(TLogEntry logEntry) + { + if (!SupportSubmissions) + throw new InvalidOperationException("Provider does not support submissions on cluster " + Services.MyClusterId); + + if (stats != null) stats.EventCounters["TryAppendCalled"]++; + + Services.Verbose2("TryAppend"); + + var promise = new TaskCompletionSource(); + + SubmitInternal(DateTime.UtcNow, logEntry, GetConfirmedVersion() + pending.Count, promise); + + worker.Notify(); + + return promise.Task; + } + + /// + public Task TryAppendRange(IEnumerable logEntries) + { + if (!SupportSubmissions) + throw new InvalidOperationException("Provider does not support submissions on cluster " + Services.MyClusterId); + + if (stats != null) stats.EventCounters["TryAppendRangeCalled"]++; + + Services.Verbose2("TryAppendRange"); + + var promise = new TaskCompletionSource(); + var time = DateTime.UtcNow; + var pos = GetConfirmedVersion() + pending.Count; + + bool first = true; + foreach (var e in logEntries) + { + SubmitInternal(time, e, pos++, first ? promise : null); + first = false; + } + + worker.Notify(); + + return promise.Task; + } + + + private const int unconditional = -1; + + private void SubmitInternal(DateTime time, TLogEntry logentry, int conditionalPosition = unconditional, TaskCompletionSource resultPromise = null) + { + // create a submission entry + var submissionentry = this.MakeSubmissionEntry(logentry); + submissionentry.SubmissionTime = time; + submissionentry.ResultPromise = resultPromise; + submissionentry.ConditionalPosition = conditionalPosition; + + // add submission to queue + pending.Add(submissionentry); + + // if we have a tentative state in use, update it + if (this.tentativeStateInternal != null) + { + try + { + Host.UpdateView(this.tentativeStateInternal, logentry); + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(SubmitInternal), e); + } + } + + try + { + Host.OnViewChanged(true, false); + } + catch (Exception e) + { + Services.CaughtUserCodeException("OnViewChanged", nameof(SubmitInternal), e); + } + } + + /// + public TLogView TentativeView + { + get + { + if (stats != null) + stats.EventCounters["TentativeViewCalled"]++; + + if (tentativeStateInternal == null) + CalculateTentativeState(); + + return tentativeStateInternal; + } + } + + /// + public TLogView ConfirmedView + { + get + { + if (stats != null) + stats.EventCounters["ConfirmedViewCalled"]++; + + return LastConfirmedView(); + } + } + + /// + public int ConfirmedVersion + { + get + { + if (stats != null) + stats.EventCounters["ConfirmedVersionCalled"]++; + + return GetConfirmedVersion(); + } + } + + /// + /// Called from network + /// + /// + /// + public async Task OnProtocolMessageReceived(IProtocolMessage payLoad) + { + var notificationMessage = payLoad as INotificationMessage; + + if (notificationMessage != null) + { + Services.Verbose("NotificationReceived v{0}", notificationMessage.Version); + + OnNotificationReceived(notificationMessage); + + // poke worker so it will process the notifications + worker.Notify(); + + return null; + } + else + { + //it's a protocol message + return await OnMessageReceived(payLoad); + } + } + + + /// + /// Called by MultiClusterOracle when there is a configuration change. + /// + /// + public async Task OnMultiClusterConfigurationChange(MultiCluster.MultiClusterConfiguration newConfig) + { + Debug.Assert(newConfig != null); + + var oldConfig = Configuration; + + // process only if newer than what we already have + if (!MultiClusterConfiguration.OlderThan(oldConfig, newConfig)) + return; + + Services.Verbose("Processing Configuration {0}", newConfig); + + await this.OnConfigurationChange(newConfig); // updates Configuration and does any work required + + var added = oldConfig == null ? newConfig.Clusters : newConfig.Clusters.Except(oldConfig.Clusters); + + // if the multi-cluster is operated correctly, this grain should not be active before we are joined to the multicluster + // but if we detect that anyway here, enforce a refresh to reduce risk of missed notifications + if (!needInitialRead && added.Contains(Services.MyClusterId)) + { + needRefresh = true; + Services.Verbose("Refresh Because of Join"); + worker.Notify(); + } + + if (notificationTracker != null) + { + var remoteInstances = Services.RegistrationStrategy.GetRemoteInstances(newConfig, Services.MyClusterId).ToList(); + notificationTracker.UpdateNotificationTargets(remoteInstances); + } + } + + + + #endregion + + /// + /// method is virtual so subclasses can add their own events + /// + public virtual void EnableStatsCollection() + { + + stats = new LogConsistencyStatistics() + { + EventCounters = new Dictionary(), + StabilizationLatenciesInMsecs = new List() + }; + + stats.EventCounters.Add("TentativeViewCalled", 0); + stats.EventCounters.Add("ConfirmedViewCalled", 0); + stats.EventCounters.Add("ConfirmedVersionCalled", 0); + stats.EventCounters.Add("SubmitCalled", 0); + stats.EventCounters.Add("SubmitRangeCalled", 0); + stats.EventCounters.Add("TryAppendCalled", 0); + stats.EventCounters.Add("TryAppendRangeCalled", 0); + stats.EventCounters.Add("ConfirmSubmittedEntriesCalled", 0); + stats.EventCounters.Add("SynchronizeNowCalled", 0); + + stats.EventCounters.Add("WritebackEvents", 0); + + stats.StabilizationLatenciesInMsecs = new List(); + } + + /// + /// Disable stats collection + /// + public void DisableStatsCollection() + { + stats = null; + } + + /// + /// Get states + /// + /// + public LogConsistencyStatistics GetStats() + { + return stats; + } + + private void CalculateTentativeState() + { + // copy the master + this.tentativeStateInternal = (TLogView)SerializationManager.DeepCopy(LastConfirmedView()); + + // Now apply all operations in pending + foreach (var u in this.pending) + try + { + Host.UpdateView(this.tentativeStateInternal, u.Entry); + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(CalculateTentativeState), e); + } + } + + + /// + /// batch worker performs reads from and writes to global state. + /// only one work cycle is active at any time. + /// + internal async Task Work() + { + Services.Verbose("<1 ProcessNotifications"); + + var version = GetConfirmedVersion(); + + ProcessNotifications(); + + Services.Verbose("<2 NotifyViewChanges"); + + NotifyViewChanges(ref version); + + bool haveToWrite = (pending.Count != 0); + + bool haveToRead = needInitialRead || (needRefresh && !haveToWrite); + + Services.Verbose("<3 Storage htr={0} htw={1}", haveToRead, haveToWrite); + + try + { + if (haveToRead) + { + needRefresh = needInitialRead = false; // retrieving fresh version + + await ReadAsync(); + + NotifyViewChanges(ref version); + } + + if (haveToWrite) + { + needRefresh = needInitialRead = false; // retrieving fresh version + + await UpdatePrimary(); + + if (stats != null) stats.EventCounters["WritebackEvents"]++; + } + + } + catch (Exception e) + { + // this should never happen - we are supposed to catch and store exceptions + // in the correct place (LastPrimaryException or notification trackers) + Services.ProtocolError($"Exception in Worker Cycle: {e}", true); + + } + + Services.Verbose("<4 Done"); + } + + + /// + /// This function stores the operations in the pending queue as a batch to the primary. + /// Retries until some batch commits or there are no updates left. + /// + internal async Task UpdatePrimary() + { + int version = GetConfirmedVersion(); + + while (true) + { + try + { + // find stale conditional updates, remove them, and notify waiters + RemoveStaleConditionalUpdates(); + + if (pending.Count == 0) + return; // no updates to write. + + // try to write the updates as a batch + var writeResult = await WriteAsync(); + + NotifyViewChanges(ref version, writeResult); + + // if the batch write failed due to conflicts, retry. + if (writeResult == 0) + continue; + + try + { + Host.OnViewChanged(false, true); + } + catch (Exception e) + { + Services.CaughtUserCodeException("OnViewChanged", nameof(UpdatePrimary), e); + } + + // notify waiting promises of the success of conditional updates + NotifyPromises(writeResult, true); + + // record stabilization time, for statistics + if (stats != null) + { + var timeNow = DateTime.UtcNow; + for (int i = 0; i < writeResult; i++) + { + var latency = timeNow - pending[i].SubmissionTime; + stats.StabilizationLatenciesInMsecs.Add(latency.Milliseconds); + } + } + + // remove completed updates from queue + pending.RemoveRange(0, writeResult); + + return; + } + catch (Exception e) + { + // this should never happen - we are supposed to catch and store exceptions + // in the correct place (LastPrimaryException or notification trackers) + Services.ProtocolError($"Exception in {nameof(UpdatePrimary)}: {e}", true); + } + } + } + + + private void NotifyViewChanges(ref int version, int numWritten = 0) + { + var v = GetConfirmedVersion(); + bool tentativeChanged = (v != version + numWritten); + bool confirmedChanged = (v != version); + if (tentativeChanged || confirmedChanged) + { + tentativeStateInternal = null; // conservative. + try + { + Host.OnViewChanged(tentativeChanged, confirmedChanged); + } + catch (Exception e) + { + Services.CaughtUserCodeException("OnViewChanged", nameof(NotifyViewChanges), e); + } + version = v; + } + } + + /// + /// returns a list of all connection health issues that have not been restored yet. + /// Such issues are observed while communicating with the primary, or while trying to + /// notify other clusters, for example. + /// + public IEnumerable UnresolvedConnectionIssues + { + get + { + if (LastPrimaryIssue.Issue != null) + yield return LastPrimaryIssue.Issue; + if (notificationTracker != null) + foreach (var x in notificationTracker.UnresolvedConnectionIssues) + yield return x; + } + } + + /// + /// Store the last issue that occurred while reading or updating primary. + /// Is null if successful. + /// + protected RecordedConnectionIssue LastPrimaryIssue; + + + + /// + public async Task SynchronizeNowAsync() + { + if (stats != null) + stats.EventCounters["SynchronizeNowCalled"]++; + + Services.Verbose("SynchronizeNowStart"); + + needRefresh = true; + await worker.NotifyAndWaitForWorkToBeServiced(); + + Services.Verbose("SynchronizeNowComplete"); + } + + /// + public IEnumerable UnconfirmedSuffix + { + get + { + return pending.Select(te => te.Entry); + } + } + + /// + public async Task ConfirmSubmittedEntriesAsync() + { + if (stats != null) + stats.EventCounters["ConfirmSubmittedEntriesCalled"]++; + + Services.Verbose("ConfirmSubmittedEntriesStart"); + + if (pending.Count != 0) + await worker.WaitForCurrentWorkToBeServiced(); + + Services.Verbose("ConfirmSubmittedEntriesEnd"); + } + + /// + /// send failure notifications + /// + protected void NotifyPromises(int count, bool success) + { + for (int i = 0; i < count; i++) + { + var promise = pending[i].ResultPromise; + if (promise != null) + promise.SetResult(success); + } + } + + /// + /// go through updates and remove all the conditional updates that have already failed + /// + protected void RemoveStaleConditionalUpdates() + { + int version = GetConfirmedVersion(); + bool foundFailedConditionalUpdates = false; + + for (int pos = 0; pos < pending.Count; pos++) + { + var submissionEntry = pending[pos]; + if (submissionEntry.ConditionalPosition != unconditional + && (foundFailedConditionalUpdates || + submissionEntry.ConditionalPosition != (version + pos))) + { + foundFailedConditionalUpdates = true; + if (submissionEntry.ResultPromise != null) + submissionEntry.ResultPromise.SetResult(false); + } + pos++; + } + + if (foundFailedConditionalUpdates) + { + pending.RemoveAll(e => e.ConditionalPosition != unconditional); + tentativeStateInternal = null; + try + { + Host.OnViewChanged(true, false); + } + catch (Exception e) + { + Services.CaughtUserCodeException("OnViewChanged", nameof(RemoveStaleConditionalUpdates), e); + } + } + } + + /// + /// Send a notification message to all remote instances + /// + /// the notification message to send + /// if non-null, exclude this cluster id from the notification + protected void BroadcastNotification(INotificationMessage msg, string exclude = null) + { + var remoteinstances = Services.RegistrationStrategy.GetRemoteInstances(Configuration, Services.MyClusterId); + + // if there is only one cluster, don't send notifications. + if (remoteinstances.Count() == 0) + return; + + // create notification tracker if we haven't already + if (notificationTracker == null) + notificationTracker = new NotificationTracker(this.Services, remoteinstances, max_notification_batch_size, Host); + + notificationTracker.BroadcastNotification(msg, exclude); + } + } + + /// + /// Base class for submission entries stored in pending queue. + /// + /// The type of entry for this submission + public class SubmissionEntry + { + /// The log entry that is submitted. + public TLogEntry Entry; + + /// A timestamp for this submission. + public DateTime SubmissionTime; + + /// For conditional updates, a promise that resolves once it is known whether the update was successful or not. + public TaskCompletionSource ResultPromise; + + /// For conditional updates, the log position at which this update is supposed to be applied. + public int ConditionalPosition; + } + + +} diff --git a/src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs b/src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs new file mode 100644 index 0000000000..33700f7b0e --- /dev/null +++ b/src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.LogConsistency; + +namespace Orleans.EventSourcing.Protocols +{ + /// + /// Utility class for recording connection issues. + /// It is public, not internal, because it is a useful building block for implementing other consistency providers. + /// + public struct RecordedConnectionIssue + { + /// + /// The recorded connection issue, or null if none + /// + public ConnectionIssue Issue { get; private set; } + + /// + /// record a connection issue, filling in timestamps etc. + /// and notify the listener + /// + /// the connection issue to be recorded + /// the listener for connection issues + /// for reporting exceptions in listener + public void Record(ConnectionIssue newIssue, IConnectionIssueListener listener, IProtocolServices services) + { + newIssue.TimeStamp = DateTime.UtcNow; + if (Issue != null) + { + newIssue.TimeOfFirstFailure = Issue.TimeOfFirstFailure; + newIssue.NumberOfConsecutiveFailures = Issue.NumberOfConsecutiveFailures + 1; + newIssue.RetryDelay = newIssue.ComputeRetryDelay(Issue.RetryDelay); + } + else + { + newIssue.TimeOfFirstFailure = newIssue.TimeStamp; + newIssue.NumberOfConsecutiveFailures = 1; + newIssue.RetryDelay = newIssue.ComputeRetryDelay(null); + } + try + { + listener.OnConnectionIssue(newIssue); + } + catch (Exception e) + { + services.CaughtUserCodeException("OnConnectionIssue", nameof(Record), e); + } + } + + /// + /// if there is a recorded issue, notify listener and clear it. + /// + /// the listener for connection issues + /// for reporting exceptions in listener + public void Resolve(IConnectionIssueListener listener, IProtocolServices services) + { + if (Issue != null) + { + try + { + listener.OnConnectionIssueResolved(Issue); + } + catch (Exception e) + { + services.CaughtUserCodeException("OnConnectionIssueResolved", nameof(Record), e); + } + Issue = null; + } + } + + /// + /// delays if there was an issue in last attempt, for the duration specified by the retry delay + /// + /// + public async Task DelayBeforeRetry() + { + if (Issue == null) + return; + + await Task.Delay(Issue.RetryDelay); + } + + /// + public override string ToString() + { + if (Issue == null) + return ""; + else + return Issue.ToString(); + } + } + +} diff --git a/src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs b/src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs new file mode 100644 index 0000000000..8b2ea56756 --- /dev/null +++ b/src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs @@ -0,0 +1,30 @@ +using Orleans.LogConsistency; +using Orleans.Providers; +using Orleans.Runtime; +using Orleans.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.VersionedStateStorage +{ + internal class DefaultAdaptorFactory : ILogViewAdaptorFactory + { + public bool UsesStorageProvider + { + get + { + return true; + } + } + + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + where T : class, new() where E : class + { + return new LogViewAdaptor(hostgrain, initialstate, storageProvider, graintypename, services); + } + + } +} diff --git a/src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs b/src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs new file mode 100644 index 0000000000..8ac07e0659 --- /dev/null +++ b/src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.VersionedStateStorage +{ + /// + /// A class that extends grain state with versioning metadata, so that a grain with log-view consistency + /// can use a standard storage provider via + /// + /// The type used for log view + [Serializable] + public class GrainStateWithMetaDataAndETag : IGrainState where TView : class, new() + { + /// + /// Gets and Sets StateAndMetaData + /// + public GrainStateWithMetaData StateAndMetaData { get; set; } + + /// + /// Gets and Sets Etag + /// + public string ETag { get; set; } + + object IGrainState.State + { + get + { + return StateAndMetaData; + } + set + { + StateAndMetaData = (GrainStateWithMetaData)value; + } + } + + /// + /// Initialize a new instance of GrainStateWithMetaDataAndETag class with a initialVew + /// + public GrainStateWithMetaDataAndETag(TView initialview) + { + StateAndMetaData = new GrainStateWithMetaData(initialview); + } + + /// + /// Initializes a new instance of GrainStateWithMetaDataAndETag class + /// + public GrainStateWithMetaDataAndETag() + { + StateAndMetaData = new GrainStateWithMetaData(); + } + + /// + /// Convert current GrainStateWithMetaDataAndETag object information to a string + /// + public override string ToString() + { + return string.Format("v{0} Flags={1} ETag={2} Data={3}", StateAndMetaData.GlobalVersion, StateAndMetaData.WriteVector, ETag, StateAndMetaData.State); + } + } + + + /// + /// A class that extends grain state with versioning metadata, so that a log-consistent grain + /// can use a standard storage provider via + /// + /// + [Serializable] + public class GrainStateWithMetaData where TView : class, new() + { + /// + /// The stored view of the log + /// + public TView State { get; set; } + + /// + /// The length of the log + /// + public int GlobalVersion { get; set; } + + + /// + /// Metadata that is used to avoid duplicate appends. + /// Logically, this is a (string->bit) map, the keys being replica ids + /// But this map is represented compactly as a simple string to reduce serialization/deserialization overhead + /// Bits are read by and flipped by . + /// Bits are toggled when writing, so that the retry logic can avoid appending an entry twice + /// when retrying a failed append. + /// + public string WriteVector { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public GrainStateWithMetaData() + { + State = new TView(); + GlobalVersion = 0; + WriteVector = ""; + } + + /// + /// Initializes a new instance of the class. + /// + /// The initial state of the view + public GrainStateWithMetaData(TView initialstate) + { + this.State = initialstate; + GlobalVersion = 0; + WriteVector = ""; + } + + // BitVector of replicas is implemented as a set of replica strings encoded within a string + // The bitvector is represented as the set of replica ids whose bit is 1 + // This set is written as a string that contains the replica ids preceded by a comma each + // + // Assuming our replicas are named A, B, and BB, then + // "" represents {} represents 000 + // ",A" represents {A} represents 100 + // ",A,B" represents {A,B} represents 110 + // ",BB,A,B" represents {A,B,BB} represents 111 + + /// + /// Gets one of the bits in + /// + /// The replica for which we want to look up the bit + /// + public bool GetBit(string Replica) + { + var pos = WriteVector.IndexOf(Replica); + return pos != -1 && WriteVector[pos - 1] == ','; + } + + /// + /// toggle one of the bits in and return the new value. + /// + /// The replica for which we want to flip the bit + /// the state of the bit after flipping it + public bool FlipBit(string Replica) + { + var pos = WriteVector.IndexOf(Replica); + if (pos != -1 && WriteVector[pos - 1] == ',') + { + var pos2 = WriteVector.IndexOf(',', pos + 1); + if (pos2 == -1) + pos2 = WriteVector.Length; + WriteVector = WriteVector.Remove(pos - 1, pos2 - pos + 1); + return false; + } + else + { + WriteVector = string.Format(",{0}{1}", Replica, WriteVector); + return true; + } + } + + } +} diff --git a/src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs new file mode 100644 index 0000000000..da8f9b6a0c --- /dev/null +++ b/src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Runtime; +using Orleans.Storage; +using System.Threading; +using Orleans.Providers; + +namespace Orleans.EventSourcing.VersionedStateStorage +{ + /// + /// A log-consistency provider that stores the latest view in primary storage, using any standard storage provider. + /// Supports multiple clusters connecting to the same primary storage (doing optimistic concurrency control via e-tags) + /// + /// The log itself is transient, i.e. not actually saved to storage - only the latest view (snapshot) and some + /// metadata (the log position, and write flags) are stored in the primary. + /// + /// + public class LogConsistencyProvider : ILogConsistencyProvider + { + /// + public string Name { get; private set; } + + /// + public Logger Log { get; private set; } + + /// + public bool UsesStorageProvider + { + get + { + return true; + } + } + + private static int counter; // used for constructing a unique id + private int id; + + /// + protected virtual string GetLoggerName() + { + return string.Format("LogViews.{0}.{1}", GetType().Name, id); + } + + /// + /// Init method + /// + /// Consistency provider name + /// Provider runtime + /// Provider config + public Task Init(string name, IProviderRuntime providerRuntime, IProviderConfiguration config) + { + Name = name; + id = Interlocked.Increment(ref counter); // unique id for this provider; matters only for tracing + + Log = providerRuntime.GetLogger(GetLoggerName()); + Log.Info("Init (Severity={0})", Log.SeverityLevel); + + return TaskDone.Done; + } + + /// + /// Close method + /// + public Task Close() + { + return TaskDone.Done; + } + + /// + /// Make log view adaptor + /// + /// The type of the view + /// The type of the log entries + /// The grain that is hosting this adaptor + /// The initial state for this view + /// The type name of the grain + /// Runtime services for multi-cluster coherence protocols + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services) + where TView : class, new() + where TEntry : class + { + return new LogViewAdaptor(hostGrain, initialState, storageProvider, grainTypeName, services); + } + + } + +} \ No newline at end of file diff --git a/src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs new file mode 100644 index 0000000000..075e519413 --- /dev/null +++ b/src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs @@ -0,0 +1,360 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Storage; +using Orleans.EventSourcing.Protocols; + +namespace Orleans.EventSourcing.VersionedStateStorage +{ + /// + /// A log view adaptor that wraps around a traditional storage adaptor, and uses batching and e-tags + /// to append entries. + /// + /// The log itself is transient, i.e. not actually saved to storage - only the latest view and some + /// metadata (the log position, and write flags) are stored. + /// + /// + /// Type of log view + /// Type of log entry + internal class LogViewAdaptor : PrimaryBasedLogViewAdaptor> where TLogView : class, new() where TLogEntry : class + { + /// + /// Initialize a StorageProviderLogViewAdaptor class + /// + public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, IProtocolServices services) + : base(host, initialState, services) + { + this.globalStorageProvider = globalStorageProvider; + this.grainTypeName = grainTypeName; + } + + + private const int maxEntriesInNotifications = 200; + + + IStorageProvider globalStorageProvider; + string grainTypeName; // stores the confirmed state including metadata + GrainStateWithMetaDataAndETag GlobalStateCache; + + /// + protected override TLogView LastConfirmedView() + { + return GlobalStateCache.StateAndMetaData.State; + } + + /// + protected override int GetConfirmedVersion() + { + return GlobalStateCache.StateAndMetaData.GlobalVersion; + } + + /// + protected override void InitializeConfirmedView(TLogView initialstate) + { + GlobalStateCache = new GrainStateWithMetaDataAndETag(initialstate); + } + + // no special tagging is required, thus we create a plain submission entry + /// + protected override SubmissionEntry MakeSubmissionEntry(TLogEntry entry) + { + return new SubmissionEntry() { Entry = entry }; + } + + /// + protected override async Task ReadAsync() + { + enter_operation("ReadAsync"); + + while (true) + { + try + { + // for manual testing + //await Task.Delay(5000); + + await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); + + Services.Verbose("read success {0}", GlobalStateCache); + + LastPrimaryIssue.Resolve(Host, Services); + + break; // successful + } + catch (Exception e) + { + LastPrimaryIssue.Record(new ReadFromStorageFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + + await LastPrimaryIssue.DelayBeforeRetry(); + } + + exit_operation("ReadAsync"); + } + + + /// + protected override async Task WriteAsync() + { + enter_operation("WriteAsync"); + + var state = CopyTentativeState(); + var updates = GetCurrentBatchOfUpdates(); + bool batchsuccessfullywritten = false; + + var nextglobalstate = new GrainStateWithMetaDataAndETag(state); + nextglobalstate.StateAndMetaData.WriteVector = GlobalStateCache.StateAndMetaData.WriteVector; + nextglobalstate.StateAndMetaData.GlobalVersion = GlobalStateCache.StateAndMetaData.GlobalVersion + updates.Length; + nextglobalstate.ETag = GlobalStateCache.ETag; + + var writebit = nextglobalstate.StateAndMetaData.FlipBit(Services.MyClusterId); + + try + { + // for manual testing + //await Task.Delay(5000); + + await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, nextglobalstate); + + batchsuccessfullywritten = true; + + GlobalStateCache = nextglobalstate; + + Services.Verbose("write ({0} updates) success {1}", updates.Length, GlobalStateCache); + + LastPrimaryIssue.Resolve(Host, Services); + } + catch (Exception e) + { + LastPrimaryIssue.Record(new UpdateStorageFailed() { Exception = e }, Host, Services); + } + + if (!batchsuccessfullywritten) + { + Services.Verbose("write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue); + + while (true) // be stubborn until we can read what is there + { + + await LastPrimaryIssue.DelayBeforeRetry(); + + try + { + await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); + + Services.Verbose("read success {0}", GlobalStateCache); + + LastPrimaryIssue.Resolve(Host, Services); + + break; + } + catch (Exception e) + { + LastPrimaryIssue.Record(new ReadFromStorageFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + } + + // check if last apparently failed write was in fact successful + + if (writebit == GlobalStateCache.StateAndMetaData.GetBit(Services.MyClusterId)) + { + GlobalStateCache = nextglobalstate; + + Services.Verbose("last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache); + + batchsuccessfullywritten = true; + } + } + + + // broadcast notifications to all other clusters + if (batchsuccessfullywritten) + BroadcastNotification(new UpdateNotificationMessage() + { + Version = GlobalStateCache.StateAndMetaData.GlobalVersion, + Updates = updates.Select(se => se.Entry).ToList(), + Origin = Services.MyClusterId, + ETag = GlobalStateCache.ETag + }); + + exit_operation("WriteAsync"); + + if (!batchsuccessfullywritten) + return 0; + + return updates.Length; + } + + + /// + /// Describes a connection issue that occurred when updating the primary storage. + /// + [Serializable] + public class UpdateStorageFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"update storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// Describes a connection issue that occurred when reading from the primary storage. + /// + [Serializable] + public class ReadFromStorageFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"read from storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// A notification message sent to remote instances after updating this grain in storage. + /// + [Serializable] + protected class UpdateNotificationMessage : INotificationMessage + { + /// + public int Version { get; set; } + + /// The cluster that performed the update + public string Origin { get; set; } + + /// The list of updates that were applied + public List Updates { get; set; } + + /// The e-tag of the storage after applying the updates + public string ETag { get; set; } + + /// + public override string ToString() + { + return string.Format("v{0} ({1} updates by {2}) etag={2}", Version, Updates.Count, Origin, ETag); + } + } + + /// + protected override INotificationMessage Merge(INotificationMessage earlierMessage, INotificationMessage laterMessage) + { + var earlier = earlierMessage as UpdateNotificationMessage; + var later = laterMessage as UpdateNotificationMessage; + + if (earlier != null + && later != null + && earlier.Origin == later.Origin + && earlier.Version + later.Updates.Count == later.Version + && earlier.Updates.Count + later.Updates.Count < maxEntriesInNotifications) + + return new UpdateNotificationMessage() + { + Version = later.Version, + Origin = later.Origin, + Updates = earlier.Updates.Concat(later.Updates).ToList(), + ETag = later.ETag + }; + + else + return base.Merge(earlierMessage, laterMessage); // keep only the version number + } + + private SortedList notifications = new SortedList(); + + /// + protected override void OnNotificationReceived(INotificationMessage payload) + { + var um = payload as UpdateNotificationMessage; + if (um != null) + notifications.Add(um.Version - um.Updates.Count, um); + else + base.OnNotificationReceived(payload); + } + + /// + protected override void ProcessNotifications() + { + // discard notifications that are behind our already confirmed state + while (notifications.Count > 0 && notifications.ElementAt(0).Key < GlobalStateCache.StateAndMetaData.GlobalVersion) + { + Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + notifications.RemoveAt(0); + } + + // process notifications that reflect next global version + while (notifications.Count > 0 && notifications.ElementAt(0).Key == GlobalStateCache.StateAndMetaData.GlobalVersion) + { + var updateNotification = notifications.ElementAt(0).Value; + notifications.RemoveAt(0); + + // Apply all operations in pending + foreach (var u in updateNotification.Updates) + try + { + Host.UpdateView(GlobalStateCache.StateAndMetaData.State, u); + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(ProcessNotifications), e); + } + + GlobalStateCache.StateAndMetaData.GlobalVersion = updateNotification.Version; + + GlobalStateCache.StateAndMetaData.FlipBit(updateNotification.Origin); + + GlobalStateCache.ETag = updateNotification.ETag; + + Services.Verbose("notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalStateCache); + } + + Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + + base.ProcessNotifications(); + + } + + + #region non-reentrancy assertions + +#if DEBUG + bool operation_in_progress; +#endif + + [Conditional("DEBUG")] + private void enter_operation(string name) + { +#if DEBUG + Services.Verbose2("/-- enter {0}", name); + Debug.Assert(!operation_in_progress); + operation_in_progress = true; +#endif + } + + [Conditional("DEBUG")] + private void exit_operation(string name) + { +#if DEBUG + Services.Verbose2("\\-- exit {0}", name); + Debug.Assert(operation_in_progress); + operation_in_progress = false; +#endif + } + + + + #endregion + } +} diff --git a/src/OrleansRuntime/Catalog/Catalog.cs b/src/OrleansRuntime/Catalog/Catalog.cs index 3157aa60ef..4ee4798dd0 100644 --- a/src/OrleansRuntime/Catalog/Catalog.cs +++ b/src/OrleansRuntime/Catalog/Catalog.cs @@ -9,7 +9,9 @@ using Orleans.CodeGeneration; using Orleans.GrainDirectory; +using Orleans.MultiCluster; using Orleans.Providers; +using Orleans.LogConsistency; using Orleans.Runtime.Configuration; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Messaging; @@ -127,7 +129,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont private readonly OrleansTaskScheduler scheduler; private readonly ActivationDirectory activations; private IStorageProviderManager storageProviderManager; - + private ILogConsistencyProviderManager logConsistencyProviderManager; private readonly Logger logger; private int collectionNumber; private int destroyActivationsNumber; @@ -205,6 +207,11 @@ internal void SetStorageManager(IStorageProviderManager storageManager) storageProviderManager = storageManager; } + internal void SetLogConsistencyManager(ILogConsistencyProviderManager logConsistencyManager) + { + logConsistencyProviderManager = logConsistencyManager; + } + internal void Start() { if (gcTimer != null) gcTimer.Dispose(); @@ -675,22 +682,28 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri { Grain grain; - //Create a new instance of a stateless grain - if (stateObjectType == null) + //Create a new instance of the given grain type + grain = grainCreator.CreateGrainInstance(grainType, data.Identity); + + //for stateful grains, install storage bridge + if (grain is IStatefulGrain) { - //Create a new instance of the given grain type - grain = grainCreator.CreateGrainInstance(grainType, data.Identity); + SetupStorageProvider(grainType, data); + grainCreator.InstallStorageBridge(grain, grainType, grainTypeData.StateObjectType, data.StorageProvider); } - //Create a new instance of a stateful grain - else + + //for log-view grains, install log-view adaptor + else if (grain is ILogConsistentGrain) { - SetupStorageProvider(grainType, data); - grain = grainCreator.CreateGrainInstance(grainType, data.Identity, stateObjectType, data.StorageProvider); + var consistencyProvider = SetupLogConsistencyProvider(grain, grainType, data); + grainCreator.InstallLogViewAdaptor(grain, grainType, grainTypeData.StateObjectType, consistencyProvider, data.StorageProvider); } + grain.Data = data; data.SetGrainInstance(grain); } + activations.IncrementGrainCounter(grainClassName); if (logger.IsVerbose) logger.Verbose("CreateGrainInstance {0}{1}", data.Grain, data.ActivationId); @@ -740,6 +753,73 @@ private void SetupStorageProvider(Type grainType, ActivationData data) } } + private ILogViewAdaptorFactory SetupLogConsistencyProvider(Grain grain, Type grainType, ActivationData data) + { + var attr = grainType.GetTypeInfo().GetCustomAttributes(true).FirstOrDefault(); + var consistencyProviderName = attr?.ProviderName; + + ILogConsistencyProvider consistencyProvider; + + if (logConsistencyProviderManager == null) + { + var errMsg = string.Format("No consistency provider manager found loading grain type {0}", grainType.FullName); + logger.Error(ErrorCode.Provider_CatalogNoLogConsistencyProvider, errMsg); + throw new BadProviderConfigException(errMsg); + } + + if (!string.IsNullOrWhiteSpace(consistencyProviderName)) + { + // find the named consistency provider; throw exception if it is not in the config + if (!logConsistencyProviderManager.TryGetProvider(consistencyProviderName, out consistencyProvider, false)) + { + var errMsg = string.Format( + "Cannot find consistency provider with Name={0} for grain type {1}", attr.ProviderName, + grainType.FullName); + logger.Error(ErrorCode.Provider_CatalogNoLogConsistencyProvider, errMsg); + throw new BadProviderConfigException(errMsg); + } + } + else + { + // See if the config specifies a "Default" consistency provider; if so use that + logConsistencyProviderManager.TryGetProvider(Constants.DEFAULT_LOG_CONSISTENCY_PROVIDER_NAME, out consistencyProvider, true); + } + + if (consistencyProvider != null) + { + // we found a log consistency provider in the configuration file + + // if it depends on a storage provider, find that one too + if (consistencyProvider.UsesStorageProvider) + SetupStorageProvider(grainType, data); + + string msg = string.Format("Assigned log consistency provider with Name={0} to grain type {1}", + attr.ProviderName, grainType.FullName); + logger.Verbose2(ErrorCode.Provider_CatalogLogConsistencyProviderAllocated, msg); + + return consistencyProvider; + } + + // Case 2 : no log consistency provider was specified in the configuration file. + // now we check if the grain type specifies a default adaptor factory + + var defaultFactory = ((ILogConsistentGrain)grain).DefaultAdaptorFactory; + + if (defaultFactory == null) + { + var errMsg = string.Format("No log consistency provider found loading grain type {0}", grainType.FullName); + logger.Error(ErrorCode.Provider_CatalogNoLogConsistencyProvider, errMsg); + throw new BadProviderConfigException(errMsg); + }; + + // if it depends on a storage provider, find that one too + if (defaultFactory.UsesStorageProvider) + SetupStorageProvider(grainType, data); + + return defaultFactory; + } + + private async Task SetupActivationState(ActivationData result, string grainType) { var statefulGrain = result.GrainInstance as IStatefulGrain; @@ -1099,6 +1179,11 @@ private async Task CallGrainActivate(ActivationData activation, Dictionary CallGrainDeactivateAndCleanupStreams(Activati logger.Warn(ErrorCode.Catalog_DeactivateStreamResources_Exception, String.Format("DeactivateStreamResources Grain type = {0} Activation = {1} failed.", grainTypeName, activation), exc); } } + + if (activation.GrainInstance is IProtocolParticipant) + { + await ((IProtocolParticipant)activation.GrainInstance).DeactivateProtocolParticipant(); + } } catch(Exception exc) { diff --git a/src/OrleansRuntime/Catalog/GrainCreator.cs b/src/OrleansRuntime/Catalog/GrainCreator.cs index 7e6728bad6..0a63ff020f 100644 --- a/src/OrleansRuntime/Catalog/GrainCreator.cs +++ b/src/OrleansRuntime/Catalog/GrainCreator.cs @@ -2,7 +2,10 @@ using System.Collections.Concurrent; using Microsoft.Extensions.DependencyInjection; using Orleans.Core; +using Orleans.LogConsistency; using Orleans.Storage; +using Orleans.Runtime.LogConsistency; +using Orleans.GrainDirectory; namespace Orleans.Runtime { @@ -59,32 +62,44 @@ public Grain CreateGrainInstance(Type grainType, IGrainIdentity identity) } /// - /// Create a new instance of a grain + /// Install the storage bridge into a stateful grain. /// + /// The grain. /// The grain type. - /// Identity for the new grain - /// If the grain is a stateful grain, the type of the state it persists. - /// If the grain is a stateful grain, the storage provider used to persist the state. - /// The newly created grain. - public Grain CreateGrainInstance(Type grainType, IGrainIdentity identity, Type stateType, IStorageProvider storageProvider) + /// The type of the state it persists. + /// The provider used to store the state. + public void InstallStorageBridge(Grain grain, Type grainType, Type stateType, IStorageProvider storageProvider) { - // Create a new instance of the grain - var grain = this.CreateGrainInstance(grainType, identity); + var statefulgrain = (IStatefulGrain) grain; - var statefulGrain = grain as IStatefulGrain; + var storage = new GrainStateStorageBridge(grainType.FullName, statefulgrain, storageProvider); - if (statefulGrain == null) - { - return grain; - } + //Inject state and storage data into the grain + statefulgrain.GrainState.State = Activator.CreateInstance(stateType); + statefulgrain.SetStorage(storage); + } - var storage = new GrainStateStorageBridge(grainType.FullName, statefulGrain, storageProvider); - //Inject state and storage data into the grain - statefulGrain.GrainState.State = Activator.CreateInstance(stateType); - statefulGrain.SetStorage(storage); + /// + /// Install the log-view adaptor into a log-consistent grain. + /// + /// The grain. + /// The grain type. + /// The type of the grain state. + /// The consistency adaptor factory + /// The storage provider, or null if none needed + /// The newly created grain. + public void InstallLogViewAdaptor(Grain grain, Type grainType, Type stateType, ILogViewAdaptorFactory factory, IStorageProvider storageProvider) + { + // try to find a suitable logger that we can use to trace consistency protocol information + var logger = (factory as ILogConsistencyProvider)?.Log ?? storageProvider?.Log; + + // encapsulate runtime services used by consistency adaptors + var svc = new ProtocolServices(grain, logger, MultiClusterRegistrationStrategy.FromGrainType(grain.GetType())); - return grain; + var state = Activator.CreateInstance(stateType); + + ((ILogConsistentGrain)grain).InstallAdaptor(factory, state, grainType.FullName, storageProvider, svc); } } } \ No newline at end of file diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs index f45cf61d30..ff4d1b769e 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs @@ -5,6 +5,7 @@ using System.Reflection; using Orleans.CodeGeneration; +using Orleans.Core; using Orleans.Concurrency; using Orleans.GrainDirectory; using Orleans.MultiCluster; @@ -25,6 +26,7 @@ internal class GrainTypeData internal bool IsReentrant { get; private set; } internal bool IsStatelessWorker { get; private set; } internal Func MayInterleave { get; private set; } + internal MultiClusterRegistrationStrategy MultiClusterRegistrationStrategy { get; private set; } public GrainTypeData(Type type, Type stateObjectType) { diff --git a/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs b/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs index f35c582997..e2e55873f4 100644 --- a/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs +++ b/src/OrleansRuntime/GrainTypeManager/SiloAssemblyLoader.cs @@ -5,6 +5,8 @@ using System.Reflection; using System.Text; using Orleans.CodeGeneration; +using Orleans.Serialization; +using Orleans.LogConsistency; using Orleans.Providers; using Orleans.Runtime.Configuration; @@ -95,7 +97,7 @@ private void LoadApplicationAssemblies() if (parentTypeInfo.IsGenericType) { var definition = parentTypeInfo.GetGenericTypeDefinition(); - if (definition == typeof(Grain<>)) + if (definition == typeof(Grain<>) || definition == typeof(LogConsistentGrainBase<>)) { var stateArg = parentType.GetGenericArguments()[0]; if (stateArg.GetTypeInfo().IsClass || stateArg.GetTypeInfo().IsValueType) diff --git a/src/OrleansRuntime/LogConsistency/ILogConsistencyProviderManager.cs b/src/OrleansRuntime/LogConsistency/ILogConsistencyProviderManager.cs new file mode 100644 index 0000000000..8398b66886 --- /dev/null +++ b/src/OrleansRuntime/LogConsistency/ILogConsistencyProviderManager.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using Orleans.Providers; +using Orleans.Runtime; + +namespace Orleans.LogConsistency +{ + internal interface ILogConsistencyProviderManager : IProviderManager + { + Logger GetLogger(string loggerName); + + IEnumerable GetProviderNames(); + + int GetLoadedProvidersNum(); + + bool TryGetProvider(string name, out ILogConsistencyProvider provider, bool caseInsensitive = false); + } + + +} diff --git a/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs b/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs new file mode 100644 index 0000000000..2d8348e9c5 --- /dev/null +++ b/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Orleans.Core; +using Orleans.Providers; +using Orleans.Runtime.Configuration; +using Orleans.Runtime.Providers; +using Orleans.LogConsistency; +using Orleans.Storage; + +namespace Orleans.Runtime.LogConsistency +{ + internal class LogConsistencyProviderManager : ILogConsistencyProviderManager, ILogConsistencyProviderRuntime + { + private ProviderLoader providerLoader; + private IProviderRuntime providerRuntime; + + public IGrainFactory GrainFactory { get; private set; } + public IServiceProvider ServiceProvider { get; private set; } + + public LogConsistencyProviderManager(IGrainFactory grainFactory, IServiceProvider serviceProvider) + { + GrainFactory = grainFactory; + ServiceProvider = serviceProvider; + } + + internal Task LoadLogConsistencyProviders(IDictionary configs) + { + providerLoader = new ProviderLoader(); + providerRuntime = SiloProviderRuntime.Instance; + + if (!configs.ContainsKey(ProviderCategoryConfiguration.LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME)) + return TaskDone.Done; + + providerLoader.LoadProviders(configs[ProviderCategoryConfiguration.LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME].Providers, this); + return providerLoader.InitProviders(this); + } + + internal void UnloadLogConsistencyProviders() + { + foreach (var provider in providerLoader.GetProviders()) + { + var disp = provider as IDisposable; + if (disp != null) + disp.Dispose(); + } + } + + public int GetLoadedProvidersNum() + { + return providerLoader.GetNumLoadedProviders(); + } + + public IList GetProviders() + { + return providerLoader.GetProviders(); + } + + public void SetInvokeInterceptor(InvokeInterceptor interceptor) + { + providerRuntime.SetInvokeInterceptor(interceptor); + } + + public InvokeInterceptor GetInvokeInterceptor() + { + return providerRuntime.GetInvokeInterceptor(); + } + + public Logger GetLogger(string loggerName) + { + return LogManager.GetLogger(loggerName, LoggerType.Provider); + } + + public Guid ServiceId + { + get { return providerRuntime.ServiceId; } + } + + public string SiloIdentity + { + get { return providerRuntime.SiloIdentity; } + } + + /// + /// Get list of providers loaded in this silo. + /// + /// + public IEnumerable GetProviderNames() + { + var providers = providerLoader.GetProviders(); + return providers.Select(p => p.GetType().FullName).ToList(); + } + + public bool TryGetProvider(string name, out ILogConsistencyProvider provider, bool caseInsensitive = false) + { + return providerLoader.TryGetProvider(name, out provider, caseInsensitive); + } + + public IProvider GetProvider(string name) + { + return providerLoader.GetProvider(name, true); + } + + } +} diff --git a/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs b/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs new file mode 100644 index 0000000000..1f49a338d0 --- /dev/null +++ b/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.Core; +using Orleans.MultiCluster; +using Orleans.Runtime; +using Orleans.SystemTargetInterfaces; +using Orleans.Concurrency; + +namespace Orleans.Runtime.LogConsistency +{ + [Reentrant] + internal class ProtocolGateway : SystemTarget, IProtocolGateway + { + public ProtocolGateway(SiloAddress silo) + : base(Constants.ProtocolGatewayId, silo) + { + } + + public async Task RelayMessage(GrainId id, IProtocolMessage payload) + { + var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(GrainReference.FromGrainId(id)); + return await g.OnProtocolMessageReceived(payload); + } + + } +} diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs new file mode 100644 index 0000000000..35fba26c23 --- /dev/null +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.Core; +using Orleans.LogConsistency; +using Orleans.MultiCluster; +using Orleans.Runtime; +using Orleans.SystemTargetInterfaces; +using Orleans.GrainDirectory; + +namespace Orleans.Runtime.LogConsistency +{ + /// + /// Functionality for use by log view adaptors that run distributed protocols. + /// This class allows access to these services to providers that cannot see runtime-internals. + /// It also stores grain-specific information like the grain reference, and caches + /// + internal class ProtocolServices : IProtocolServices + { + + public GrainReference GrainReference { get { return grain.GrainReference; } } + + private Logger log; + + public IMultiClusterRegistrationStrategy RegistrationStrategy { get; private set; } + + private Grain grain; // links to the grain that owns this service object + + internal ProtocolServices(Grain gr, Logger log, IMultiClusterRegistrationStrategy strategy) + { + this.grain = gr; + this.log = log; + this.RegistrationStrategy = strategy; + + if (!Silo.CurrentSilo.GlobalConfig.HasMultiClusterNetwork) + PseudoMultiClusterConfiguration = new MultiClusterConfiguration(DateTime.UtcNow, new string[] { PseudoReplicaId }.ToList()); + } + + + public async Task SendMessage(IProtocolMessage payload, string clusterId) + { + var silo = Silo.CurrentSilo; + var mycluster = silo.ClusterId; + var oracle = silo.LocalMultiClusterOracle; + + log?.Verbose3("SendMessage {0}->{1}: {2}", mycluster, clusterId, payload); + + if (mycluster == clusterId) + { + var g = (IProtocolParticipant)grain; + // we are on the same scheduler, so we can call the method directly + return await g.OnProtocolMessageReceived(payload); + } + + if (PseudoMultiClusterConfiguration != null) + throw new ProtocolTransportException("no such cluster"); + + if (log != null && log.IsVerbose3) + { + var gws = oracle.GetGateways(); + log.Verbose3("Available Gateways:\n{0}", string.Join("\n", gws.Select((gw) => gw.ToString()))); + } + + var clusterGateway = oracle.GetRandomClusterGateway(clusterId); + + if (clusterGateway == null) + throw new ProtocolTransportException("no active gateways found for cluster"); + + var repAgent = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget(Constants.ProtocolGatewayId, clusterGateway); + + // test hook + var filter = (oracle as MultiClusterNetwork.MultiClusterOracle).ProtocolMessageFilterForTesting; + if (filter != null && !filter(payload)) + return null; + + try + { + var retMessage = await repAgent.RelayMessage(GrainReference.GrainId, payload); + return retMessage; + } + catch (Exception e) + { + throw new ProtocolTransportException("failed sending message to cluster", e); + } + } + + // pseudo-configuration to use if there is no actual multicluster network + private static MultiClusterConfiguration PseudoMultiClusterConfiguration; + private static string PseudoReplicaId = "I"; + + + public bool MultiClusterEnabled + { + get + { + return (PseudoMultiClusterConfiguration == null); + } + } + + public string MyClusterId + { + get + { + if (PseudoMultiClusterConfiguration != null) + return PseudoReplicaId; + else + return Silo.CurrentSilo.ClusterId; + } + } + + public MultiClusterConfiguration MultiClusterConfiguration + { + get + { + if (PseudoMultiClusterConfiguration != null) + return PseudoMultiClusterConfiguration; + else + return Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration(); + } + } + + public IEnumerable GetRemoteInstances() + { + if (PseudoMultiClusterConfiguration == null + && RegistrationStrategy != ClusterLocalRegistration.Singleton) + { + var myclusterid = Silo.CurrentSilo.ClusterId; + + foreach (var cluster in Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration().Clusters) + { + if (cluster != myclusterid) + yield return cluster; + } + } + } + + public void SubscribeToMultiClusterConfigurationChanges() + { + if (PseudoMultiClusterConfiguration == null) + { + // subscribe this grain to configuration change events + Silo.CurrentSilo.LocalMultiClusterOracle.SubscribeToMultiClusterConfigurationEvents(GrainReference); + } + } + + public void UnsubscribeFromMultiClusterConfigurationChanges() + { + if (PseudoMultiClusterConfiguration == null) + { + // unsubscribe this grain from configuration change events + Silo.CurrentSilo.LocalMultiClusterOracle.UnSubscribeFromMultiClusterConfigurationEvents(GrainReference); + } + + } + + + public IEnumerable ActiveClusters + { + get + { + if (PseudoMultiClusterConfiguration != null) + return PseudoMultiClusterConfiguration.Clusters; + else + return Silo.CurrentSilo.LocalMultiClusterOracle.GetActiveClusters(); + } + } + + public void ProtocolError(string msg, bool throwexception) + { + + log?.Error((int)(throwexception ? ErrorCode.LogConsistency_ProtocolFatalError : ErrorCode.LogConsistency_ProtocolError), + string.Format("{0}{1} Protocol Error: {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration == null ? "" : (" " + Silo.CurrentSilo.ClusterId), + msg)); + + if (!throwexception) + return; + + if (PseudoMultiClusterConfiguration != null) + throw new OrleansException(string.Format("{0} (grain={1})", msg, grain.GrainReference)); + else + throw new OrleansException(string.Format("{0} (grain={1}, cluster={2})", msg, grain.GrainReference, Silo.CurrentSilo.ClusterId)); + } + + public void CaughtException(string where, Exception e) + { + log?.Error((int)ErrorCode.LogConsistency_CaughtException, + string.Format("{0}{1} Exception Caught at {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration == null ? "" : (" " + Silo.CurrentSilo.ClusterId), + where),e); + } + + + + public void CaughtUserCodeException(string callback, string where, Exception e) + { + log?.Warn((int)ErrorCode.LogConsistency_UserCodeException, + string.Format("{0}{1} Exception caught in user code for {2}, called from {3}", + grain.GrainReference, + PseudoMultiClusterConfiguration == null ? "" : (" " + Silo.CurrentSilo.ClusterId), + callback, + where), e); + } + + + public void Info(string format, params object[] args) + { + log?.Info("{0}{1} {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), + string.Format(format, args)); + } + + public void Verbose(string format, params object[] args) + { + if (log != null && log.IsVerbose) + { + log.Verbose("{0}{1} {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), + string.Format(format, args)); + } + } + + /// Output the specified message at Verbose2 log level. + public void Verbose2(string format, params object[] args) + { + if (log != null && log.IsVerbose2) + { + log.Verbose2("{0}{1} {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), + string.Format(format, args)); + } + } + + /// Output the specified message at Verbose3 log level. + public void Verbose3(string format, params object[] args) + { + if (log != null && log.IsVerbose3) + { + log.Verbose3("{0}{1} {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), + string.Format(format, args)); + } + } + + } + +} diff --git a/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs b/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs index 95df81e5f4..471f73299d 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Orleans.MultiCluster; +using System; namespace Orleans.Runtime.MultiClusterNetwork { @@ -55,5 +56,22 @@ internal interface IMultiClusterOracle /// a gateway address, or null if none is found for the given cluster SiloAddress GetRandomClusterGateway(string cluster); + /// + /// Subscribe to multicluster configuration change events. + /// + /// An observer to receive configuration change notifications. + /// bool value indicating that subscription succeeded or not. + bool SubscribeToMultiClusterConfigurationEvents(GrainReference observer); + + /// + /// UnSubscribe from multicluster configuration change events. + /// + /// bool value indicating that subscription succeeded or not. + bool UnSubscribeFromMultiClusterConfigurationEvents(GrainReference observer); + + /// + /// A test hook for dropping protocol messages between replicated grain instances + /// + Func ProtocolMessageFilterForTesting { get; set; } } } diff --git a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs index f549631fa6..60213372d0 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs @@ -114,6 +114,21 @@ public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus sta PublishChanges(); } + public bool SubscribeToMultiClusterConfigurationEvents(GrainReference observer) + { + return localData.SubscribeToMultiClusterConfigurationEvents(observer); + } + + public bool UnSubscribeFromMultiClusterConfigurationEvents(GrainReference observer) + { + return localData.UnSubscribeFromMultiClusterConfigurationEvents(observer); + } + + + /// + public Func ProtocolMessageFilterForTesting { get; set; } + + public async Task Start(ISiloStatusOracle oracle) { logger.Info(ErrorCode.MultiClusterNetwork_Starting, "MultiClusterOracle starting on {0}, Severity={1} ", Silo, logger.SeverityLevel); diff --git a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs index fd6dc0587d..42f2f4327d 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs @@ -1,5 +1,7 @@ -using Orleans.MultiCluster; +using System; using System.Collections.Generic; +using Orleans.MultiCluster; +using System.Linq; namespace Orleans.Runtime.MultiClusterNetwork { @@ -14,6 +16,8 @@ internal class MultiClusterOracleData } private volatile IReadOnlyDictionary> activeGatewaysByCluster; + private readonly HashSet confListeners; + private readonly Logger logger; internal MultiClusterData Current { get { return localData; } } @@ -23,6 +27,7 @@ internal MultiClusterOracleData(Logger log) logger = log; localData = new MultiClusterData(); activeGatewaysByCluster = new Dictionary>(); + confListeners = new HashSet(); } private void ComputeAvailableGatewaysPerCluster() @@ -41,6 +46,33 @@ private void ComputeAvailableGatewaysPerCluster() activeGatewaysByCluster = gws; } + internal bool SubscribeToMultiClusterConfigurationEvents(GrainReference observer) + { + if (logger.IsVerbose2) + logger.Verbose2("SubscribeToMultiClusterConfigurationEvents: {0}", observer); + + lock (confListeners) + { + if (confListeners.Contains(observer)) + return false; + + confListeners.Add(observer); + return true; + } + } + + + internal bool UnSubscribeFromMultiClusterConfigurationEvents(GrainReference observer) + { + if (logger.IsVerbose3) + logger.Verbose3("UnSubscribeFromMultiClusterConfigurationEvents: {0}", observer); + + lock (confListeners) + { + return confListeners.Remove(observer); + } + } + public MultiClusterData ApplyDataAndNotify(MultiClusterData data) { if (data.IsEmpty) @@ -66,9 +98,35 @@ public MultiClusterData ApplyDataAndNotify(MultiClusterData data) if (delta.Configuration != null) { // notify configuration listeners of change - // code will be added in separate PR + + List listenersToNotify; + lock (confListeners) + { + // make a copy under the lock + listenersToNotify = confListeners.ToList(); + } + + foreach (var listener in listenersToNotify) + { + try + { + if (logger.IsVerbose2) + logger.Verbose2("-NotificationWork: notify IProtocolParticipant {0} of configuration {1}", listener, delta.Configuration); + + // enqueue conf change event as grain call + var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(listener); + g.OnMultiClusterConfigurationChange(delta.Configuration).Ignore(); + } + catch (Exception exc) + { + logger.Error(ErrorCode.MultiClusterNetwork_LocalSubscriberException, + String.Format("IProtocolParticipant {0} threw exception processing configuration {1}", + listener, delta.Configuration), exc); + } + } } + return delta; } } diff --git a/src/OrleansRuntime/OrleansRuntime.csproj b/src/OrleansRuntime/OrleansRuntime.csproj index 86b6ef7640..1cf9cf1432 100644 --- a/src/OrleansRuntime/OrleansRuntime.csproj +++ b/src/OrleansRuntime/OrleansRuntime.csproj @@ -112,6 +112,10 @@ + + + + diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 339d5fd497..32f0c0abbc 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -10,6 +10,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Core; +using Orleans.LogConsistency; using Orleans.GrainDirectory; using Orleans.Providers; using Orleans.Runtime.Configuration; @@ -21,6 +23,7 @@ using Orleans.Runtime.MultiClusterNetwork; using Orleans.Runtime.Placement; using Orleans.Runtime.Providers; +using Orleans.Runtime.LogConsistency; using Orleans.Runtime.Scheduler; using Orleans.Runtime.Startup; using Orleans.Runtime.Storage; @@ -67,6 +70,7 @@ public enum SiloType private readonly InsideRuntimeClient runtimeClient; private readonly AssemblyProcessor assemblyProcessor; private StorageProviderManager storageProviderManager; + private LogConsistencyProviderManager logConsistencyProviderManager; private StatisticsProviderManager statisticsProviderManager; private BootstrapProviderManager bootstrapProviderManager; private IReminderService reminderService; @@ -97,6 +101,7 @@ public enum SiloType internal ISiloStatusOracle LocalSiloStatusOracle { get { return membershipOracle; } } internal IMultiClusterOracle LocalMultiClusterOracle { get { return multiClusterOracle; } } internal IConsistentRingProvider RingProvider { get; private set; } + internal ILogConsistencyProviderManager LogConsistencyProviderManager { get { return logConsistencyProviderManager; } } internal IStorageProviderManager StorageProviderManager { get { return storageProviderManager; } } internal IProviderManager StatisticsProviderManager { get { return statisticsProviderManager; } } internal IStreamProviderManager StreamProviderManager { get { return grainRuntime.StreamProviderManager; } } @@ -369,6 +374,9 @@ private void CreateSystemTargets() logger.Verbose("Creating {0} System Target", "StreamProviderUpdateAgent"); RegisterSystemTarget(new StreamProviderManagerAgent(this, allSiloProviders)); + logger.Verbose("Creating {0} System Target", "ProtocolGateway"); + RegisterSystemTarget(new ProtocolGateway(this.SiloAddress)); + logger.Verbose("Creating {0} System Target", "DeploymentLoadPublisher"); RegisterSystemTarget(Services.GetRequiredService()); @@ -531,6 +539,15 @@ private void DoStart() allSiloProviders.AddRange(storageProviderManager.GetProviders()); if (logger.IsVerbose) { logger.Verbose("Storage provider manager created successfully."); } + // Initialize log consistency providers once we have a basic silo runtime environment operating + logConsistencyProviderManager = new LogConsistencyProviderManager(grainFactory, Services); + scheduler.QueueTask( + () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations), + providerManagerSystemTarget.SchedulingContext) + .WaitWithThrow(initTimeout); + catalog.SetLogConsistencyManager(logConsistencyProviderManager); + if (logger.IsVerbose) { logger.Verbose("Log consistency provider manager created successfully."); } + // Load and init stream providers before silo becomes active var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager; scheduler.QueueTask( diff --git a/src/OrleansTestingHost/AppDomainSiloHost.cs b/src/OrleansTestingHost/AppDomainSiloHost.cs index fc3b270cba..3e37250746 100644 --- a/src/OrleansTestingHost/AppDomainSiloHost.cs +++ b/src/OrleansTestingHost/AppDomainSiloHost.cs @@ -14,6 +14,8 @@ using Orleans.Runtime.Placement; using Orleans.Runtime.TestHooks; using Orleans.Storage; +using Orleans.Runtime.MultiClusterNetwork; +using Orleans.MultiCluster; namespace Orleans.TestingHost { @@ -172,7 +174,21 @@ internal void UnblockSiloCommunication() mc.ShouldDrop = null; simulatedMessageLoss.Clear(); } - + + internal Func ProtocolMessageFilterForTesting + { + get + { + var mco = this.silo.LocalMultiClusterOracle; + return mco.ProtocolMessageFilterForTesting; + } + set + { + var mco = this.silo.LocalMultiClusterOracle; + mco.ProtocolMessageFilterForTesting = value; + } + } + private bool ShouldDrop(Message msg) { if (simulatedMessageLoss != null) diff --git a/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs b/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs new file mode 100644 index 0000000000..fc5b39dc25 --- /dev/null +++ b/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs @@ -0,0 +1,52 @@ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans.Runtime.Configuration; +using Orleans.TestingHost; +using Orleans.Runtime; + +namespace Orleans.TestingHost +{ + /// A static class with functionality shared by various log-consistency provider tests. + public static class LogConsistencyProviderConfiguration + { + // change this as needed for debugging failing tests + private const Severity LogConsistencyProviderTraceLevel = Severity.Verbose2; + + /// + /// Initializes a bunch of different + /// log consistency providers with different configuration settings. + /// + /// the data connection string + /// The configuration to modify + public static void ConfigureLogConsistencyProvidersForTesting(string dataConnectionString, ClusterConfiguration config) + { + { + var props = new Dictionary(); + props.Add("DataConnectionString", dataConnectionString); + config.Globals.RegisterStorageProvider("Orleans.Storage.AzureTableStorage", "AzureStore", props); + } + { + var props = new Dictionary(); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.VersionedStateStorage.LogConsistencyProvider", "VersionedStateStorage", props); + } + { + var props = new Dictionary(); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomVersionedStateStorage.LogConsistencyProvider", "CustomStorage", props); + } + { + var props = new Dictionary(); + props.Add("PrimaryCluster", "A"); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomVersionedStateStorage.LogConsistencyProvider", "CustomStoragePrimaryCluster", props); + } + + // logging + foreach (var o in config.Overrides) + o.Value.TraceLevelOverrides.Add(new Tuple("LogViews", Severity.Verbose2)); + + } + } +} diff --git a/src/OrleansTestingHost/OrleansTestingHost.csproj b/src/OrleansTestingHost/OrleansTestingHost.csproj index f855a4295f..688645f5fa 100644 --- a/src/OrleansTestingHost/OrleansTestingHost.csproj +++ b/src/OrleansTestingHost/OrleansTestingHost.csproj @@ -60,6 +60,7 @@ + diff --git a/test/TestGrainInterfaces/IJournaledPersonGrain.cs b/test/TestGrainInterfaces/IJournaledPersonGrain.cs index 569c9491b7..4ae95ad40e 100644 --- a/test/TestGrainInterfaces/IJournaledPersonGrain.cs +++ b/test/TestGrainInterfaces/IJournaledPersonGrain.cs @@ -26,5 +26,9 @@ public interface IJournaledPersonGrain : Orleans.IGrainWithGuidKey Task Marry(IJournaledPersonGrain spouse); Task GetPersonalAttributes(); + + // Tests + + Task RunTentativeConfirmedStateTest(); } } diff --git a/test/TestGrainInterfaces/ILogConsistentGrain.cs b/test/TestGrainInterfaces/ILogConsistentGrain.cs new file mode 100644 index 0000000000..9746bfb953 --- /dev/null +++ b/test/TestGrainInterfaces/ILogConsistentGrain.cs @@ -0,0 +1,94 @@ +using System; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using System.Collections.Generic; + +namespace UnitTests.GrainInterfaces +{ + /// + /// A grain used for testing log-consistency providers. + /// + public interface ILogConsistentGrain: IGrainWithIntegerKey + { + #region Queries + + // read A + + Task GetAGlobal(); + + Task GetALocal(); + + // read both + + Task GetBothGlobal(); + + Task GetBothLocal(); + + // reservations + + Task GetReservationsGlobal(); + + // version + + Task GetConfirmedVersion(); + + // exception + Task> GetUnresolvedConnectionIssues(); + + #endregion + + + #region Updates + + // set or increment A + + Task SetAGlobal(int a); + + Task> SetAConditional(int a); + + Task SetALocal(int a); + + Task IncrementALocal(); + + Task IncrementAGlobal(); + + // set B + + Task SetBGlobal(int b); + + Task SetBLocal(int b); + + // reservations + + Task AddReservationLocal(int x); + + Task RemoveReservationLocal(int x); + + #endregion + + + Task> Read(); + Task Update(IReadOnlyList updates, int expectedversion); + + #region Other + + // other operations + + Task SynchronizeGlobalState(); + Task Deactivate(); + + #endregion + } + + /// + /// Used by unit tests. + /// The fields don't really have any meaning. + /// The point of the struct is just that a grain method can return both A and B at the same time. + /// + public struct AB + { + public int A; + public int B; + } +} diff --git a/test/TestGrainInterfaces/TestGrainInterfaces.csproj b/test/TestGrainInterfaces/TestGrainInterfaces.csproj index 7215ff5a32..bf8d3233a0 100644 --- a/test/TestGrainInterfaces/TestGrainInterfaces.csproj +++ b/test/TestGrainInterfaces/TestGrainInterfaces.csproj @@ -105,6 +105,7 @@ + diff --git a/test/TestGrains/EventSourcing/JournaledPersonGrain.cs b/test/TestGrains/EventSourcing/JournaledPersonGrain.cs index f250c807a1..214a33ffa6 100644 --- a/test/TestGrains/EventSourcing/JournaledPersonGrain.cs +++ b/test/TestGrains/EventSourcing/JournaledPersonGrain.cs @@ -4,15 +4,24 @@ using Orleans.EventSourcing; using Orleans.Providers; using TestGrainInterfaces; +using System.Collections.Generic; +using Orleans.Runtime; namespace TestGrains { - [StorageProvider(ProviderName = "MemoryStore")] - public class JournaledPersonGrain : JournaledGrain, IJournaledPersonGrain + public class JournaledPersonGrain : JournaledGrain, IJournaledPersonGrain { + public Task RegisterBirth(PersonAttributes props) { - return RaiseStateEvent(new PersonRegistered(props.FirstName, props.LastName, props.Gender)); + if (this.State.FirstName == null) + { + RaiseEvent(new PersonRegistered(props.FirstName, props.LastName, props.Gender)); + + return ConfirmEvents(); + } + + return TaskDone.Done; } public async Task Marry(IJournaledPersonGrain spouse) @@ -22,30 +31,111 @@ public async Task Marry(IJournaledPersonGrain spouse) var spouseData = await spouse.GetPersonalAttributes(); - await RaiseStateEvent( - new PersonMarried(spouse.GetPrimaryKey(), spouseData.FirstName, spouseData.LastName), - commit: false); // We are not storing the first event here + var events = new List(); + + events.Add(new PersonMarried(spouse.GetPrimaryKey(), spouseData.FirstName, spouseData.LastName)); if (State.LastName != spouseData.LastName) { - await RaiseStateEvent( - new PersonLastNameChanged(spouseData.LastName), - commit: false); + events.Add(new PersonLastNameChanged(spouseData.LastName)); } - // We might need a different, more explicit, persstence API for ES. - // Reusing the current API for now. - await this.WriteStateAsync(); + RaiseEvents(events); + + await ConfirmEvents(); + } + + public Task ChangeLastName(string lastName) + { + RaiseEvent(new PersonLastNameChanged(lastName)); + + return TaskDone.Done; } - + + public Task ConfirmChanges() + { + return ConfirmEvents(); + } + public Task GetPersonalAttributes() { return Task.FromResult(new PersonAttributes - { - FirstName = State.FirstName, - LastName = State.LastName, - Gender = State.Gender - }); + { + FirstName = State.FirstName, + LastName = State.LastName, + Gender = State.Gender + }); + } + + public Task GetConfirmedPersonalAttributes() + { + return Task.FromResult(new PersonAttributes + { + FirstName = ConfirmedState.FirstName, + LastName = ConfirmedState.LastName, + Gender = ConfirmedState.Gender + }); + } + + public Task GetConfirmedVersion() + { + return Task.FromResult(ConfirmedVersion); + } + + public Task GetVersion() + { + return Task.FromResult(Version); + } + + private static void AssertEqual(T a, T b) + { + if (!Object.Equals(a, b)) + throw new OrleansException($"Test failed. Expected = {a}. Actual = {b}."); + } + + public async Task RunTentativeConfirmedStateTest() + { + // initially both the confirmed version and the tentative version are the same: version 0 + AssertEqual(0, ConfirmedVersion); + AssertEqual(0, Version); + AssertEqual(null, ConfirmedState.LastName); + AssertEqual(null, State.LastName); + + // now we change the last name + await ChangeLastName("Organa"); + + // while the udpate is pending, the confirmed version and the tentative version are different + AssertEqual(0, ConfirmedVersion); + AssertEqual(1, Version); + AssertEqual(null, ConfirmedState.LastName); + AssertEqual("Organa", State.LastName); + + // let's wait until the update has been confirmed. + await ConfirmChanges(); + + // now the two versions are the same again + AssertEqual(1, ConfirmedVersion); + AssertEqual(1, Version); + AssertEqual("Organa", ConfirmedState.LastName); + AssertEqual("Organa", State.LastName); + + // issue another change + await ChangeLastName("Solo"); + + // again, the confirmed and the tentative versions are different + AssertEqual(1, ConfirmedVersion); + AssertEqual(2, Version); + AssertEqual("Organa", ConfirmedState.LastName); + AssertEqual("Solo", State.LastName); + + // this time, we wait for (what should be) enough time to commit to MemoryStorage. + await Task.Delay(20); + + // now the two versions should be the same again + AssertEqual(2, ConfirmedVersion); + AssertEqual(2, Version); + AssertEqual("Solo", ConfirmedState.LastName); + AssertEqual("Solo", State.LastName); } } } diff --git a/test/TestGrains/EventSourcing/JournaledPerson_Events.cs b/test/TestGrains/EventSourcing/JournaledPerson_Events.cs index a6323c2f43..9dcf6dcc45 100644 --- a/test/TestGrains/EventSourcing/JournaledPerson_Events.cs +++ b/test/TestGrains/EventSourcing/JournaledPerson_Events.cs @@ -3,7 +3,11 @@ namespace TestGrains { - public class PersonRegistered + // we use a marker interface, so we get a bit more typechecking than with plain objects + public interface IPersonEvent { } + + + public class PersonRegistered : IPersonEvent { public string FirstName { get; private set; } public string LastName { get; private set; } @@ -17,7 +21,7 @@ public PersonRegistered(string firstName, string lastName, GenderType gender) } } - public class PersonMarried + public class PersonMarried : IPersonEvent { public Guid SpouseId { get; private set; } public string SpouseFirstName { get; private set; } @@ -31,7 +35,7 @@ public PersonMarried(Guid spouseId, string spouseFirstName, string spouseLastNam } } - public class PersonLastNameChanged + public class PersonLastNameChanged : IPersonEvent { public string LastName { get; private set; } diff --git a/test/TestGrains/EventSourcing/PersonState.cs b/test/TestGrains/EventSourcing/PersonState.cs index 9070ab3b8c..6a31c44c93 100644 --- a/test/TestGrains/EventSourcing/PersonState.cs +++ b/test/TestGrains/EventSourcing/PersonState.cs @@ -1,9 +1,12 @@ +using System; +using Orleans; using Orleans.EventSourcing; using TestGrainInterfaces; namespace TestGrains { - public class PersonState : JournaledGrainState + [Serializable] + public class PersonState { public string FirstName { get; set; } public string LastName { get; set; } @@ -14,7 +17,7 @@ public void Apply(PersonRegistered @event) { this.FirstName = @event.FirstName; this.LastName = @event.LastName; - this.Gender = @event. Gender; + this.Gender = @event.Gender; } public void Apply(PersonMarried @event) diff --git a/test/TestGrains/LogConsistentGrain.cs b/test/TestGrains/LogConsistentGrain.cs new file mode 100644 index 0000000000..0621695494 --- /dev/null +++ b/test/TestGrains/LogConsistentGrain.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Orleans; +using Orleans.Providers; +using Orleans.LogConsistency; +using UnitTests.GrainInterfaces; +using Orleans.EventSourcing; + +namespace UnitTests.Grains +{ + + [Serializable] + public class MyGrainState + { + public int A; + public int B; + public Dictionary Reservations; + + public MyGrainState() + { + Reservations = new Dictionary(); + } + + public override string ToString() + { + return string.Format("A={0} B={1} R={{{2}}}", A, B, string.Join(", ", Reservations.Select(kvp => string.Format("{0}:{1}", kvp.Key, kvp.Value)))); + } + + // dynamic dispatch to the cases listed below + public void Apply(dynamic o) { Apply(o); } + + // all the update operations are listed here + public void Apply(UpdateA x) { A = x.Val; } + public void Apply(UpdateB x) { B = x.Val; } + public void Apply(IncrementA x) { A++; } + + public void Apply(AddReservation x) { Reservations[x.Val.ToString()] = x.Val; } + public void Apply(RemoveReservation x) { Reservations.Remove(x.Val.ToString()); } + } + + + [Serializable] + public class UpdateA { public int Val; } + [Serializable] + public class UpdateB { public int Val; } + [Serializable] + public class IncrementA { public int Val; } + [Serializable] + public class AddReservation { public int Val; } + [Serializable] + public class RemoveReservation { public int Val; } + + + + /// + /// A grain used for testing log-consistency providers. + /// has two fields A, B that can be updated or incremented; + /// and a dictionary of reservations thatcan be aded and removed + /// We subclass this to create variations for all storage providers + /// + public abstract class LogConsistentGrain : JournaledGrain, GrainInterfaces.ILogConsistentGrain + { + public async Task SetAGlobal(int x) + { + RaiseEvent(new UpdateA() { Val = x }); + await ConfirmEvents(); + } + + public async Task> SetAConditional(int x) + { + int version = this.ConfirmedVersion; + bool success = await RaiseConditionalEvent(new UpdateA() { Val = x }); + return new Tuple(version, success); + } + + public Task SetALocal(int x) + { + RaiseEvent(new UpdateA() { Val = x }); + return TaskDone.Done; + } + public async Task SetBGlobal(int x) + { + RaiseEvent(new UpdateB() { Val = x }); + await ConfirmEvents(); + } + + public Task SetBLocal(int x) + { + RaiseEvent(new UpdateB() { Val = x }); + return TaskDone.Done; + } + + public async Task IncrementAGlobal() + { + RaiseEvent(new IncrementA()); + await ConfirmEvents(); + } + + public Task IncrementALocal() + { + RaiseEvent(new IncrementA()); + return TaskDone.Done; + + } + + public async Task GetAGlobal() + { + await FetchAllEventsNow(); + return ConfirmedState.A; + } + + public Task GetALocal() + { + return Task.FromResult(State.A); + } + + public async Task GetBothGlobal() + { + await FetchAllEventsNow(); + return new AB() { A = ConfirmedState.A, B = ConfirmedState.B }; + } + + public Task GetBothLocal() + { + return Task.FromResult(new AB() { A = State.A, B = State.B }); + } + + public Task AddReservationLocal(int val) + { + RaiseEvent(new AddReservation() { Val = val }); + return TaskDone.Done; + + } + public Task RemoveReservationLocal(int val) + { + RaiseEvent(new RemoveReservation() { Val = val }); + return TaskDone.Done; + + } + public async Task GetReservationsGlobal() + { + await FetchAllEventsNow(); + return ConfirmedState.Reservations.Values.ToArray(); + } + + public Task SynchronizeGlobalState() + { + return FetchAllEventsNow(); + } + + public Task GetConfirmedVersion() + { + return Task.FromResult(this.ConfirmedVersion); + } + + public Task> GetUnresolvedConnectionIssues() + { + return Task.FromResult(this.UnresolvedConnectionIssues); + } + + public async Task> Read() + { + await FetchAllEventsNow(); + return new KeyValuePair(ConfirmedVersion, ConfirmedState); + } + public async Task Update(IReadOnlyList updates, int expectedversion) + { + if (expectedversion > ConfirmedVersion) + await FetchAllEventsNow(); + if (expectedversion != ConfirmedVersion) + return false; + return await RaiseConditionalEvents(updates); + } + + public Task Deactivate() + { + DeactivateOnIdle(); + return TaskDone.Done; + } + + } +} \ No newline at end of file diff --git a/test/TestGrains/LogConsistentGrainVariations.cs b/test/TestGrains/LogConsistentGrainVariations.cs new file mode 100644 index 0000000000..1edf56dac1 --- /dev/null +++ b/test/TestGrains/LogConsistentGrainVariations.cs @@ -0,0 +1,119 @@ +using Orleans; +using Orleans.MultiCluster; +using Orleans.Providers; +using Orleans.Serialization; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnitTests.GrainInterfaces; + +namespace UnitTests.Grains +{ + + // use azure storage and a explicitly configured consistency provider + [OneInstancePerCluster] + [StorageProvider(ProviderName = "AzureStore")] + [LogConsistencyProvider(ProviderName = "VersionedStateStorage")] + public class LogConsistentGrainSharedStorage : LogConsistentGrain + { + } + + // use the default storage provider as the shared storage + [OneInstancePerCluster] + public class LogConsistentGrainDefaultStorage : LogConsistentGrain + { + } + + // use a single-instance log-consistent grain + [GlobalSingleInstance] + [StorageProvider(ProviderName = "AzureStore")] + [LogConsistencyProvider(ProviderName = "VersionedStateStorage")] + public class GsiLogConsistentGrain : LogConsistentGrain + { + } + + // use MemoryStore (which uses GSI grain) + [OneInstancePerCluster] + [StorageProvider(ProviderName = "MemoryStore")] + public class LogConsistentGrainMemoryStorage : LogConsistentGrain + { + } + + // use the explictly specified "CustomStorage" log-consistency provider with symmetric access from all clusters + [OneInstancePerCluster] + [LogConsistencyProvider(ProviderName = "CustomStorage")] + public class LogConsistentGrainCustomStorage : LogConsistentGrain, + Orleans.EventSourcing.CustomVersionedStateStorage.ICustomStorageInterface + { + + // we use another impl of this grain as the primary. + ILogConsistentGrain storagegrain; + + private ILogConsistentGrain GetStorageGrain() + { + if (storagegrain == null) + { + storagegrain = GrainFactory.GetGrain(this.GetPrimaryKeyLong(), "UnitTests.Grains.LogConsistentGrainSharedStorage"); + } + return storagegrain; + } + + + public Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int expectedversion) + { + return GetStorageGrain().Update(updates, expectedversion); + } + + public async Task> ReadStateFromStorageAsync() + { + var kvp = await GetStorageGrain().Read(); + return new KeyValuePair(kvp.Key, (MyGrainState)kvp.Value); + } + } + + // use the explictly specified "CustomStorage" log-consistency provider with access from primary cluster only + [OneInstancePerCluster] + [LogConsistencyProvider(ProviderName = "CustomStoragePrimaryCluster")] + public class LogConsistentGrainCustomStoragePrimaryCluster : LogConsistentGrain, + Orleans.EventSourcing.CustomVersionedStateStorage.ICustomStorageInterface + { + + // we use fake in-memory state as the storage + MyGrainState state; + int version; + + public Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int expectedversion) + { + if (state == null) + { + state = new MyGrainState(); + version = 0; + } + + if (expectedversion != version) + return Task.FromResult(false); + + foreach (var u in updates) + { + state.Apply(u); + version++; + } + + return Task.FromResult(true); + } + + public Task> ReadStateFromStorageAsync() + { + if (state == null) + { + state = new MyGrainState(); + version = 0; + } + return Task.FromResult(new KeyValuePair(version, (MyGrainState)SerializationManager.DeepCopy(state))); + } + } + + +} diff --git a/test/TestGrains/TestGrains.csproj b/test/TestGrains/TestGrains.csproj index ec88362560..d64547f376 100644 --- a/test/TestGrains/TestGrains.csproj +++ b/test/TestGrains/TestGrains.csproj @@ -41,6 +41,7 @@ 4014 + @@ -113,9 +114,11 @@ + + diff --git a/test/Tester/EventSourcingTests/JournaledGrainTests.cs b/test/Tester/EventSourcingTests/JournaledGrainTests.cs new file mode 100644 index 0000000000..77db4b94cd --- /dev/null +++ b/test/Tester/EventSourcingTests/JournaledGrainTests.cs @@ -0,0 +1,81 @@ +using System; +using System.Threading.Tasks; +using System.Linq; +using Orleans; +using TestGrainInterfaces; +using Xunit; +using Assert = Xunit.Assert; +using TestExtensions; +using Xunit.Abstractions; +using Orleans.Runtime; + +namespace UnitTests.EventSourcingTests +{ + public class JournaledGrainTests : HostedTestClusterEnsureDefaultStarted, IDisposable + { + private const string LoggerPrefix = "Storage.MemoryStorage.1"; + + public JournaledGrainTests(DefaultClusterFixture fixture, ITestOutputHelper output) : base(fixture) + { + var mgmt = GrainClient.GrainFactory.GetGrain(0); + var hosts = mgmt.GetHosts().Result.Select(kvp => kvp.Key).ToArray(); + mgmt.SetLogLevel(hosts, LoggerPrefix, (int)Severity.Verbose2); + } + + public void Dispose() + { + var mgmt = GrainClient.GrainFactory.GetGrain(0); + var hosts = mgmt.GetHosts().Result.Select(kvp => kvp.Key).ToArray(); + mgmt.SetLogLevel(hosts, LoggerPrefix, (int)Severity.Info); + } + + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledGrainTests_Activate() + { + var grainWithState = GrainClient.GrainFactory.GetGrain(Guid.Empty); + + Assert.NotNull(await grainWithState.GetPersonalAttributes()); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledGrainTests_Persist() + { + var grainWithState = GrainClient.GrainFactory.GetGrain(Guid.Empty); + + await grainWithState.RegisterBirth(new PersonAttributes { FirstName = "Luke", LastName = "Skywalker", Gender = GenderType.Male }); + + var attributes = await grainWithState.GetPersonalAttributes(); + + Assert.NotNull(attributes); + Assert.Equal("Luke", attributes.FirstName); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledGrainTests_AppendMoreEvents() + { + var leia = GrainClient.GrainFactory.GetGrain(Guid.NewGuid()); + await leia.RegisterBirth(new PersonAttributes { FirstName = "Leia", LastName = "Organa", Gender = GenderType.Female }); + + var han = GrainClient.GrainFactory.GetGrain(Guid.NewGuid()); + await han.RegisterBirth(new PersonAttributes { FirstName = "Han", LastName = "Solo", Gender = GenderType.Male }); + + await leia.Marry(han); + + var attributes = await leia.GetPersonalAttributes(); + Assert.NotNull(attributes); + Assert.Equal("Leia", attributes.FirstName); + Assert.Equal("Solo", attributes.LastName); + } + + [Fact, TestCategory("EventSourcing"), TestCategory("Functional")] + public async Task JournaledGrainTests_TentativeConfirmedState() + { + var leia = GrainClient.GrainFactory.GetGrain(Guid.NewGuid()); + + // the whole test has to run inside the grain, otherwise the interleaving of + // the individual steps is nondeterministic + await leia.RunTentativeConfirmedStateTest(); + } + } +} \ No newline at end of file diff --git a/test/Tester/Tester.csproj b/test/Tester/Tester.csproj index b5ba207260..285a206151 100644 --- a/test/Tester/Tester.csproj +++ b/test/Tester/Tester.csproj @@ -47,6 +47,7 @@ + diff --git a/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs new file mode 100644 index 0000000000..e124f635c8 --- /dev/null +++ b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs @@ -0,0 +1,125 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; +using UnitTests.GrainInterfaces; +using Orleans.TestingHost; +using Xunit; +using Assert = Xunit.Assert; +using TestExtensions; + +namespace Tests.GeoClusterTests +{ + public class BasicLogConsistentGrainTests : TestingSiloHost + { + + + public BasicLogConsistentGrainTests() : + base( + new TestingSiloOptions + { + StartFreshOrleans = true, + StartPrimary = true, + StartSecondary = false, + SiloConfigFile = new FileInfo("OrleansConfigurationForTesting.xml"), + AdjustConfig = cfg => LogConsistencyProviderConfiguration.ConfigureLogConsistencyProvidersForTesting(TestDefaultConfiguration.DataConnectionString,cfg) + } + ) + + { + this.random = new Random(); + } + + private Random random; + + [Fact, TestCategory("GeoCluster")] + public async Task DefaultStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainDefaultStorage"); + } + [Fact, TestCategory("GeoCluster")] + public async Task MemoryStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainMemoryStorage"); + } + [Fact, TestCategory("GeoCluster")] + public async Task SharedStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainSharedStorage"); + } + [Fact, TestCategory("GeoCluster")] + public async Task CustomStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainCustomStorage"); + } + [Fact, TestCategory("GeoCluster")] + public async Task GsiStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.GsiLogConsistentGrain"); + } + + private int GetRandom() + { + lock (random) + return random.Next(); + } + + + private async Task DoBasicLogConsistentGrainTest(string grainClass, int phases = 100) + { + await ThreeCheckers(grainClass, phases); + } + + private async Task ThreeCheckers(string grainClass, int phases) + { + // Global + Func checker1 = async () => + { + int x = GetRandom(); + var grain = GrainFactory.GetGrain(x, grainClass); + await grain.SetAGlobal(x); + int a = await grain.GetAGlobal(); + Assert.Equal(x, a); // value of A survive grain call + Assert.Equal(1, await grain.GetConfirmedVersion()); + }; + + // Local + Func checker2 = async () => + { + int x = GetRandom(); + var grain = GrainFactory.GetGrain(x, grainClass); + Assert.Equal(0, await grain.GetConfirmedVersion()); + await grain.SetALocal(x); + int a = await grain.GetALocal(); + Assert.Equal(x, a); // value of A survive grain call + }; + + // Local then Global + Func checker3 = async () => + { + // Local then Global + int x = GetRandom(); + var grain = GrainFactory.GetGrain(x, grainClass); + await grain.SetALocal(x); + int a = await grain.GetAGlobal(); + Assert.Equal(x, a); + Assert.Equal(1, await grain.GetConfirmedVersion()); + }; + + // test them in sequence + await checker1(); + await checker2(); + await checker3(); + + // test (phases) instances of each checker, all in parallel + var tasks = new List(); + for (int i = 0; i < phases; i++) + { + tasks.Add(checker1()); + tasks.Add(checker2()); + tasks.Add(checker3()); + } + await Task.WhenAll(tasks); + } + } +} diff --git a/test/TesterInternal/GeoClusterTests/BasicMultiClusterTest.cs b/test/TesterInternal/GeoClusterTests/BasicMultiClusterTest.cs index e0d66aa256..53fd00aea3 100644 --- a/test/TesterInternal/GeoClusterTests/BasicMultiClusterTest.cs +++ b/test/TesterInternal/GeoClusterTests/BasicMultiClusterTest.cs @@ -8,52 +8,56 @@ namespace Tests.GeoClusterTests { - public class BasicMultiClusterTest : TestingClusterHost + public class BasicMultiClusterTest { - public BasicMultiClusterTest(ITestOutputHelper output) : base(output) - { } - - // We use ClientWrapper to load a client object in a new app domain. // This allows us to create multiple clients that are connected to different silos. // this client is used to call into the management grain. - public class ClientWrapper : ClientWrapperBase + public class ClientWrapper : TestingClusterHost.ClientWrapperBase { public ClientWrapper(string name, int gatewayport, string clusterId, Action customizer) - // use null clusterId, in this test, because we are testing non-geo clients + // use null clusterId, in this test, because we are testing non-geo clients : base(name, gatewayport, null, customizer) { systemManagement = GrainClient.GrainFactory.GetGrain(0); } IManagementGrain systemManagement; - public Dictionary GetHosts() + public Dictionary GetHosts() { - return systemManagement.GetHosts().Result; + return systemManagement.GetHosts().GetResult(); } } + public BasicMultiClusterTest(ITestOutputHelper output) + { + this.output = output; + } + private ITestOutputHelper output; + [Fact, TestCategory("GeoCluster"), TestCategory("Functional")] - //[Timeout(120000)] public void CreateTwoIndependentClusters() { - // create cluster A with one silo and clientA - var clusterA = "A"; - NewCluster(clusterA, 1); - var clientA = NewClient(clusterA, 0); - - // create cluster B with 5 silos and clientB - var clusterB = "B"; - NewCluster(clusterB, 5); - var clientB = NewClient(clusterB, 0); - - // call management grain in each cluster to count the silos - var silos_in_A = clientA.GetHosts().Count; - var silos_in_B = clientB.GetHosts().Count; - - Assert.Equal(1, silos_in_A); - Assert.Equal(5, silos_in_B); + using (var host = new TestingClusterHost(output)) + { + // create cluster A with one silo and clientA + var clusterA = "A"; + host.NewCluster(clusterA, 1); + var clientA = host.NewClient(clusterA, 0); + + // create cluster B with 5 silos and clientB + var clusterB = "B"; + host.NewCluster(clusterB, 5); + var clientB = host.NewClient(clusterB, 0); + + // call management grain in each cluster to count the silos + var silos_in_A = clientA.GetHosts().Count; + var silos_in_B = clientB.GetHosts().Count; + + Assert.Equal(1, silos_in_A); + Assert.Equal(5, silos_in_B); + } } } } diff --git a/test/TesterInternal/GeoClusterTests/GlobalSingleInstanceClusterTests.cs b/test/TesterInternal/GeoClusterTests/GlobalSingleInstanceClusterTests.cs index 26eae0663b..43df2355fa 100644 --- a/test/TesterInternal/GeoClusterTests/GlobalSingleInstanceClusterTests.cs +++ b/test/TesterInternal/GeoClusterTests/GlobalSingleInstanceClusterTests.cs @@ -68,12 +68,12 @@ public int CallGrain(int i) var grainRef = GrainClient.GrainFactory.GetGrain(i); Task toWait = grainRef.SayHelloAsync(); toWait.Wait(); - return toWait.Result; + return toWait.GetResult(); } public void InjectMultiClusterConf(params string[] args) { - systemManagement.InjectMultiClusterConfiguration(args).Wait(); + systemManagement.InjectMultiClusterConfiguration(args).GetResult(); } IManagementGrain systemManagement; diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs new file mode 100644 index 0000000000..4c625e2543 --- /dev/null +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs @@ -0,0 +1,496 @@ +using Orleans; +using Orleans.Runtime; +using Orleans.Runtime.Configuration; +using Orleans.LogConsistency; +using Orleans.TestingHost; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnitTests.GrainInterfaces; +using Xunit; +using Xunit.Abstractions; +using TestExtensions; +using Orleans.EventSourcing.Protocols; + +namespace Tests.GeoClusterTests +{ + /// + /// A fixture that provides a collection of semantic tests for log-consistency providers + /// (concurrent reading and updating, update propagation, conflict resolution) + /// on a multicluster with the desired number of clusters + /// + public class LogConsistencyTestFixture : TestingClusterHost + { + + #region client wrappers + + public class ClientWrapper : Tests.GeoClusterTests.TestingClusterHost.ClientWrapperBase + { + public ClientWrapper(string name, int gatewayport, string clusterId, Action customizer) + : base(name, gatewayport, clusterId, customizer) + { + systemManagement = GrainClient.GrainFactory.GetGrain(0); + } + + public string GetGrainRef(string grainclass, int i) + { + return GrainClient.GrainFactory.GetGrain(i, grainclass).ToString(); + } + + public void SetALocal(string grainclass, int i, int a) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.SetALocal(a).GetResult(); + } + + public void SetAGlobal(string grainclass, int i, int a) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.SetAGlobal(a).GetResult(); + } + + public Tuple SetAConditional(string grainclass, int i, int a) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.SetAConditional(a).GetResult(); + } + + public void IncrementAGlobal(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.IncrementAGlobal().GetResult(); + } + + public void IncrementALocal(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.IncrementALocal().GetResult(); + } + + public int GetAGlobal(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.GetAGlobal().GetResult(); + } + + public int GetALocal(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.GetALocal().GetResult(); + } + + public void AddReservationLocal(string grainclass, int i, int x) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.AddReservationLocal(x).GetResult(); + } + + public void RemoveReservationLocal(string grainclass, int i, int x) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.RemoveReservationLocal(x).GetResult(); + } + + public int[] GetReservationsGlobal(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.GetReservationsGlobal().GetResult(); + } + + public void Synchronize(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + grainRef.SynchronizeGlobalState().GetResult(); + } + + public void InjectClusterConfiguration(params string[] clusters) + { + systemManagement.InjectMultiClusterConfiguration(clusters).GetResult(); + } + IManagementGrain systemManagement; + + public long GetConfirmedVersion(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.GetConfirmedVersion().GetResult(); + } + + public IEnumerable GetUnresolvedConnectionIssues(string grainclass, int i) + { + var grainRef = GrainClient.GrainFactory.GetGrain(i, grainclass); + return grainRef.GetUnresolvedConnectionIssues().GetResult(); + } + + } + + #endregion + + + public void StartClustersIfNeeded(int numclusters, ITestOutputHelper output) + { + this.output = output; + + if (Clusters.Count != numclusters) + { + if (Clusters.Count > 0) + StopAllClientsAndClusters(); + + + WriteLog("Creating {0} clusters and clients...", numclusters); + this.numclusters = numclusters; + + Assert.True(numclusters >= 2); + + var stopwatch = new System.Diagnostics.Stopwatch(); + stopwatch.Start(); + + // use a random global service id for testing purposes + var globalserviceid = Guid.NewGuid(); + random = new Random(); + + System.Threading.ThreadPool.SetMaxThreads(8, 8); + + // Create clusters and clients + Cluster = new string[numclusters]; + Client = new ClientWrapper[numclusters]; + for (int i = 0; i < numclusters; i++) + { + var clustername = Cluster[i] = ((char)('A' + i)).ToString(); + NewGeoCluster(globalserviceid, clustername, 1, + cfg => LogConsistencyProviderConfiguration.ConfigureLogConsistencyProvidersForTesting(TestDefaultConfiguration.DataConnectionString, cfg)); + Client[i] = NewClient(clustername, 0); + } + + WriteLog("Clusters and clients are ready (elapsed = {0})", stopwatch.Elapsed); + + // wait for configuration to stabilize + WaitForLivenessToStabilizeAsync().WaitWithThrow(TimeSpan.FromMinutes(1)); + + Client[0].InjectClusterConfiguration(Cluster); + WaitForMultiClusterGossipToStabilizeAsync(false).WaitWithThrow(TimeSpan.FromMinutes(System.Diagnostics.Debugger.IsAttached ? 60 : 1)); + + stopwatch.Stop(); + WriteLog("Multicluster is ready (elapsed = {0}).", stopwatch.Elapsed); + } + else + { + WriteLog("Reusing existing {0} clusters and clients.", numclusters); + } + } + + public override void Dispose() + { + base.output = null; // cannot trace during dispose from fixtures + base.Dispose(); + } + + protected ClientWrapper[] Client; + protected string[] Cluster; + protected Random random; + protected int numclusters; + + private const int Xyz = 333; + + public async Task RunChecksOnGrainClass(string grainClass, bool may_update_in_all_clusters, int phases) + { + Random random = new Random(); + + Func GetRandom = () => + { + lock (random) + return random.Next(); + }; + + Func checker1 = () => Task.Run(() => + { + int x = GetRandom(); + var grainidentity = string.Format("grainref={0}", Client[0].GetGrainRef(grainClass, x)); + // force creation of replicas + for (int i = 0; i < numclusters; i++) + AssertEqual(0, Client[i].GetALocal(grainClass, x), grainidentity); + // write global on client 0 + Client[0].SetAGlobal(grainClass, x, Xyz); + // read global on other clients + for (int i = 1; i < numclusters; i++) + { + int r = Client[i].GetAGlobal(grainClass, x); + AssertEqual(Xyz, r, grainidentity); + } + // check local stability + for (int i = 0; i < numclusters; i++) + AssertEqual(Xyz, Client[i].GetALocal(grainClass, x), grainidentity); + // check versions + for (int i = 0; i < numclusters; i++) + AssertEqual(1, Client[i].GetConfirmedVersion(grainClass, x), grainidentity); + }); + + Func checker2 = () => Task.Run(() => + { + int x = GetRandom(); + var grainidentity = string.Format("grainref={0}", Client[0].GetGrainRef(grainClass, x)); + // increment on replica 0 + Client[0].IncrementAGlobal(grainClass, x); + // expect on other replicas + for (int i = 1; i < numclusters; i++) + { + int r = Client[i].GetAGlobal(grainClass, x); + AssertEqual(1, r, grainidentity); + } + // check versions + for (int i = 0; i < numclusters; i++) + AssertEqual(1, Client[i].GetConfirmedVersion(grainClass, x), grainidentity); + }); + + Func checker2b = () => Task.Run(() => + { + int x = GetRandom(); + var grainidentity = string.Format("grainref={0}", Client[0].GetGrainRef(grainClass, x)); + // force first creation on replica 1 + AssertEqual(0, Client[1].GetAGlobal(grainClass, x), grainidentity); + // increment on replica 0 + Client[0].IncrementAGlobal(grainClass, x); + // expect on other replicas + for (int i = 1; i < numclusters; i++) + { + int r = Client[i].GetAGlobal(grainClass, x); + AssertEqual(1, r, grainidentity); + } + // check versions + for (int i = 0; i < numclusters; i++) + AssertEqual(1, Client[i].GetConfirmedVersion(grainClass, x), grainidentity); + }); + + Func checker3 = (int numupdates) => Task.Run(() => + { + int x = GetRandom(); + var grainidentity = string.Format("grainref={0}", Client[0].GetGrainRef(grainClass, x)); + + // concurrently chaotically increment (numupdates) times + Parallel.For(0, numupdates, i => + { + var target = may_update_in_all_clusters ? i % numclusters : 0; + Client[target].IncrementALocal(grainClass, x); + }); + + if (may_update_in_all_clusters) + { + for (int i = 1; i < numclusters; i++) + Client[i].Synchronize(grainClass, x); // push all changes + } + + // push & get all + AssertEqual(numupdates, Client[0].GetAGlobal(grainClass, x), grainidentity); + + for (int i = 1; i < numclusters; i++) + AssertEqual(numupdates, Client[i].GetAGlobal(grainClass, x), grainidentity); // get all + + // check versions + for (int i = 0; i < numclusters; i++) + AssertEqual(numupdates, Client[i].GetConfirmedVersion(grainClass, x), grainidentity); + }); + + Func checker4 = () => Task.Run(() => + { + int x = GetRandom(); + var t = new List(); + for (int i = 0; i < numclusters; i++) + { + int c = i; + t.Add(Task.Run(() => Assert.True(Client[c].GetALocal(grainClass, x) == 0))); + } + for (int i = 0; i < numclusters; i++) + { + int c = i; + t.Add(Task.Run(() => Assert.True(Client[c].GetAGlobal(grainClass, x) == 0))); + } + return Task.WhenAll(t); + }); + + Func checker5 = () => Task.Run(() => + { + var x = GetRandom(); + Task.WaitAll( + Task.Run(() => + { + Client[0].AddReservationLocal(grainClass, x, 0); + Client[0].RemoveReservationLocal(grainClass, x, 0); + Client[0].Synchronize(grainClass, x); + }), + Task.Run(() => + { + Client[1].AddReservationLocal(grainClass, x, 1); + Client[1].RemoveReservationLocal(grainClass, x, 1); + Client[1].AddReservationLocal(grainClass, x, 2); + Client[1].Synchronize(grainClass, x); + }) + ); + var result = Client[0].GetReservationsGlobal(grainClass, x); + Assert.Equal(1, result.Length); + Assert.Equal(2, result[0]); + }); + + Func checker6 = async (int preload) => + { + var x = GetRandom(); + + if (preload % 2 == 0) + Client[1].GetAGlobal(grainClass, x); + if ((preload / 2) % 2 == 0) + Client[0].GetAGlobal(grainClass, x); + + bool[] done = new bool[numclusters - 1]; + + var t = new List(); + + // create listener tasks + for (int i = 1; i < numclusters; i++) + { + int c = i; + t.Add(Task.Run(async () => + { + while (Client[c].GetALocal(grainClass, x) != 1) + await Task.Delay(100); + done[c - 1] = true; + })); + } + + // send notification + Client[0].SetALocal(grainClass, x, 1); + + await Task.WhenAny( + Task.Delay(20000), + Task.WhenAll(t) + ); + + AssertEqual(true, done.All(b => b), string.Format("checker6({0}): update did not propagate within 20 sec", preload)); + }; + + Func checker7 = (int variation) => Task.Run(async () => + { + int x = GetRandom(); + + if (variation % 2 == 0) + Client[1].GetAGlobal(grainClass, x); + if ((variation / 2) % 2 == 0) + Client[0].GetAGlobal(grainClass, x); + + var grainidentity = string.Format("grainref={0}", Client[0].GetGrainRef(grainClass, x)); + + // write conditional on client 0, should always succeed + { + var result = Client[0].SetAConditional(grainClass, x, Xyz); + AssertEqual(0, result.Item1, grainidentity); + AssertEqual(true, result.Item2, grainidentity); + AssertEqual(1, Client[0].GetConfirmedVersion(grainClass, x), grainidentity); + } + + if ((variation / 4) % 2 == 1) + await Task.Delay(100); + + // write conditional on Client[1], may or may not succeed based on timing + { + var result = Client[1].SetAConditional(grainClass, x, 444); + if (result.Item1 == 0) // was stale, thus failed + { + AssertEqual(false, result.Item2, grainidentity); + // must have updated as a result + AssertEqual(1, Client[1].GetConfirmedVersion(grainClass, x), grainidentity); + // check stability + AssertEqual(Xyz, Client[0].GetALocal(grainClass, x), grainidentity); + AssertEqual(Xyz, Client[1].GetALocal(grainClass, x), grainidentity); + AssertEqual(Xyz, Client[0].GetAGlobal(grainClass, x), grainidentity); + AssertEqual(Xyz, Client[1].GetAGlobal(grainClass, x), grainidentity); + } + else // was up-to-date, thus succeeded + { + AssertEqual(true, result.Item2, grainidentity); + AssertEqual(1, result.Item1, grainidentity); + // version is now 2 + AssertEqual(2, Client[1].GetConfirmedVersion(grainClass, x), grainidentity); + // check stability + AssertEqual(444, Client[1].GetALocal(grainClass, x), grainidentity); + AssertEqual(444, Client[0].GetAGlobal(grainClass, x), grainidentity); + AssertEqual(444, Client[1].GetAGlobal(grainClass, x), grainidentity); + } + } + }); + + WriteLog("Running individual short tests"); + + // first, run short ones in sequence + await checker1(); + await checker2(); + await checker2b(); + await checker3(4); + await checker3(20); + await checker4(); + + if (may_update_in_all_clusters) + await checker5(); + + await checker6(0); + await checker6(1); + await checker6(2); + await checker6(3); + + if (may_update_in_all_clusters) + { + await checker7(0); + await checker7(4); + await checker7(7); + + // run tests under blocked notification to force race one way + SetProtocolMessageFilterForTesting(Cluster[0], msg => ! (msg is INotificationMessage)); + await checker7(0); + await checker7(1); + await checker7(2); + await checker7(3); + SetProtocolMessageFilterForTesting(Cluster[0], _ => true); + } + + WriteLog("Running individual longer tests"); + + // then, run slightly longer tests + if (phases != 0) + { + await checker3(20); + await checker3(phases); + } + + WriteLog("Running many concurrent test instances"); + + var tasks = new List(); + for (int i = 0; i < phases; i++) + { + tasks.Add(checker1()); + tasks.Add(checker2()); + tasks.Add(checker2b()); + tasks.Add(checker3(4)); + tasks.Add(checker4()); + + if (may_update_in_all_clusters) + tasks.Add(checker5()); + + tasks.Add(checker6(0)); + tasks.Add(checker6(1)); + tasks.Add(checker6(2)); + tasks.Add(checker6(3)); + + if (may_update_in_all_clusters) + { + tasks.Add(checker7(0)); + tasks.Add(checker7(1)); + tasks.Add(checker7(2)); + tasks.Add(checker7(3)); + tasks.Add(checker7(4)); + tasks.Add(checker7(5)); + tasks.Add(checker7(6)); + tasks.Add(checker7(7)); + } + } + await Task.WhenAll(tasks); + } + } +} diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs new file mode 100644 index 0000000000..3c6be48a0e --- /dev/null +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Orleans.TestingHost; +using UnitTests.GrainInterfaces; +using Orleans.Runtime; +using Tests.GeoClusterTests; +using Xunit; +using Xunit.Abstractions; +using Orleans.Runtime.Configuration; +using Tester; + +namespace Tests.GeoClusterTests +{ + public class LogConsistencyTestsFourClusters : + IClassFixture + { + + public LogConsistencyTestsFourClusters(ITestOutputHelper output, Fixture fixture) + { + this.fixture = fixture; + fixture.StartClustersIfNeeded(4, output); + } + Fixture fixture; + + public class Fixture : LogConsistencyTestFixture + { + } + + const int phases = 100; + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_SharedStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_GsiDefaultStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.GsiLogConsistentGrain", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_CustomStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainCustomStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_CustomStorageProvider_PrimaryCluster() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainCustomStoragePrimaryCluster", false, phases); + } + + } +} diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs new file mode 100644 index 0000000000..163261538d --- /dev/null +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Orleans.TestingHost; +using UnitTests.GrainInterfaces; +using Orleans.Runtime; +using Tests.GeoClusterTests; +using Xunit; +using Xunit.Abstractions; +using Orleans.Runtime.Configuration; + +namespace Tests.GeoClusterTests +{ + public class LogConsistencyTestsTwoClusters: + IClassFixture + { + + public LogConsistencyTestsTwoClusters(ITestOutputHelper output, Fixture fixture) + { + this.fixture = fixture; + fixture.StartClustersIfNeeded(2, output); + } + Fixture fixture; + + public class Fixture : LogConsistencyTestFixture + { + } + + const int phases = 100; + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_SharedStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_GsiDefaultStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.GsiLogConsistentGrain", true, phases); + } + + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_CustomStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainCustomStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_CustomStorageProvider_PrimaryCluster() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainCustomStoragePrimaryCluster", false, phases); + } + + + } +} diff --git a/test/TesterInternal/GeoClusterTests/MultiClusterNetworkTests.cs b/test/TesterInternal/GeoClusterTests/MultiClusterNetworkTests.cs index 5deb8f5cab..bc4fead440 100644 --- a/test/TesterInternal/GeoClusterTests/MultiClusterNetworkTests.cs +++ b/test/TesterInternal/GeoClusterTests/MultiClusterNetworkTests.cs @@ -31,22 +31,22 @@ public ClientWrapper(string name, int gatewayport, string clusterId, Action GetMultiClusterGateways() { - return systemManagement.GetMultiClusterGateways().Result; + return systemManagement.GetMultiClusterGateways().GetResult(); } public Dictionary GetHosts() { - return systemManagement.GetHosts().Result; + return systemManagement.GetHosts().GetResult(); } } diff --git a/test/TesterInternal/GeoClusterTests/MultiClusterRegistrationTests.cs b/test/TesterInternal/GeoClusterTests/MultiClusterRegistrationTests.cs index 79145a574f..0311e9a76e 100644 --- a/test/TesterInternal/GeoClusterTests/MultiClusterRegistrationTests.cs +++ b/test/TesterInternal/GeoClusterTests/MultiClusterRegistrationTests.cs @@ -178,7 +178,7 @@ public int CallGrain(int i) GrainClient.Logger.Info("Call Grain {0}", grainRef); Task toWait = grainRef.SayHelloAsync(); toWait.Wait(); - return toWait.Result; + return toWait.GetResult(); } public string GetRuntimeId(int i) { @@ -186,14 +186,14 @@ public string GetRuntimeId(int i) GrainClient.Logger.Info("GetRuntimeId {0}", grainRef); Task toWait = grainRef.GetRuntimeId(); toWait.Wait(); - return toWait.Result; + return toWait.GetResult(); } public void Deactivate(int i) { var grainRef = GrainClient.GrainFactory.GetGrain(i); GrainClient.Logger.Info("Deactivate {0}", grainRef); Task toWait = grainRef.Deactivate(); - toWait.Wait(); + toWait.GetResult(); } public void EnableStreamNotifications(int i) @@ -201,7 +201,7 @@ public void EnableStreamNotifications(int i) var grainRef = GrainClient.GrainFactory.GetGrain(i); GrainClient.Logger.Info("EnableStreamNotifications {0}", grainRef); Task toWait = grainRef.EnableStreamNotifications(); - toWait.Wait(); + toWait.GetResult(); } // observer-based notification @@ -214,7 +214,7 @@ public void Subscribe(int i, IClusterTestListener listener) listeners.Add(obj); GrainClient.Logger.Info("Subscribe {0}", grainRef); Task toWait = grainRef.Subscribe(obj); - toWait.Wait(); + toWait.GetResult(); } List listeners = new List(); // keep them from being GCed @@ -224,7 +224,7 @@ public void SubscribeStream(int i, IAsyncObserver listener) IStreamProvider streamProvider = GrainClient.GetStreamProvider("SMSProvider"); Guid guid = new Guid(i, 0, 0, new byte[8]); IAsyncStream stream = streamProvider.GetStream(guid, "notificationtest"); - handle = stream.SubscribeAsync(listener).Result; + handle = stream.SubscribeAsync(listener).GetResult(); } StreamSubscriptionHandle handle; diff --git a/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs b/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs index 211c7877b8..50814cd7d9 100644 --- a/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs +++ b/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs @@ -13,28 +13,32 @@ using TestExtensions; using Xunit; using Xunit.Abstractions; +using Orleans.MultiCluster; namespace Tests.GeoClusterTests { /// /// A utility class for tests that include multiple clusters. - /// It calls static methods on TestingSiloHost for starting and stopping silos. /// public class TestingClusterHost : IDisposable { - ITestOutputHelper output; protected readonly Dictionary Clusters; private TestingSiloHost siloHost; + + public TestingSiloOptions siloOptions { get; set; } + protected ITestOutputHelper output; + private TimeSpan gossipStabilizationTime; - public TestingClusterHost(ITestOutputHelper output) + public TestingClusterHost(ITestOutputHelper output = null) { - Clusters = new Dictionary(); this.output = output; + Clusters = new Dictionary(); TestUtils.CheckForAzureStorage(); } + protected struct ClusterInfo { @@ -44,7 +48,8 @@ protected struct ClusterInfo public void WriteLog(string format, params object[] args) { - output.WriteLine("{0} {1}", DateTime.UtcNow, string.Format(format, args)); + if (output != null) + output.WriteLine("{0} {1}", DateTime.UtcNow, string.Format(format, args)); } public async Task RunWithTimeout(string name, int msec, Func test) @@ -84,6 +89,30 @@ public void AssertEqual(T expected, T actual, string comment) throw; } } + public void AssertNull(T actual, string comment) + { + try + { + Assert.Null(actual); + } + catch (Exception e) + { + WriteLog("null assertion failed; actual={0} comment={1}", actual, comment); + throw e; + } + } + public void AssertTrue(bool actual, string comment) + { + try + { + Assert.True(actual); + } + catch (Exception e) + { + WriteLog("true assertion failed; actual={0} comment={1}", actual, comment); + throw e; + } + } /// /// Wait for the multicluster-gossip sub-system to stabilize. @@ -225,7 +254,6 @@ public virtual void Dispose() StopAllClientsAndClusters(); } - public void StopAllClientsAndClusters() { WriteLog("Stopping all Clients and Clusters..."); @@ -387,6 +415,14 @@ public void UnblockAllClusterCommunication(string from) } } + public void SetProtocolMessageFilterForTesting(string origincluster, Func filter) + { + var silos = Clusters[origincluster].Silos; + foreach (var silo in silos) + silo.AppDomainTestHook.ProtocolMessageFilterForTesting = filter; + + } + private SiloHandle GetActiveSiloInClusterByName(string clusterId, string siloName) { if (Clusters[clusterId].Silos == null) return null; diff --git a/test/TesterInternal/TesterInternal.csproj b/test/TesterInternal/TesterInternal.csproj index 85a4b7149a..65838a6ea7 100644 --- a/test/TesterInternal/TesterInternal.csproj +++ b/test/TesterInternal/TesterInternal.csproj @@ -52,6 +52,8 @@ + + @@ -91,6 +93,7 @@ + @@ -120,6 +123,7 @@ + @@ -195,6 +199,10 @@ {606b9647-cc0c-4058-bfbc-b5b7ed4f3c56} OrleansCounterControl + + {cf7ee78b-39e1-4660-beaf-0e2606098f8c} + OrleansEventSourcing + {5c177f58-a40c-449f-8c2f-a2657f963edc} OrleansHost From f774faebcf3f8507c6f6152c63077008b464d0c9 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Wed, 7 Dec 2016 15:34:37 -0800 Subject: [PATCH 02/14] add LogStorage log-consistency provider (stores entire log, using standard storage provider). Rename a bunch of classes to get more consistent naming. Create static class for encapsulating write-vector functionality. --- .../{Protocols => Common}/ConnectionIssues.cs | 2 +- .../NotificationMessage.cs | 2 +- .../NotificationTracker.cs | 2 +- .../PrimaryBasedLogViewAdaptor.cs | 2 +- .../RecordedConnectionIssue.cs | 2 +- .../Common/StringEncodedWriteVector.cs | 57 +++ .../ICustomStorageInterface.cs | 8 +- .../LogConsistencyProvider.cs | 4 +- .../LogViewAdaptor.cs | 4 +- src/OrleansEventSourcing/JournaledGrain.cs | 2 +- .../DefaultAdaptorFactory.cs | 2 +- .../LogConsistencyProvider.cs | 2 +- .../LogStorage/LogStateWithMetaData.cs | 120 ++++++ .../LogStorage/LogViewAdaptor.cs | 376 ++++++++++++++++++ .../OrleansEventSourcing.csproj | 29 +- .../StateStorage/DefaultAdaptorFactory.cs | 30 ++ .../GrainStateWithMetaData.cs | 36 +- .../StateStorage/LogConsistencyProvider.cs | 93 +++++ .../LogViewAdaptor.cs | 18 +- .../LogConsistencyProviderConfiguration.cs | 10 +- .../LogConsistentGrainVariations.cs | 20 +- .../BasicLogConsistentGrainTests.cs | 6 +- .../LogConsistencyTestFixture.cs | 2 +- .../LogConsistencyTestsFourClusters.cs | 10 +- .../LogConsistencyTestsTwoClusters.cs | 10 +- 25 files changed, 769 insertions(+), 80 deletions(-) rename src/OrleansEventSourcing/{Protocols => Common}/ConnectionIssues.cs (98%) rename src/OrleansEventSourcing/{Protocols => Common}/NotificationMessage.cs (97%) rename src/OrleansEventSourcing/{Protocols => Common}/NotificationTracker.cs (99%) rename src/OrleansEventSourcing/{Protocols => Common}/PrimaryBasedLogViewAdaptor.cs (99%) rename src/OrleansEventSourcing/{Protocols => Common}/RecordedConnectionIssue.cs (98%) create mode 100644 src/OrleansEventSourcing/Common/StringEncodedWriteVector.cs rename src/OrleansEventSourcing/{CustomVersionedStateStorage => CustomStorage}/ICustomStorageInterface.cs (74%) rename src/OrleansEventSourcing/{CustomVersionedStateStorage => CustomStorage}/LogConsistencyProvider.cs (96%) rename src/OrleansEventSourcing/{CustomVersionedStateStorage => CustomStorage}/LogViewAdaptor.cs (99%) rename src/OrleansEventSourcing/{VersionedStateStorage => LogStorage}/DefaultAdaptorFactory.cs (93%) rename src/OrleansEventSourcing/{VersionedStateStorage => LogStorage}/LogConsistencyProvider.cs (98%) create mode 100644 src/OrleansEventSourcing/LogStorage/LogStateWithMetaData.cs create mode 100644 src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs create mode 100644 src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs rename src/OrleansEventSourcing/{VersionedStateStorage => StateStorage}/GrainStateWithMetaData.cs (77%) create mode 100644 src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs rename src/OrleansEventSourcing/{VersionedStateStorage => StateStorage}/LogViewAdaptor.cs (93%) diff --git a/src/OrleansEventSourcing/Protocols/ConnectionIssues.cs b/src/OrleansEventSourcing/Common/ConnectionIssues.cs similarity index 98% rename from src/OrleansEventSourcing/Protocols/ConnectionIssues.cs rename to src/OrleansEventSourcing/Common/ConnectionIssues.cs index dc936b5c3c..742e3ce7a4 100644 --- a/src/OrleansEventSourcing/Protocols/ConnectionIssues.cs +++ b/src/OrleansEventSourcing/Common/ConnectionIssues.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Orleans.LogConsistency; -namespace Orleans.EventSourcing.Protocols +namespace Orleans.EventSourcing.Common { /// diff --git a/src/OrleansEventSourcing/Protocols/NotificationMessage.cs b/src/OrleansEventSourcing/Common/NotificationMessage.cs similarity index 97% rename from src/OrleansEventSourcing/Protocols/NotificationMessage.cs rename to src/OrleansEventSourcing/Common/NotificationMessage.cs index 83c91b94aa..ce218b0d5c 100644 --- a/src/OrleansEventSourcing/Protocols/NotificationMessage.cs +++ b/src/OrleansEventSourcing/Common/NotificationMessage.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Orleans.MultiCluster; -namespace Orleans.EventSourcing.Protocols +namespace Orleans.EventSourcing.Common { /// /// Base class for notification messages that are sent by log view adaptors to other diff --git a/src/OrleansEventSourcing/Protocols/NotificationTracker.cs b/src/OrleansEventSourcing/Common/NotificationTracker.cs similarity index 99% rename from src/OrleansEventSourcing/Protocols/NotificationTracker.cs rename to src/OrleansEventSourcing/Common/NotificationTracker.cs index 3e2119f600..ca2d33ba2f 100644 --- a/src/OrleansEventSourcing/Protocols/NotificationTracker.cs +++ b/src/OrleansEventSourcing/Common/NotificationTracker.cs @@ -5,7 +5,7 @@ using System.Text; using System.Threading.Tasks; -namespace Orleans.EventSourcing.Protocols +namespace Orleans.EventSourcing.Common { /// diff --git a/src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs similarity index 99% rename from src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs rename to src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs index 6af513c01e..620b4344d7 100644 --- a/src/OrleansEventSourcing/Protocols/PrimaryBasedLogViewAdaptor.cs +++ b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs @@ -8,7 +8,7 @@ using Orleans.Serialization; using Orleans.MultiCluster; -namespace Orleans.EventSourcing.Protocols +namespace Orleans.EventSourcing.Common { /// /// A general template for constructing log view adaptors that are based on diff --git a/src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs b/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs similarity index 98% rename from src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs rename to src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs index 33700f7b0e..d65d949798 100644 --- a/src/OrleansEventSourcing/Protocols/RecordedConnectionIssue.cs +++ b/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Orleans.LogConsistency; -namespace Orleans.EventSourcing.Protocols +namespace Orleans.EventSourcing.Common { /// /// Utility class for recording connection issues. diff --git a/src/OrleansEventSourcing/Common/StringEncodedWriteVector.cs b/src/OrleansEventSourcing/Common/StringEncodedWriteVector.cs new file mode 100644 index 0000000000..a2ce6c5acd --- /dev/null +++ b/src/OrleansEventSourcing/Common/StringEncodedWriteVector.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.Common +{ + public static class StringEncodedWriteVector + { + + // BitVector of replicas is implemented as a set of replica strings encoded within a string + // The bitvector is represented as the set of replica ids whose bit is 1 + // This set is written as a string that contains the replica ids preceded by a comma each + // + // Assuming our replicas are named A, B, and BB, then + // "" represents {} represents 000 + // ",A" represents {A} represents 100 + // ",A,B" represents {A,B} represents 110 + // ",BB,A,B" represents {A,B,BB} represents 111 + + /// + /// Gets one of the bits in writeVector + /// + /// The replica for which we want to look up the bit + /// + public static bool GetBit(string writeVector, string Replica) + { + var pos = writeVector.IndexOf(Replica); + return pos != -1 && writeVector[pos - 1] == ','; + } + + /// + /// toggle one of the bits in writeVector and return the new value. + /// + /// The write vector in which we want to flip the bit + /// The replica for which we want to flip the bit + /// the state of the bit after flipping it + public static bool FlipBit(ref string writeVector, string Replica) + { + var pos = writeVector.IndexOf(Replica); + if (pos != -1 && writeVector[pos - 1] == ',') + { + var pos2 = writeVector.IndexOf(',', pos + 1); + if (pos2 == -1) + pos2 = writeVector.Length; + writeVector = writeVector.Remove(pos - 1, pos2 - pos + 1); + return false; + } + else + { + writeVector = string.Format(",{0}{1}", Replica, writeVector); + return true; + } + } + } +} diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs b/src/OrleansEventSourcing/CustomStorage/ICustomStorageInterface.cs similarity index 74% rename from src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs rename to src/OrleansEventSourcing/CustomStorage/ICustomStorageInterface.cs index 6e3b2f2dc7..72e28ab747 100644 --- a/src/OrleansEventSourcing/CustomVersionedStateStorage/ICustomStorageInterface.cs +++ b/src/OrleansEventSourcing/CustomStorage/ICustomStorageInterface.cs @@ -4,10 +4,10 @@ using System.Text; using System.Threading.Tasks; -namespace Orleans.EventSourcing.CustomVersionedStateStorage +namespace Orleans.EventSourcing.CustomStorage { /// - /// The storage interface exposed by grains that want to use the CustomVersionedStateStorage log-consistency provider + /// The storage interface exposed by grains that want to use the CustomStorage log-consistency provider /// The type for the state of the grain. /// The type for delta objects that represent updates to the state. /// @@ -21,8 +21,8 @@ public interface ICustomStorageInterface Task> ReadStateFromStorageAsync(); /// - /// Applies the given array of deltas to storage, if the version in storage matches the expected version. - /// Otherwise, does nothing. If successful, the version of storage increases by the number of deltas. + /// Applies the given array of deltas to storage, and returns true, if the version in storage matches the expected version. + /// Otherwise, does nothing and returns false. If successful, the version of storage must be increased by the number of deltas. /// /// true if the deltas were applied, false otherwise Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int expectedversion); diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs similarity index 96% rename from src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs rename to src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs index c9700bc030..c4d6fad507 100644 --- a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogConsistencyProvider.cs +++ b/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs @@ -11,11 +11,11 @@ using Orleans.Providers; using Orleans.Storage; -namespace Orleans.EventSourcing.CustomVersionedStateStorage +namespace Orleans.EventSourcing.CustomStorage { /// /// A log-consistency provider that relies on grain-specific custom code for - /// loading states from storage, and writing deltas to storage. + /// reading states from storage, and appending deltas to storage. /// Grains that wish to use this provider must implement the /// interface, to define how state is read and how deltas are written. /// If the provider attribute "PrimaryCluster" is supplied in the provider configuration, then only the specified cluster diff --git a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs similarity index 99% rename from src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs rename to src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs index 8476ff342b..06f49fd42d 100644 --- a/src/OrleansEventSourcing/CustomVersionedStateStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs @@ -9,9 +9,9 @@ using Orleans.Runtime; using Orleans.Storage; using Orleans.MultiCluster; -using Orleans.EventSourcing.Protocols; +using Orleans.EventSourcing.Common; -namespace Orleans.EventSourcing.CustomVersionedStateStorage +namespace Orleans.EventSourcing.CustomStorage { /// /// A log consistency adaptor that uses the user-provided storage interface . diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index 87a2019173..7295665b43 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -257,7 +257,7 @@ ILogViewAdaptorFactory ILogConsistentGrain.DefaultAdaptorFactory { get { - return new VersionedStateStorage.DefaultAdaptorFactory(); + return new StateStorage.DefaultAdaptorFactory(); } } diff --git a/src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs b/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs similarity index 93% rename from src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs rename to src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs index 8b2ea56756..9e82771335 100644 --- a/src/OrleansEventSourcing/VersionedStateStorage/DefaultAdaptorFactory.cs +++ b/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs @@ -8,7 +8,7 @@ using System.Text; using System.Threading.Tasks; -namespace Orleans.EventSourcing.VersionedStateStorage +namespace Orleans.EventSourcing.LogStorage { internal class DefaultAdaptorFactory : ILogViewAdaptorFactory { diff --git a/src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs similarity index 98% rename from src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs rename to src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs index da8f9b6a0c..d03c04d1ee 100644 --- a/src/OrleansEventSourcing/VersionedStateStorage/LogConsistencyProvider.cs +++ b/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs @@ -11,7 +11,7 @@ using System.Threading; using Orleans.Providers; -namespace Orleans.EventSourcing.VersionedStateStorage +namespace Orleans.EventSourcing.LogStorage { /// /// A log-consistency provider that stores the latest view in primary storage, using any standard storage provider. diff --git a/src/OrleansEventSourcing/LogStorage/LogStateWithMetaData.cs b/src/OrleansEventSourcing/LogStorage/LogStateWithMetaData.cs new file mode 100644 index 0000000000..7802259143 --- /dev/null +++ b/src/OrleansEventSourcing/LogStorage/LogStateWithMetaData.cs @@ -0,0 +1,120 @@ +using Orleans.EventSourcing.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.LogStorage +{ + /// + /// A class that extends grain state with versioning metadata, so that a grain with log-view consistency + /// can use a standard storage provider via + /// + /// The type used for log entries + [Serializable] + public class LogStateWithMetaDataAndETag : IGrainState where TEntry : class + { + /// + /// Gets and Sets StateAndMetaData + /// + public LogStateWithMetaData StateAndMetaData { get; set; } + + /// + /// Gets and Sets Etag + /// + public string ETag { get; set; } + + object IGrainState.State + { + get + { + return StateAndMetaData; + } + set + { + StateAndMetaData = (LogStateWithMetaData)value; + } + } + + /// + /// Initializes a new instance of GrainStateWithMetaDataAndETag class + /// + public LogStateWithMetaDataAndETag() + { + StateAndMetaData = new LogStateWithMetaData(); + } + + /// + /// Convert current GrainStateWithMetaDataAndETag object information to a string + /// + public override string ToString() + { + return string.Format("v{0} Flags={1} ETag={2} Data={3}", StateAndMetaData.GlobalVersion, StateAndMetaData.WriteVector, ETag, StateAndMetaData.Log); + } + } + + + /// + /// A class that extends grain state with versioning metadata, so that a log-consistent grain + /// can use a standard storage provider via + /// + /// + [Serializable] + public class LogStateWithMetaData where TEntry : class + { + /// + /// The stored view of the log + /// + public List Log { get; set; } + + /// + /// The length of the log + /// + public int GlobalVersion { get { return Log.Count; } } + + + /// + /// Metadata that is used to avoid duplicate appends. + /// Logically, this is a (string->bit) map, the keys being replica ids + /// But this map is represented compactly as a simple string to reduce serialization/deserialization overhead + /// Bits are read by and flipped by . + /// Bits are toggled when writing, so that the retry logic can avoid appending an entry twice + /// when retrying a failed append. + /// + public string WriteVector { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public LogStateWithMetaData() + { + Log = new List(); + WriteVector = ""; + } + + + /// + /// Gets one of the bits in + /// + /// The replica for which we want to look up the bit + /// + public bool GetBit(string Replica) { + return StringEncodedWriteVector.GetBit(WriteVector, Replica); + } + + /// + /// toggle one of the bits in and return the new value. + /// + /// The replica for which we want to flip the bit + /// the state of the bit after flipping it + public bool FlipBit(string Replica) { + var str = WriteVector; + var rval = StringEncodedWriteVector.FlipBit(ref str, Replica); + WriteVector = str; + return rval; + } + + + } +} diff --git a/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs new file mode 100644 index 0000000000..45a5a25ae3 --- /dev/null +++ b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs @@ -0,0 +1,376 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Storage; +using Orleans.EventSourcing.Common; + +namespace Orleans.EventSourcing.LogStorage +{ + /// + /// A log view adaptor that wraps around a traditional storage adaptor, and uses batching and e-tags + /// to append entries. + /// + /// The log itself is transient, i.e. not actually saved to storage - only the latest view and some + /// metadata (the log position, and write flags) are stored. + /// + /// + /// Type of log view + /// Type of log entry + internal class LogViewAdaptor : PrimaryBasedLogViewAdaptor> where TLogView : class, new() where TLogEntry : class + { + /// + /// Initialize a StorageProviderLogViewAdaptor class + /// + public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, IProtocolServices services) + : base(host, initialState, services) + { + this.globalStorageProvider = globalStorageProvider; + this.grainTypeName = grainTypeName; + } + + + private const int maxEntriesInNotifications = 200; + + + IStorageProvider globalStorageProvider; + string grainTypeName; + + // the object containing the entire log, as retrieved from / sent to storage + LogStateWithMetaDataAndETag GlobalLog; + + // the confirmed view + TLogView ConfirmedViewInternal; + int ConfirmedVersionInternal; + + /// + protected override TLogView LastConfirmedView() + { + return ConfirmedViewInternal; + } + + /// + protected override int GetConfirmedVersion() + { + return ConfirmedVersionInternal; + } + + /// + protected override void InitializeConfirmedView(TLogView initialstate) + { + GlobalLog = new LogStateWithMetaDataAndETag(); + ConfirmedViewInternal = initialstate; + ConfirmedVersionInternal = 0; + } + + private void UpdateConfirmedView() + { + for (int i = ConfirmedVersionInternal; i < GlobalLog.StateAndMetaData.Log.Count; i++) + { + try + { + Host.UpdateView(ConfirmedViewInternal, GlobalLog.StateAndMetaData.Log[i]); + } + catch (Exception e) + { + Services.CaughtUserCodeException("UpdateView", nameof(UpdateConfirmedView), e); + } + } + ConfirmedVersionInternal = GlobalLog.StateAndMetaData.GlobalVersion; + } + + // no special tagging is required, thus we create a plain submission entry + /// + protected override SubmissionEntry MakeSubmissionEntry(TLogEntry entry) + { + return new SubmissionEntry() { Entry = entry }; + } + + + /// + protected override async Task ReadAsync() + { + enter_operation("ReadAsync"); + + while (true) + { + try + { + // for manual testing + //await Task.Delay(5000); + + await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); + + Services.Verbose("read success {0}", GlobalLog); + + UpdateConfirmedView(); + + LastPrimaryIssue.Resolve(Host, Services); + + break; // successful + } + catch (Exception e) + { + LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + + await LastPrimaryIssue.DelayBeforeRetry(); + } + + exit_operation("ReadAsync"); + } + + + /// + protected override async Task WriteAsync() + { + enter_operation("WriteAsync"); + + var updates = GetCurrentBatchOfUpdates(); + bool batchsuccessfullywritten = false; + + var writebit = GlobalLog.StateAndMetaData.FlipBit(Services.MyClusterId); + foreach (var x in updates) + GlobalLog.StateAndMetaData.Log.Add(x.Entry); + + try + { + // for manual testing + //await Task.Delay(5000); + + await globalStorageProvider.WriteStateAsync(grainTypeName, Services.GrainReference, GlobalLog); + + batchsuccessfullywritten = true; + + Services.Verbose("write ({0} updates) success {1}", updates.Length, GlobalLog); + + UpdateConfirmedView(); + + LastPrimaryIssue.Resolve(Host, Services); + } + catch (Exception e) + { + LastPrimaryIssue.Record(new UpdateLogStorageFailed() { Exception = e }, Host, Services); + } + + if (!batchsuccessfullywritten) + { + Services.Verbose("write apparently failed {0}", LastPrimaryIssue); + + while (true) // be stubborn until we can read what is there + { + + await LastPrimaryIssue.DelayBeforeRetry(); + + try + { + await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); + + Services.Verbose("read success {0}", GlobalLog); + + UpdateConfirmedView(); + + LastPrimaryIssue.Resolve(Host, Services); + + break; + } + catch (Exception e) + { + LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); + } + + Services.Verbose("read failed {0}", LastPrimaryIssue); + } + + // check if last apparently failed write was in fact successful + + if (writebit == GlobalLog.StateAndMetaData.GetBit(Services.MyClusterId)) + { + Services.Verbose("last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog); + + batchsuccessfullywritten = true; + } + } + + + // broadcast notifications to all other clusters + if (batchsuccessfullywritten) + BroadcastNotification(new UpdateNotificationMessage() + { + Version = GlobalLog.StateAndMetaData.GlobalVersion, + Updates = updates.Select(se => se.Entry).ToList(), + Origin = Services.MyClusterId, + ETag = GlobalLog.ETag + }); + + exit_operation("WriteAsync"); + + if (!batchsuccessfullywritten) + return 0; + + return updates.Length; + } + + + /// + /// Describes a connection issue that occurred when updating the primary storage. + /// + [Serializable] + public class UpdateLogStorageFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"write entire log to storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// Describes a connection issue that occurred when reading from the primary storage. + /// + [Serializable] + public class ReadFromLogStorageFailed : PrimaryOperationFailed + { + /// + public override string ToString() + { + return $"read entire log from storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + } + } + + + /// + /// A notification message sent to remote instances after updating this grain in storage. + /// + [Serializable] + protected class UpdateNotificationMessage : INotificationMessage + { + /// + public int Version { get; set; } + + /// The cluster that performed the update + public string Origin { get; set; } + + /// The list of updates that were applied + public List Updates { get; set; } + + /// The e-tag of the storage after applying the updates + public string ETag { get; set; } + + /// + public override string ToString() + { + return string.Format("v{0} ({1} updates by {2}) etag={2}", Version, Updates.Count, Origin, ETag); + } + } + + /// + protected override INotificationMessage Merge(INotificationMessage earlierMessage, INotificationMessage laterMessage) + { + var earlier = earlierMessage as UpdateNotificationMessage; + var later = laterMessage as UpdateNotificationMessage; + + if (earlier != null + && later != null + && earlier.Origin == later.Origin + && earlier.Version + later.Updates.Count == later.Version + && earlier.Updates.Count + later.Updates.Count < maxEntriesInNotifications) + + return new UpdateNotificationMessage() + { + Version = later.Version, + Origin = later.Origin, + Updates = earlier.Updates.Concat(later.Updates).ToList(), + ETag = later.ETag + }; + + else + return base.Merge(earlierMessage, laterMessage); // keep only the version number + } + + private SortedList notifications = new SortedList(); + + /// + protected override void OnNotificationReceived(INotificationMessage payload) + { + var um = payload as UpdateNotificationMessage; + if (um != null) + notifications.Add(um.Version - um.Updates.Count, um); + else + base.OnNotificationReceived(payload); + } + + /// + protected override void ProcessNotifications() + { + // discard notifications that are behind our already confirmed state + while (notifications.Count > 0 && notifications.ElementAt(0).Key < GlobalLog.StateAndMetaData.GlobalVersion) + { + Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + notifications.RemoveAt(0); + } + + // process notifications that reflect next global version + while (notifications.Count > 0 && notifications.ElementAt(0).Key == GlobalLog.StateAndMetaData.GlobalVersion) + { + var updateNotification = notifications.ElementAt(0).Value; + notifications.RemoveAt(0); + + // append all operations in pending + foreach (var u in updateNotification.Updates) + GlobalLog.StateAndMetaData.Log.Add(u); + + GlobalLog.StateAndMetaData.FlipBit(updateNotification.Origin); + + GlobalLog.ETag = updateNotification.ETag; + + UpdateConfirmedView(); + + Services.Verbose("notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalLog); + } + + Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + + base.ProcessNotifications(); + + } + + + #region non-reentrancy assertions + +#if DEBUG + bool operation_in_progress; +#endif + + [Conditional("DEBUG")] + private void enter_operation(string name) + { +#if DEBUG + Services.Verbose2("/-- enter {0}", name); + Debug.Assert(!operation_in_progress); + operation_in_progress = true; +#endif + } + + [Conditional("DEBUG")] + private void exit_operation(string name) + { +#if DEBUG + Services.Verbose2("\\-- exit {0}", name); + Debug.Assert(operation_in_progress); + operation_in_progress = false; +#endif + } + + + + #endregion + } +} diff --git a/src/OrleansEventSourcing/OrleansEventSourcing.csproj b/src/OrleansEventSourcing/OrleansEventSourcing.csproj index 7f93c6b209..5cea4e5ab4 100644 --- a/src/OrleansEventSourcing/OrleansEventSourcing.csproj +++ b/src/OrleansEventSourcing/OrleansEventSourcing.csproj @@ -45,19 +45,24 @@ Properties\GlobalAssemblyInfo.cs - - - - - + + + + + + + + + + - - - - - - - + + + + + + + diff --git a/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs b/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs new file mode 100644 index 0000000000..3e6b242548 --- /dev/null +++ b/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs @@ -0,0 +1,30 @@ +using Orleans.LogConsistency; +using Orleans.Providers; +using Orleans.Runtime; +using Orleans.Storage; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.EventSourcing.StateStorage +{ + internal class DefaultAdaptorFactory : ILogViewAdaptorFactory + { + public bool UsesStorageProvider + { + get + { + return true; + } + } + + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + where T : class, new() where E : class + { + return new LogViewAdaptor(hostgrain, initialstate, storageProvider, graintypename, services); + } + + } +} diff --git a/src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs b/src/OrleansEventSourcing/StateStorage/GrainStateWithMetaData.cs similarity index 77% rename from src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs rename to src/OrleansEventSourcing/StateStorage/GrainStateWithMetaData.cs index 8ac07e0659..fb1b69147d 100644 --- a/src/OrleansEventSourcing/VersionedStateStorage/GrainStateWithMetaData.cs +++ b/src/OrleansEventSourcing/StateStorage/GrainStateWithMetaData.cs @@ -1,10 +1,11 @@ -using System; +using Orleans.EventSourcing.Common; +using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; -namespace Orleans.EventSourcing.VersionedStateStorage +namespace Orleans.EventSourcing.StateStorage { /// /// A class that extends grain state with versioning metadata, so that a grain with log-view consistency @@ -112,15 +113,6 @@ public GrainStateWithMetaData(TView initialstate) WriteVector = ""; } - // BitVector of replicas is implemented as a set of replica strings encoded within a string - // The bitvector is represented as the set of replica ids whose bit is 1 - // This set is written as a string that contains the replica ids preceded by a comma each - // - // Assuming our replicas are named A, B, and BB, then - // "" represents {} represents 000 - // ",A" represents {A} represents 100 - // ",A,B" represents {A,B} represents 110 - // ",BB,A,B" represents {A,B,BB} represents 111 /// /// Gets one of the bits in @@ -129,8 +121,7 @@ public GrainStateWithMetaData(TView initialstate) /// public bool GetBit(string Replica) { - var pos = WriteVector.IndexOf(Replica); - return pos != -1 && WriteVector[pos - 1] == ','; + return StringEncodedWriteVector.GetBit(WriteVector, Replica); } /// @@ -140,21 +131,10 @@ public bool GetBit(string Replica) /// the state of the bit after flipping it public bool FlipBit(string Replica) { - var pos = WriteVector.IndexOf(Replica); - if (pos != -1 && WriteVector[pos - 1] == ',') - { - var pos2 = WriteVector.IndexOf(',', pos + 1); - if (pos2 == -1) - pos2 = WriteVector.Length; - WriteVector = WriteVector.Remove(pos - 1, pos2 - pos + 1); - return false; - } - else - { - WriteVector = string.Format(",{0}{1}", Replica, WriteVector); - return true; - } + var str = WriteVector; + var rval = StringEncodedWriteVector.FlipBit(ref str, Replica); + WriteVector = str; + return rval; } - } } diff --git a/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs new file mode 100644 index 0000000000..f56221b222 --- /dev/null +++ b/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Orleans; +using Orleans.LogConsistency; +using Orleans.Runtime; +using Orleans.Storage; +using System.Threading; +using Orleans.Providers; + +namespace Orleans.EventSourcing.StateStorage +{ + /// + /// A log-consistency provider that stores the latest view in primary storage, using any standard storage provider. + /// Supports multiple clusters connecting to the same primary storage (doing optimistic concurrency control via e-tags) + /// + /// The log itself is transient, i.e. not actually saved to storage - only the latest view (snapshot) and some + /// metadata (the log position, and write flags) are stored in the primary. + /// + /// + public class LogConsistencyProvider : ILogConsistencyProvider + { + /// + public string Name { get; private set; } + + /// + public Logger Log { get; private set; } + + /// + public bool UsesStorageProvider + { + get + { + return true; + } + } + + private static int counter; // used for constructing a unique id + private int id; + + /// + protected virtual string GetLoggerName() + { + return string.Format("LogViews.{0}.{1}", GetType().Name, id); + } + + /// + /// Init method + /// + /// Consistency provider name + /// Provider runtime + /// Provider config + public Task Init(string name, IProviderRuntime providerRuntime, IProviderConfiguration config) + { + Name = name; + id = Interlocked.Increment(ref counter); // unique id for this provider; matters only for tracing + + Log = providerRuntime.GetLogger(GetLoggerName()); + Log.Info("Init (Severity={0})", Log.SeverityLevel); + + return TaskDone.Done; + } + + /// + /// Close method + /// + public Task Close() + { + return TaskDone.Done; + } + + /// + /// Make log view adaptor + /// + /// The type of the view + /// The type of the log entries + /// The grain that is hosting this adaptor + /// The initial state for this view + /// The type name of the grain + /// Runtime services for multi-cluster coherence protocols + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services) + where TView : class, new() + where TEntry : class + { + return new LogViewAdaptor(hostGrain, initialState, storageProvider, grainTypeName, services); + } + + } + +} \ No newline at end of file diff --git a/src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs similarity index 93% rename from src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs rename to src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs index 075e519413..0dcf8ef344 100644 --- a/src/OrleansEventSourcing/VersionedStateStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs @@ -7,9 +7,9 @@ using Orleans; using Orleans.LogConsistency; using Orleans.Storage; -using Orleans.EventSourcing.Protocols; +using Orleans.EventSourcing.Common; -namespace Orleans.EventSourcing.VersionedStateStorage +namespace Orleans.EventSourcing.StateStorage { /// /// A log view adaptor that wraps around a traditional storage adaptor, and uses batching and e-tags @@ -88,7 +88,7 @@ protected override async Task ReadAsync() } catch (Exception e) { - LastPrimaryIssue.Record(new ReadFromStorageFailed() { Exception = e }, Host, Services); + LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } Services.Verbose("read failed {0}", LastPrimaryIssue); @@ -133,7 +133,7 @@ protected override async Task WriteAsync() } catch (Exception e) { - LastPrimaryIssue.Record(new UpdateStorageFailed() { Exception = e }, Host, Services); + LastPrimaryIssue.Record(new UpdateStateStorageFailed() { Exception = e }, Host, Services); } if (!batchsuccessfullywritten) @@ -157,7 +157,7 @@ protected override async Task WriteAsync() } catch (Exception e) { - LastPrimaryIssue.Record(new ReadFromStorageFailed() { Exception = e }, Host, Services); + LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } Services.Verbose("read failed {0}", LastPrimaryIssue); @@ -199,12 +199,12 @@ protected override async Task WriteAsync() /// Describes a connection issue that occurred when updating the primary storage. /// [Serializable] - public class UpdateStorageFailed : PrimaryOperationFailed + public class UpdateStateStorageFailed : PrimaryOperationFailed { /// public override string ToString() { - return $"update storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + return $"write state to storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; } } @@ -213,12 +213,12 @@ public override string ToString() /// Describes a connection issue that occurred when reading from the primary storage. /// [Serializable] - public class ReadFromStorageFailed : PrimaryOperationFailed + public class ReadFromStateStorageFailed : PrimaryOperationFailed { /// public override string ToString() { - return $"read from storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; + return $"read state from storage failed: caught {Exception.GetType().Name}: {Exception.Message}"; } } diff --git a/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs b/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs index fc5b39dc25..25d289022d 100644 --- a/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs +++ b/src/OrleansTestingHost/LogConsistencyProviderConfiguration.cs @@ -31,16 +31,20 @@ public static void ConfigureLogConsistencyProvidersForTesting(string dataConnect } { var props = new Dictionary(); - config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.VersionedStateStorage.LogConsistencyProvider", "VersionedStateStorage", props); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.StateStorage.LogConsistencyProvider", "StateStorage", props); } { var props = new Dictionary(); - config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomVersionedStateStorage.LogConsistencyProvider", "CustomStorage", props); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.LogStorage.LogConsistencyProvider", "LogStorage", props); + } + { + var props = new Dictionary(); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomStorage.LogConsistencyProvider", "CustomStorage", props); } { var props = new Dictionary(); props.Add("PrimaryCluster", "A"); - config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomVersionedStateStorage.LogConsistencyProvider", "CustomStoragePrimaryCluster", props); + config.Globals.RegisterLogConsistencyProvider("Orleans.EventSourcing.CustomStorage.LogConsistencyProvider", "CustomStoragePrimaryCluster", props); } // logging diff --git a/test/TestGrains/LogConsistentGrainVariations.cs b/test/TestGrains/LogConsistentGrainVariations.cs index 1edf56dac1..ea7e94117b 100644 --- a/test/TestGrains/LogConsistentGrainVariations.cs +++ b/test/TestGrains/LogConsistentGrainVariations.cs @@ -15,8 +15,16 @@ namespace UnitTests.Grains // use azure storage and a explicitly configured consistency provider [OneInstancePerCluster] [StorageProvider(ProviderName = "AzureStore")] - [LogConsistencyProvider(ProviderName = "VersionedStateStorage")] - public class LogConsistentGrainSharedStorage : LogConsistentGrain + [LogConsistencyProvider(ProviderName = "StateStorage")] + public class LogConsistentGrainSharedStateStorage : LogConsistentGrain + { + } + + // use azure storage and a explicitly configured consistency provider + [OneInstancePerCluster] + [StorageProvider(ProviderName = "AzureStore")] + [LogConsistencyProvider(ProviderName = "LogStorage")] + public class LogConsistentGrainSharedLogStorage : LogConsistentGrain { } @@ -29,7 +37,7 @@ public class LogConsistentGrainDefaultStorage : LogConsistentGrain // use a single-instance log-consistent grain [GlobalSingleInstance] [StorageProvider(ProviderName = "AzureStore")] - [LogConsistencyProvider(ProviderName = "VersionedStateStorage")] + [LogConsistencyProvider(ProviderName = "StateStorage")] public class GsiLogConsistentGrain : LogConsistentGrain { } @@ -45,7 +53,7 @@ public class LogConsistentGrainMemoryStorage : LogConsistentGrain [OneInstancePerCluster] [LogConsistencyProvider(ProviderName = "CustomStorage")] public class LogConsistentGrainCustomStorage : LogConsistentGrain, - Orleans.EventSourcing.CustomVersionedStateStorage.ICustomStorageInterface + Orleans.EventSourcing.CustomStorage.ICustomStorageInterface { // we use another impl of this grain as the primary. @@ -55,7 +63,7 @@ private ILogConsistentGrain GetStorageGrain() { if (storagegrain == null) { - storagegrain = GrainFactory.GetGrain(this.GetPrimaryKeyLong(), "UnitTests.Grains.LogConsistentGrainSharedStorage"); + storagegrain = GrainFactory.GetGrain(this.GetPrimaryKeyLong(), "UnitTests.Grains.LogConsistentGrainSharedStateStorage"); } return storagegrain; } @@ -77,7 +85,7 @@ public Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int [OneInstancePerCluster] [LogConsistencyProvider(ProviderName = "CustomStoragePrimaryCluster")] public class LogConsistentGrainCustomStoragePrimaryCluster : LogConsistentGrain, - Orleans.EventSourcing.CustomVersionedStateStorage.ICustomStorageInterface + Orleans.EventSourcing.CustomStorage.ICustomStorageInterface { // we use fake in-memory state as the storage diff --git a/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs index e124f635c8..3a0c5cc17d 100644 --- a/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs +++ b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs @@ -45,9 +45,13 @@ public async Task MemoryStorage() [Fact, TestCategory("GeoCluster")] public async Task SharedStorage() { - await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainSharedStorage"); + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainSharedStateStorage"); } [Fact, TestCategory("GeoCluster")] + public async Task SharedLogStorage() + { + await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainSharedLogStorage"); + } public async Task CustomStorage() { await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainCustomStorage"); diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs index 4c625e2543..2655c395ee 100644 --- a/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestFixture.cs @@ -12,7 +12,7 @@ using Xunit; using Xunit.Abstractions; using TestExtensions; -using Orleans.EventSourcing.Protocols; +using Orleans.EventSourcing.Common; namespace Tests.GeoClusterTests { diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs index 3c6be48a0e..0feff32f47 100644 --- a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs @@ -31,9 +31,15 @@ public class Fixture : LogConsistencyTestFixture const int phases = 100; [Fact, TestCategory("GeoCluster")] - public async Task TestBattery_SharedStorageProvider() + public async Task TestBattery_SharedStateStorageProvider() { - await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStorage", true, phases); + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStateStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_SharedLogStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedLogStorage", true, phases); } [Fact, TestCategory("GeoCluster")] diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs index 163261538d..54855f4516 100644 --- a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs @@ -30,9 +30,15 @@ public class Fixture : LogConsistencyTestFixture const int phases = 100; [Fact, TestCategory("GeoCluster")] - public async Task TestBattery_SharedStorageProvider() + public async Task TestBattery_SharedStateStorageProvider() { - await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStorage", true, phases); + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedStateStorage", true, phases); + } + + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_SharedLogStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainSharedLogStorage", true, phases); } [Fact, TestCategory("GeoCluster")] From d6a19806f0b14116b9418c36f62c259c4d535f1d Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 8 Dec 2016 15:02:02 -0800 Subject: [PATCH 03/14] change some details to fix vNext build --- .../Configuration/GlobalConfiguration.cs | 5 +++-- .../MultiClusterRegistrationStrategy.cs | 8 +++----- .../LogConsistency/IProtocolServices.cs | 3 +++ src/OrleansRuntime/Catalog/Catalog.cs | 4 +++- src/OrleansRuntime/Catalog/GrainCreator.cs | 7 +++++-- .../GrainTypeManager/GrainTypeData.cs | 1 + .../EventSourcing/JournaledPerson_Events.cs | 18 ++++++++++-------- test/TestGrains/LogConsistentGrain.cs | 4 ---- .../TestGrains/LogConsistentGrainVariations.cs | 2 +- 9 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Orleans/Configuration/GlobalConfiguration.cs b/src/Orleans/Configuration/GlobalConfiguration.cs index e282e0fa6c..ce05ae6d64 100644 --- a/src/Orleans/Configuration/GlobalConfiguration.cs +++ b/src/Orleans/Configuration/GlobalConfiguration.cs @@ -1092,8 +1092,9 @@ public void RegisterLogConsistencyProvider(string providerTypeFullName, string p public void RegisterLogConsistencyProvider(string providerName, IDictionary properties = null) where T : ILogConsistencyProvider { Type providerType = typeof(T); - if (providerType.IsAbstract || - providerType.IsGenericType || + var providerTypeInfo = providerType.GetTypeInfo(); + if (providerTypeInfo.IsAbstract || + providerTypeInfo.IsGenericType || !typeof(ILogConsistencyProvider).IsAssignableFrom(providerType)) throw new ArgumentException("Expected non-generic, non-abstract type which implements ILogConsistencyProvider interface", "typeof(T)"); diff --git a/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs b/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs index 9bc59c0c16..9b641996c9 100644 --- a/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs +++ b/src/Orleans/GrainDirectory/MultiClusterRegistrationStrategy.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Orleans.Runtime.Configuration; using Orleans.MultiCluster; using System.Collections.Generic; @@ -51,12 +52,9 @@ internal static MultiClusterRegistrationStrategy GetDefault() return defaultStrategy; } - internal static MultiClusterRegistrationStrategy FromGrainType(Type graintype) + internal static MultiClusterRegistrationStrategy FromAttributes(IEnumerable attributes) { - var attrs = graintype.GetCustomAttributes(typeof(RegistrationAttribute), true); - if (attrs.Length == 0) - return defaultStrategy; - return ((RegistrationAttribute)attrs[0]).RegistrationStrategy; + return (attributes.FirstOrDefault() as RegistrationAttribute)?.RegistrationStrategy ?? defaultStrategy; } public abstract IEnumerable GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId); diff --git a/src/Orleans/LogConsistency/IProtocolServices.cs b/src/Orleans/LogConsistency/IProtocolServices.cs index 97c433dbd1..20a12fc3e0 100644 --- a/src/Orleans/LogConsistency/IProtocolServices.cs +++ b/src/Orleans/LogConsistency/IProtocolServices.cs @@ -121,9 +121,12 @@ public ProtocolTransportException(string msg) public ProtocolTransportException(string msg, Exception exc) : base(msg, exc) { } + +#if !NETSTANDARD protected ProtocolTransportException(SerializationInfo info, StreamingContext context) : base(info, context) { } +#endif public override string ToString() { diff --git a/src/OrleansRuntime/Catalog/Catalog.cs b/src/OrleansRuntime/Catalog/Catalog.cs index 4ee4798dd0..4a158b702b 100644 --- a/src/OrleansRuntime/Catalog/Catalog.cs +++ b/src/OrleansRuntime/Catalog/Catalog.cs @@ -696,7 +696,9 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri else if (grain is ILogConsistentGrain) { var consistencyProvider = SetupLogConsistencyProvider(grain, grainType, data); - grainCreator.InstallLogViewAdaptor(grain, grainType, grainTypeData.StateObjectType, consistencyProvider, data.StorageProvider); + grainCreator.InstallLogViewAdaptor(grain, grainType, + grainTypeData.StateObjectType, grainTypeData.MultiClusterRegistrationStrategy, + consistencyProvider, data.StorageProvider); } grain.Data = data; diff --git a/src/OrleansRuntime/Catalog/GrainCreator.cs b/src/OrleansRuntime/Catalog/GrainCreator.cs index 0a63ff020f..7f00edd850 100644 --- a/src/OrleansRuntime/Catalog/GrainCreator.cs +++ b/src/OrleansRuntime/Catalog/GrainCreator.cs @@ -86,16 +86,19 @@ public void InstallStorageBridge(Grain grain, Type grainType, Type stateType, IS /// The grain. /// The grain type. /// The type of the grain state. + /// The multi-cluster registration strategy. /// The consistency adaptor factory /// The storage provider, or null if none needed /// The newly created grain. - public void InstallLogViewAdaptor(Grain grain, Type grainType, Type stateType, ILogViewAdaptorFactory factory, IStorageProvider storageProvider) + public void InstallLogViewAdaptor(Grain grain, Type grainType, + Type stateType, IMultiClusterRegistrationStrategy mcRegistrationStrategy, + ILogViewAdaptorFactory factory, IStorageProvider storageProvider) { // try to find a suitable logger that we can use to trace consistency protocol information var logger = (factory as ILogConsistencyProvider)?.Log ?? storageProvider?.Log; // encapsulate runtime services used by consistency adaptors - var svc = new ProtocolServices(grain, logger, MultiClusterRegistrationStrategy.FromGrainType(grain.GetType())); + var svc = new ProtocolServices(grain, logger, mcRegistrationStrategy); var state = Activator.CreateInstance(stateType); diff --git a/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs b/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs index ff4d1b769e..f53667e617 100644 --- a/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs +++ b/src/OrleansRuntime/GrainTypeManager/GrainTypeData.cs @@ -39,6 +39,7 @@ public GrainTypeData(Type type, Type stateObjectType) RemoteInterfaceTypes = GetRemoteInterfaces(type); ; StateObjectType = stateObjectType; MayInterleave = GetMayInterleavePredicate(typeInfo) ?? (_ => false); + MultiClusterRegistrationStrategy = MultiClusterRegistrationStrategy.FromAttributes(typeInfo.GetCustomAttributes(typeof(RegistrationAttribute), true)); } /// diff --git a/test/TestGrains/EventSourcing/JournaledPerson_Events.cs b/test/TestGrains/EventSourcing/JournaledPerson_Events.cs index 9dcf6dcc45..9707471d6b 100644 --- a/test/TestGrains/EventSourcing/JournaledPerson_Events.cs +++ b/test/TestGrains/EventSourcing/JournaledPerson_Events.cs @@ -6,12 +6,12 @@ namespace TestGrains // we use a marker interface, so we get a bit more typechecking than with plain objects public interface IPersonEvent { } - + [Serializable] public class PersonRegistered : IPersonEvent { - public string FirstName { get; private set; } - public string LastName { get; private set; } - public GenderType Gender { get; private set; } + public string FirstName { get; set; } + public string LastName { get; set; } + public GenderType Gender { get; set; } public PersonRegistered(string firstName, string lastName, GenderType gender) { @@ -21,11 +21,12 @@ public PersonRegistered(string firstName, string lastName, GenderType gender) } } + [Serializable] public class PersonMarried : IPersonEvent { - public Guid SpouseId { get; private set; } - public string SpouseFirstName { get; private set; } - public string SpouseLastName { get; private set; } + public Guid SpouseId { get; set; } + public string SpouseFirstName { get; set; } + public string SpouseLastName { get; set; } public PersonMarried(Guid spouseId, string spouseFirstName, string spouseLastName) { @@ -35,9 +36,10 @@ public PersonMarried(Guid spouseId, string spouseFirstName, string spouseLastNam } } + [Serializable] public class PersonLastNameChanged : IPersonEvent { - public string LastName { get; private set; } + public string LastName { get; set; } public PersonLastNameChanged(string lastName) { diff --git a/test/TestGrains/LogConsistentGrain.cs b/test/TestGrains/LogConsistentGrain.cs index 0621695494..ed544c408a 100644 --- a/test/TestGrains/LogConsistentGrain.cs +++ b/test/TestGrains/LogConsistentGrain.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Threading.Tasks; using Orleans; -using Orleans.Providers; using Orleans.LogConsistency; using UnitTests.GrainInterfaces; using Orleans.EventSourcing; @@ -28,9 +27,6 @@ public override string ToString() return string.Format("A={0} B={1} R={{{2}}}", A, B, string.Join(", ", Reservations.Select(kvp => string.Format("{0}:{1}", kvp.Key, kvp.Value)))); } - // dynamic dispatch to the cases listed below - public void Apply(dynamic o) { Apply(o); } - // all the update operations are listed here public void Apply(UpdateA x) { A = x.Val; } public void Apply(UpdateB x) { B = x.Val; } diff --git a/test/TestGrains/LogConsistentGrainVariations.cs b/test/TestGrains/LogConsistentGrainVariations.cs index ea7e94117b..9ab45f1644 100644 --- a/test/TestGrains/LogConsistentGrainVariations.cs +++ b/test/TestGrains/LogConsistentGrainVariations.cs @@ -105,7 +105,7 @@ public Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int foreach (var u in updates) { - state.Apply(u); + this.TransitionState(state, u); version++; } From c00cbf1ee8cee7e444ee1219f3c3bf32597429a2 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 9 Dec 2016 14:48:44 -0800 Subject: [PATCH 04/14] add two tests and one JournaledGrain method that I accidentally omitted. --- src/OrleansEventSourcing/JournaledGrain.cs | 9 +++++++++ .../GeoClusterTests/LogConsistencyTestsFourClusters.cs | 6 ++++++ .../GeoClusterTests/LogConsistencyTestsTwoClusters.cs | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index 7295665b43..a202decc14 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -165,6 +165,15 @@ protected Task FetchAllEventsNow() } + /// + /// Returns the current queue of unconfirmed events. + /// + public IEnumerable UnconfirmedUpdates + { + get { return LogViewAdaptor.UnconfirmedSuffix; } + } + + /// /// Called when the underlying persistence or replication protocol is running into some sort of connection trouble. /// Override this to monitor the health of the log-consistency protocol and/or diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs index 0feff32f47..8f6c31cb5e 100644 --- a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsFourClusters.cs @@ -48,6 +48,12 @@ public async Task TestBattery_GsiDefaultStorageProvider() await fixture.RunChecksOnGrainClass("UnitTests.Grains.GsiLogConsistentGrain", true, phases); } + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_MemoryStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainMemoryStorage", true, phases); + } + [Fact, TestCategory("GeoCluster")] public async Task TestBattery_CustomStorageProvider() { diff --git a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs index 54855f4516..aea1eb5d4b 100644 --- a/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs +++ b/test/TesterInternal/GeoClusterTests/LogConsistencyTestsTwoClusters.cs @@ -47,6 +47,11 @@ public async Task TestBattery_GsiDefaultStorageProvider() await fixture.RunChecksOnGrainClass("UnitTests.Grains.GsiLogConsistentGrain", true, phases); } + [Fact, TestCategory("GeoCluster")] + public async Task TestBattery_MemoryStorageProvider() + { + await fixture.RunChecksOnGrainClass("UnitTests.Grains.LogConsistentGrainMemoryStorage", true, phases); + } [Fact, TestCategory("GeoCluster")] public async Task TestBattery_CustomStorageProvider() From c6fe836ccb66b9a4b683509dd7ae5c08e425df72 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 9 Dec 2016 14:53:53 -0800 Subject: [PATCH 05/14] copy-paste error in previous commit - used wrong name for method. Sigh. --- src/OrleansEventSourcing/JournaledGrain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index a202decc14..340f7e6484 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -168,7 +168,7 @@ protected Task FetchAllEventsNow() /// /// Returns the current queue of unconfirmed events. /// - public IEnumerable UnconfirmedUpdates + public IEnumerable UnconfirmedEvents { get { return LogViewAdaptor.UnconfirmedSuffix; } } From 7c9008a537c6cb888d3fa68adc31f4dc21368ac9 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Fri, 9 Dec 2016 15:22:14 -0800 Subject: [PATCH 06/14] rename FetchAllEventsNow() to RefreshNow() because the former is too specific and not always accurate (depending on the consistency provider) --- src/OrleansEventSourcing/JournaledGrain.cs | 4 ++-- test/TestGrains/LogConsistentGrain.cs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index 340f7e6484..8ab99362b6 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -155,11 +155,11 @@ protected Task ConfirmEvents() } /// - /// Retrieves all events now, and confirms all previously raised events. + /// Retrieves the latest state now, and confirms all previously raised events. /// Effectively, this enforces synchronization with the global state. /// /// - protected Task FetchAllEventsNow() + protected Task RefreshNow() { return LogViewAdaptor.SynchronizeNowAsync(); } diff --git a/test/TestGrains/LogConsistentGrain.cs b/test/TestGrains/LogConsistentGrain.cs index ed544c408a..a5a91f9f38 100644 --- a/test/TestGrains/LogConsistentGrain.cs +++ b/test/TestGrains/LogConsistentGrain.cs @@ -103,7 +103,7 @@ public Task IncrementALocal() public async Task GetAGlobal() { - await FetchAllEventsNow(); + await RefreshNow(); return ConfirmedState.A; } @@ -114,7 +114,7 @@ public Task GetALocal() public async Task GetBothGlobal() { - await FetchAllEventsNow(); + await RefreshNow(); return new AB() { A = ConfirmedState.A, B = ConfirmedState.B }; } @@ -137,13 +137,13 @@ public Task RemoveReservationLocal(int val) } public async Task GetReservationsGlobal() { - await FetchAllEventsNow(); + await RefreshNow(); return ConfirmedState.Reservations.Values.ToArray(); } public Task SynchronizeGlobalState() { - return FetchAllEventsNow(); + return RefreshNow(); } public Task GetConfirmedVersion() @@ -158,13 +158,13 @@ public Task> GetUnresolvedConnectionIssues() public async Task> Read() { - await FetchAllEventsNow(); + await RefreshNow(); return new KeyValuePair(ConfirmedVersion, ConfirmedState); } public async Task Update(IReadOnlyList updates, int expectedversion) { if (expectedversion > ConfirmedVersion) - await FetchAllEventsNow(); + await RefreshNow(); if (expectedversion != ConfirmedVersion) return false; return await RaiseConditionalEvents(updates); From 08160dc283606c340b9fd12e19f368569ae04034 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Mon, 12 Dec 2016 13:28:43 -0800 Subject: [PATCH 07/14] break activation of log view adaptor into two steps - so initial load behaves deterministically instead of starting after timed delay --- src/Orleans/LogConsistency/ILogViewAdaptor.cs | 5 +++- .../MultiCluster/IProtocolParticipant.cs | 8 ++++- .../Common/PrimaryBasedLogViewAdaptor.cs | 30 +++++++++++-------- src/OrleansEventSourcing/JournaledGrain.cs | 14 +++++++-- src/OrleansRuntime/Catalog/Catalog.cs | 7 ++++- .../LogConsistentGrainVariations.cs | 11 +++++++ .../BasicLogConsistentGrainTests.cs | 1 + 7 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/Orleans/LogConsistency/ILogViewAdaptor.cs b/src/Orleans/LogConsistency/ILogViewAdaptor.cs index aadad9cf40..fbe30dea13 100644 --- a/src/Orleans/LogConsistency/ILogViewAdaptor.cs +++ b/src/Orleans/LogConsistency/ILogViewAdaptor.cs @@ -22,7 +22,10 @@ public interface ILogViewAdaptor : where TLogView : new() { /// Called during activation, right before the user grain activation code is run. - Task Activate(); + Task PreActivate(); + + /// Called during activation, right after the user grain activation code is run. + Task PostActivate(); /// Called during deactivation, right after the user grain deactivation code is run. Task Deactivate(); diff --git a/src/Orleans/MultiCluster/IProtocolParticipant.cs b/src/Orleans/MultiCluster/IProtocolParticipant.cs index 548eb81fdc..dc14dbfe6d 100644 --- a/src/Orleans/MultiCluster/IProtocolParticipant.cs +++ b/src/Orleans/MultiCluster/IProtocolParticipant.cs @@ -30,7 +30,13 @@ public interface IProtocolParticipant : IGrain /// Called immediately before the user-level OnActivateAsync, on same scheduler. /// /// - Task ActivateProtocolParticipant(); + Task PreActivateProtocolParticipant(); + + /// + /// Called immediately after the user-level OnActivateAsync, on same scheduler. + /// + /// + Task PostActivateProtocolParticipant(); /// /// Called immediately after the user-level OnDeactivateAsync, on same scheduler. diff --git a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs index 620b4344d7..5bb079f681 100644 --- a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs +++ b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs @@ -184,30 +184,34 @@ protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) } /// - public virtual async Task Activate() + public virtual async Task PreActivate() { - Services.Verbose2("Activation Started"); + Services.Verbose2("PreActivation Started"); - Services.SubscribeToMultiClusterConfigurationChanges(); + // this flag indicates we have not done an initial load from storage yet + // we do not act on this yet, but wait until after user OnActivate has run. + needInitialRead = true; - // initial load happens async - KickOffInitialRead().Ignore(); + Services.SubscribeToMultiClusterConfigurationChanges(); var latestconf = Services.MultiClusterConfiguration; if (latestconf != null) await OnMultiClusterConfigurationChange(latestconf); - Services.Verbose2("Activation Complete"); + Services.Verbose2("PreActivation Complete"); } - private async Task KickOffInitialRead() + public virtual Task PostActivate() { - needInitialRead = true; - // kick off notification for initial read cycle with a bit of delay - // so that we don't do this several times if user does strong sync - await Task.Delay(TimeSpan.FromMilliseconds(10)); - Services.Verbose2("Notify ({0})", nameof(KickOffInitialRead)); - worker.Notify(); + Services.Verbose2("PostActivation Started"); + + // start worker, if it has not already happened + if (needInitialRead) + worker.Notify(); + + Services.Verbose2("PostActivation Complete"); + + return TaskDone.Done; } /// diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index 8ab99362b6..b0d0e515db 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -281,11 +281,19 @@ ILogViewAdaptorFactory ILogConsistentGrain.DefaultAdaptorFactory } /// - /// Notify log view adaptor of activation + /// Notify log view adaptor of activation (called before user-level OnActivate) /// - async Task IProtocolParticipant.ActivateProtocolParticipant() + async Task IProtocolParticipant.PreActivateProtocolParticipant() { - await LogViewAdaptor.Activate(); + await LogViewAdaptor.PreActivate(); + } + + /// + /// Notify log view adaptor of activation (called after user-level OnActivate) + /// + async Task IProtocolParticipant.PostActivateProtocolParticipant() + { + await LogViewAdaptor.PostActivate(); } /// diff --git a/src/OrleansRuntime/Catalog/Catalog.cs b/src/OrleansRuntime/Catalog/Catalog.cs index 4a158b702b..7718a825b9 100644 --- a/src/OrleansRuntime/Catalog/Catalog.cs +++ b/src/OrleansRuntime/Catalog/Catalog.cs @@ -1183,7 +1183,7 @@ private async Task CallGrainActivate(ActivationData activation, Dictionary CallGrainDeactivateAndCleanupStreams(ActivationData activation) diff --git a/test/TestGrains/LogConsistentGrainVariations.cs b/test/TestGrains/LogConsistentGrainVariations.cs index 9ab45f1644..8d3d348556 100644 --- a/test/TestGrains/LogConsistentGrainVariations.cs +++ b/test/TestGrains/LogConsistentGrainVariations.cs @@ -92,6 +92,17 @@ public class LogConsistentGrainCustomStoragePrimaryCluster : LogConsistentGrain, MyGrainState state; int version; + // simulate an async call during activation. This caused deadlock in earlier version, + // so I add it here to catch regressions. + public override async Task OnActivateAsync() + { + await Task.Run(async () => + { + await Task.Delay(10); + }); + } + + public Task ApplyUpdatesToStorageAsync(IReadOnlyList updates, int expectedversion) { if (state == null) diff --git a/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs index 3a0c5cc17d..6ad3b7f8f0 100644 --- a/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs +++ b/test/TesterInternal/GeoClusterTests/BasicLogConsistentGrainTests.cs @@ -52,6 +52,7 @@ public async Task SharedLogStorage() { await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainSharedLogStorage"); } + [Fact, TestCategory("GeoCluster")] public async Task CustomStorage() { await DoBasicLogConsistentGrainTest("UnitTests.Grains.LogConsistentGrainCustomStorage"); From 7c14e2ea8bab20b07766cb317497fd3b9a00a9bf Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Tue, 13 Dec 2016 09:34:40 -0800 Subject: [PATCH 08/14] add general check to catch errors where protocol messages are sent to remote instances even though there is just a single global instance --- src/OrleansRuntime/LogConsistency/ProtocolServices.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs index 35fba26c23..cc7fa00c6f 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -48,6 +48,7 @@ public async Task SendMessage(IProtocolMessage payload, string log?.Verbose3("SendMessage {0}->{1}: {2}", mycluster, clusterId, payload); + // send the message to ourself if we are the destination cluster if (mycluster == clusterId) { var g = (IProtocolParticipant)grain; @@ -55,6 +56,12 @@ public async Task SendMessage(IProtocolMessage payload, string return await g.OnProtocolMessageReceived(payload); } + // cannot send to remote instance if there is only one instance + if (RegistrationStrategy.Equals(GlobalSingleInstanceRegistration.Singleton)) + { + throw new ProtocolTransportException("cannot send protocol message to remote instance because there is only one global instance"); + } + if (PseudoMultiClusterConfiguration != null) throw new ProtocolTransportException("no such cluster"); From 5e3f603dc2a70f2c927518ee671732f58214cf95 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 15 Dec 2016 08:48:41 -0800 Subject: [PATCH 09/14] some renaming (LogConsistencyProtocol instead of Protocol), and a few edits on comments --- .../LogConsistency/ConnectionIssues.cs | 2 +- ....cs => ILogConsistencyProtocolServices.cs} | 4 ++-- .../LogConsistency/ILogConsistencyProvider.cs | 2 +- .../LogConsistency/ILogConsistentGrain.cs | 2 +- src/Orleans/LogConsistency/ILogViewAdaptor.cs | 14 ++++++------- .../MultiCluster/IProtocolParticipant.cs | 8 ++++---- src/Orleans/Orleans.csproj | 4 ++-- ...y.cs => ILogConsistencyProtocolGateway.cs} | 4 ++-- .../Common/NotificationMessage.cs | 2 +- .../Common/NotificationTracker.cs | 4 ++-- .../Common/PrimaryBasedLogViewAdaptor.cs | 14 ++++++------- .../Common/RecordedConnectionIssue.cs | 4 ++-- .../CustomStorage/LogConsistencyProvider.cs | 2 +- .../CustomStorage/LogViewAdaptor.cs | 10 +++++----- src/OrleansEventSourcing/JournaledGrain.cs | 20 +++++++++---------- .../LogStorage/DefaultAdaptorFactory.cs | 2 +- .../LogStorage/LogConsistencyProvider.cs | 2 +- .../LogStorage/LogViewAdaptor.cs | 2 +- .../StateStorage/DefaultAdaptorFactory.cs | 2 +- .../StateStorage/LogConsistencyProvider.cs | 2 +- .../StateStorage/LogViewAdaptor.cs | 2 +- src/OrleansRuntime/Catalog/Catalog.cs | 12 +++++------ .../LogConsistency/ProtocolGateway.cs | 6 +++--- .../LogConsistency/ProtocolServices.cs | 8 ++++---- .../IMultiClusterOracle.cs | 2 +- .../MultiClusterNetwork/MultiClusterOracle.cs | 2 +- .../MultiClusterOracleData.cs | 2 +- src/OrleansTestingHost/AppDomainSiloHost.cs | 2 +- .../GeoClusterTests/TestingClusterHost.cs | 2 +- 29 files changed, 72 insertions(+), 72 deletions(-) rename src/Orleans/LogConsistency/{IProtocolServices.cs => ILogConsistencyProtocolServices.cs} (96%) rename src/Orleans/SystemTargetInterfaces/{IProtocolGateway.cs => ILogConsistencyProtocolGateway.cs} (71%) diff --git a/src/Orleans/LogConsistency/ConnectionIssues.cs b/src/Orleans/LogConsistency/ConnectionIssues.cs index d093ff8cad..872212c013 100644 --- a/src/Orleans/LogConsistency/ConnectionIssues.cs +++ b/src/Orleans/LogConsistency/ConnectionIssues.cs @@ -10,7 +10,7 @@ namespace Orleans.LogConsistency /// /// Represents information about connection issues encountered inside log consistency protocols. /// It is used both inside the protocol to track retry loops, and is made visible to users - /// who want to monitor their log-view grains for communication issues. + /// who want to monitor their log-consistent grains for communication issues. /// [Serializable] public abstract class ConnectionIssue diff --git a/src/Orleans/LogConsistency/IProtocolServices.cs b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs similarity index 96% rename from src/Orleans/LogConsistency/IProtocolServices.cs rename to src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs index 20a12fc3e0..fb3a726962 100644 --- a/src/Orleans/LogConsistency/IProtocolServices.cs +++ b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs @@ -15,7 +15,7 @@ namespace Orleans.LogConsistency /// Functionality for use by log view adaptors that use custom consistency or replication protocols. /// Abstracts communication between replicas of the log-consistent grain in different clusters. /// - public interface IProtocolServices + public interface ILogConsistencyProtocolServices { /// /// Send a message to a remote cluster. @@ -23,7 +23,7 @@ public interface IProtocolServices /// the message /// the destination cluster id /// - Task SendMessage(IProtocolMessage payload, string clusterId); + Task SendMessage(ILogConsistencyProtocolMessage payload, string clusterId); /// diff --git a/src/Orleans/LogConsistency/ILogConsistencyProvider.cs b/src/Orleans/LogConsistency/ILogConsistencyProvider.cs index 9b847d32ea..edd8cf5b88 100644 --- a/src/Orleans/LogConsistency/ILogConsistencyProvider.cs +++ b/src/Orleans/LogConsistency/ILogConsistencyProvider.cs @@ -38,7 +38,7 @@ public interface ILogViewAdaptorFactory TLogView initialstate, string graintypename, IStorageProvider storageProvider, - IProtocolServices services) + ILogConsistencyProtocolServices services) where TLogView : class, new() where TLogEntry : class; diff --git a/src/Orleans/LogConsistency/ILogConsistentGrain.cs b/src/Orleans/LogConsistency/ILogConsistentGrain.cs index 01431a03fe..09d3e50895 100644 --- a/src/Orleans/LogConsistency/ILogConsistentGrain.cs +++ b/src/Orleans/LogConsistency/ILogConsistentGrain.cs @@ -24,7 +24,7 @@ public interface ILogConsistentGrain /// The type name of the grain /// The storage provider, if needed /// Protocol services - void InstallAdaptor(ILogViewAdaptorFactory factory, object state, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services); + void InstallAdaptor(ILogViewAdaptorFactory factory, object state, string grainTypeName, IStorageProvider storageProvider, ILogConsistencyProtocolServices services); /// /// Gets the default adaptor factory to use, or null if there is no default diff --git a/src/Orleans/LogConsistency/ILogViewAdaptor.cs b/src/Orleans/LogConsistency/ILogViewAdaptor.cs index fbe30dea13..e7f12ebf9b 100644 --- a/src/Orleans/LogConsistency/ILogViewAdaptor.cs +++ b/src/Orleans/LogConsistency/ILogViewAdaptor.cs @@ -21,17 +21,17 @@ public interface ILogViewAdaptor : ILogConsistencyDiagnostics where TLogView : new() { - /// Called during activation, right before the user grain activation code is run. - Task PreActivate(); + /// Called during activation, right before the user-defined . + Task PreOnActivate(); - /// Called during activation, right after the user grain activation code is run. - Task PostActivate(); + /// Called during activation, right after the user-defined .. + Task PostOnActivate(); - /// Called during deactivation, right after the user grain deactivation code is run. - Task Deactivate(); + /// Called during deactivation, right after the user-defined . + Task PostOnDeactivate(); /// Called when a grain receives a message from a remote instance. - Task OnProtocolMessageReceived(IProtocolMessage payload); + Task OnProtocolMessageReceived(ILogConsistencyProtocolMessage payload); /// Called after the silo receives a new multi-cluster configuration. Task OnMultiClusterConfigurationChange(MultiClusterConfiguration next); diff --git a/src/Orleans/MultiCluster/IProtocolParticipant.cs b/src/Orleans/MultiCluster/IProtocolParticipant.cs index dc14dbfe6d..412101747d 100644 --- a/src/Orleans/MultiCluster/IProtocolParticipant.cs +++ b/src/Orleans/MultiCluster/IProtocolParticipant.cs @@ -4,9 +4,9 @@ namespace Orleans.MultiCluster { /// - /// Grain interface for grains that participate in multi-cluster-protocols. + /// Grain interface for grains that participate in multi-cluster log-consistency protocols. /// - public interface IProtocolParticipant : IGrain + public interface ILogConsistencyProtocolParticipant : IGrain { /// /// Called when a message is received from another cluster. @@ -15,7 +15,7 @@ public interface IProtocolParticipant : IGrain /// the protocol message to be delivered /// [AlwaysInterleave] - Task OnProtocolMessageReceived(IProtocolMessage payload); + Task OnProtocolMessageReceived(ILogConsistencyProtocolMessage payload); /// /// Called when a configuration change notification is received. @@ -49,7 +49,7 @@ public interface IProtocolParticipant : IGrain /// interface to mark classes that represent protocol messages. /// All such classes must be serializable. /// - public interface IProtocolMessage + public interface ILogConsistencyProtocolMessage { } } diff --git a/src/Orleans/Orleans.csproj b/src/Orleans/Orleans.csproj index 73b385e859..2f0e1937ab 100644 --- a/src/Orleans/Orleans.csproj +++ b/src/Orleans/Orleans.csproj @@ -135,7 +135,7 @@ - + @@ -166,7 +166,7 @@ - + diff --git a/src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs b/src/Orleans/SystemTargetInterfaces/ILogConsistencyProtocolGateway.cs similarity index 71% rename from src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs rename to src/Orleans/SystemTargetInterfaces/ILogConsistencyProtocolGateway.cs index 62b85c9c9b..3e4e2677cb 100644 --- a/src/Orleans/SystemTargetInterfaces/IProtocolGateway.cs +++ b/src/Orleans/SystemTargetInterfaces/ILogConsistencyProtocolGateway.cs @@ -14,9 +14,9 @@ namespace Orleans.SystemTargetInterfaces /// The protocol gateway is a relay that forwards incoming protocol messages from other clusters /// to the appropriate grain in this cluster. /// - internal interface IProtocolGateway : ISystemTarget + internal interface ILogConsistencyProtocolGateway : ISystemTarget { - Task RelayMessage(GrainId id, IProtocolMessage payload); + Task RelayMessage(GrainId id, ILogConsistencyProtocolMessage payload); } } diff --git a/src/OrleansEventSourcing/Common/NotificationMessage.cs b/src/OrleansEventSourcing/Common/NotificationMessage.cs index ce218b0d5c..010c14d176 100644 --- a/src/OrleansEventSourcing/Common/NotificationMessage.cs +++ b/src/OrleansEventSourcing/Common/NotificationMessage.cs @@ -11,7 +11,7 @@ namespace Orleans.EventSourcing.Common /// Base class for notification messages that are sent by log view adaptors to other /// clusters, after updating the log. All subclasses must be serializable. /// - public interface INotificationMessage : IProtocolMessage + public interface INotificationMessage : ILogConsistencyProtocolMessage { ///The version number. int Version { get; } diff --git a/src/OrleansEventSourcing/Common/NotificationTracker.cs b/src/OrleansEventSourcing/Common/NotificationTracker.cs index ca2d33ba2f..94efaf970f 100644 --- a/src/OrleansEventSourcing/Common/NotificationTracker.cs +++ b/src/OrleansEventSourcing/Common/NotificationTracker.cs @@ -13,13 +13,13 @@ namespace Orleans.EventSourcing.Common /// internal class NotificationTracker { - internal IProtocolServices services; + internal ILogConsistencyProtocolServices services; internal IConnectionIssueListener listener; internal int maxNotificationBatchSize; private Dictionary sendWorkers; - public NotificationTracker(IProtocolServices services, IEnumerable remoteInstances, int maxNotificationBatchSize, IConnectionIssueListener listener) + public NotificationTracker(ILogConsistencyProtocolServices services, IEnumerable remoteInstances, int maxNotificationBatchSize, IConnectionIssueListener listener) { this.services = services; this.listener = listener; diff --git a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs index 5bb079f681..214164348a 100644 --- a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs +++ b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs @@ -80,7 +80,7 @@ public abstract class PrimaryBasedLogViewAdaptor /// Handle protocol messages. /// - protected virtual Task OnMessageReceived(IProtocolMessage payload) + protected virtual Task OnMessageReceived(ILogConsistencyProtocolMessage payload) { // subclasses that define custom protocol messages must override this throw new NotImplementedException(); @@ -156,7 +156,7 @@ protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) /// /// The runtime services required for implementing notifications between grain instances in different cluster. /// - protected IProtocolServices Services { get; private set; } + protected ILogConsistencyProtocolServices Services { get; private set; } /// /// The current multi-cluster configuration for this grain instance. @@ -174,7 +174,7 @@ protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) /// Construct an instance, for the given parameters. /// protected PrimaryBasedLogViewAdaptor(ILogViewAdaptorHost host, - TLogView initialstate, IProtocolServices services) + TLogView initialstate, ILogConsistencyProtocolServices services) { Debug.Assert(host != null && services != null && initialstate != null); this.Host = host; @@ -184,7 +184,7 @@ protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) } /// - public virtual async Task PreActivate() + public virtual async Task PreOnActivate() { Services.Verbose2("PreActivation Started"); @@ -201,7 +201,7 @@ public virtual async Task PreActivate() Services.Verbose2("PreActivation Complete"); } - public virtual Task PostActivate() + public virtual Task PostOnActivate() { Services.Verbose2("PostActivation Started"); @@ -215,7 +215,7 @@ public virtual Task PostActivate() } /// - public virtual async Task Deactivate() + public virtual async Task PostOnDeactivate() { Services.Verbose2("Deactivation Started"); @@ -478,7 +478,7 @@ public int ConfirmedVersion /// /// /// - public async Task OnProtocolMessageReceived(IProtocolMessage payLoad) + public async Task OnProtocolMessageReceived(ILogConsistencyProtocolMessage payLoad) { var notificationMessage = payLoad as INotificationMessage; diff --git a/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs b/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs index d65d949798..7d20db324a 100644 --- a/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs +++ b/src/OrleansEventSourcing/Common/RecordedConnectionIssue.cs @@ -25,7 +25,7 @@ public struct RecordedConnectionIssue /// the connection issue to be recorded /// the listener for connection issues /// for reporting exceptions in listener - public void Record(ConnectionIssue newIssue, IConnectionIssueListener listener, IProtocolServices services) + public void Record(ConnectionIssue newIssue, IConnectionIssueListener listener, ILogConsistencyProtocolServices services) { newIssue.TimeStamp = DateTime.UtcNow; if (Issue != null) @@ -55,7 +55,7 @@ public void Record(ConnectionIssue newIssue, IConnectionIssueListener listener, /// /// the listener for connection issues /// for reporting exceptions in listener - public void Resolve(IConnectionIssueListener listener, IProtocolServices services) + public void Resolve(IConnectionIssueListener listener, ILogConsistencyProtocolServices services) { if (Issue != null) { diff --git a/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs index c4d6fad507..f4ed99a4a6 100644 --- a/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs +++ b/src/OrleansEventSourcing/CustomStorage/LogConsistencyProvider.cs @@ -75,7 +75,7 @@ public Task Close() } /// - public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, TView initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, TView initialstate, string graintypename, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) where TView : class, new() where TEntry : class { diff --git a/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs index 06f49fd42d..87d79a9146 100644 --- a/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs @@ -27,7 +27,7 @@ internal class CustomStorageAdaptor : PrimaryBasedLogViewAd /// Initialize a new instance of CustomStorageAdaptor class /// public CustomStorageAdaptor(ILogViewAdaptorHost host, TLogView initialState, - IProtocolServices services, string primaryCluster) + ILogConsistencyProtocolServices services, string primaryCluster) : base(host, initialState, services) { if (!(host is ICustomStorageInterface)) @@ -85,12 +85,12 @@ protected override SubmissionEntry MakeSubmissionEntry(TLogEntry entr } [Serializable] - private class ReadRequest : IProtocolMessage + private class ReadRequest : ILogConsistencyProtocolMessage { public int KnownVersion { get; set; } } [Serializable] - private class ReadResponse : IProtocolMessage + private class ReadResponse : ILogConsistencyProtocolMessage { public int Version { get; set; } @@ -98,7 +98,7 @@ private class ReadResponse : IProtocolMessage } /// - protected override Task OnMessageReceived(IProtocolMessage payload) + protected override Task OnMessageReceived(ILogConsistencyProtocolMessage payload) { var request = (ReadRequest) payload; @@ -111,7 +111,7 @@ protected override Task OnMessageReceived(IProtocolMessage pay if (version > request.KnownVersion) response.Value = cached; - return Task.FromResult(response); + return Task.FromResult(response); } /// diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index b0d0e515db..5f7f734cda 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -28,7 +28,7 @@ public abstract class JournaledGrain : JournaledGrain : LogConsistentGrainBase, ILogConsistentGrain, - IProtocolParticipant, + ILogConsistencyProtocolParticipant, ILogViewAdaptorHost where TGrainState : class, new() where TEventBase: class @@ -253,7 +253,7 @@ protected virtual void TransitionState(TGrainState state, TEventBase @event) /// Called right after grain is constructed, to install the adaptor. /// The log-consistency provider contains a factory method that constructs the adaptor with chosen types for this grain /// - void ILogConsistentGrain.InstallAdaptor(ILogViewAdaptorFactory factory, object initialState, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + void ILogConsistentGrain.InstallAdaptor(ILogViewAdaptorFactory factory, object initialState, string graintypename, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) { // call the log consistency provider to construct the adaptor, passing the type argument LogViewAdaptor = factory.MakeLogViewAdaptor(this, (TGrainState)initialState, graintypename, storageProvider, services); @@ -283,32 +283,32 @@ ILogViewAdaptorFactory ILogConsistentGrain.DefaultAdaptorFactory /// /// Notify log view adaptor of activation (called before user-level OnActivate) /// - async Task IProtocolParticipant.PreActivateProtocolParticipant() + async Task ILogConsistencyProtocolParticipant.PreActivateProtocolParticipant() { - await LogViewAdaptor.PreActivate(); + await LogViewAdaptor.PreOnActivate(); } /// /// Notify log view adaptor of activation (called after user-level OnActivate) /// - async Task IProtocolParticipant.PostActivateProtocolParticipant() + async Task ILogConsistencyProtocolParticipant.PostActivateProtocolParticipant() { - await LogViewAdaptor.PostActivate(); + await LogViewAdaptor.PostOnActivate(); } /// /// Notify log view adaptor of deactivation /// - Task IProtocolParticipant.DeactivateProtocolParticipant() + Task ILogConsistencyProtocolParticipant.DeactivateProtocolParticipant() { - return LogViewAdaptor.Deactivate(); + return LogViewAdaptor.PostOnDeactivate(); } /// /// Receive a protocol message from other clusters, passed on to log view adaptor. /// [AlwaysInterleave] - Task IProtocolParticipant.OnProtocolMessageReceived(IProtocolMessage payload) + Task ILogConsistencyProtocolParticipant.OnProtocolMessageReceived(ILogConsistencyProtocolMessage payload) { return LogViewAdaptor.OnProtocolMessageReceived(payload); } @@ -317,7 +317,7 @@ Task IProtocolParticipant.OnProtocolMessageReceived(IProtocolM /// Receive a configuration change, pass on to log view adaptor. /// [AlwaysInterleave] - Task IProtocolParticipant.OnMultiClusterConfigurationChange(MultiCluster.MultiClusterConfiguration next) + Task ILogConsistencyProtocolParticipant.OnMultiClusterConfigurationChange(MultiCluster.MultiClusterConfiguration next) { return LogViewAdaptor.OnMultiClusterConfigurationChange(next); } diff --git a/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs b/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs index 9e82771335..0ae16149c2 100644 --- a/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs +++ b/src/OrleansEventSourcing/LogStorage/DefaultAdaptorFactory.cs @@ -20,7 +20,7 @@ public bool UsesStorageProvider } } - public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) where T : class, new() where E : class { return new LogViewAdaptor(hostgrain, initialstate, storageProvider, graintypename, services); diff --git a/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs index d03c04d1ee..d41e8889fc 100644 --- a/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs +++ b/src/OrleansEventSourcing/LogStorage/LogConsistencyProvider.cs @@ -81,7 +81,7 @@ public Task Close() /// The initial state for this view /// The type name of the grain /// Runtime services for multi-cluster coherence protocols - public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services) + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) where TView : class, new() where TEntry : class { diff --git a/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs index 45a5a25ae3..8a02dba715 100644 --- a/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs @@ -26,7 +26,7 @@ internal class LogViewAdaptor : PrimaryBasedLogViewAdaptor< /// /// Initialize a StorageProviderLogViewAdaptor class /// - public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, IProtocolServices services) + public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, ILogConsistencyProtocolServices services) : base(host, initialState, services) { this.globalStorageProvider = globalStorageProvider; diff --git a/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs b/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs index 3e6b242548..9291c08a1a 100644 --- a/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs +++ b/src/OrleansEventSourcing/StateStorage/DefaultAdaptorFactory.cs @@ -20,7 +20,7 @@ public bool UsesStorageProvider } } - public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, IProtocolServices services) + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostgrain, T initialstate, string graintypename, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) where T : class, new() where E : class { return new LogViewAdaptor(hostgrain, initialstate, storageProvider, graintypename, services); diff --git a/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs b/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs index f56221b222..2ab2275785 100644 --- a/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs +++ b/src/OrleansEventSourcing/StateStorage/LogConsistencyProvider.cs @@ -81,7 +81,7 @@ public Task Close() /// The initial state for this view /// The type name of the grain /// Runtime services for multi-cluster coherence protocols - public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, IProtocolServices services) + public ILogViewAdaptor MakeLogViewAdaptor(ILogViewAdaptorHost hostGrain, TView initialState, string grainTypeName, IStorageProvider storageProvider, ILogConsistencyProtocolServices services) where TView : class, new() where TEntry : class { diff --git a/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs index 0dcf8ef344..7feec78ffc 100644 --- a/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs @@ -26,7 +26,7 @@ internal class LogViewAdaptor : PrimaryBasedLogViewAdaptor< /// /// Initialize a StorageProviderLogViewAdaptor class /// - public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, IProtocolServices services) + public LogViewAdaptor(ILogViewAdaptorHost host, TLogView initialState, IStorageProvider globalStorageProvider, string grainTypeName, ILogConsistencyProtocolServices services) : base(host, initialState, services) { this.globalStorageProvider = globalStorageProvider; diff --git a/src/OrleansRuntime/Catalog/Catalog.cs b/src/OrleansRuntime/Catalog/Catalog.cs index 7718a825b9..4fb8d0db8b 100644 --- a/src/OrleansRuntime/Catalog/Catalog.cs +++ b/src/OrleansRuntime/Catalog/Catalog.cs @@ -1181,9 +1181,9 @@ private async Task CallGrainActivate(ActivationData activation, Dictionary CallGrainDeactivateAndCleanupStreams(Activati } } - if (activation.GrainInstance is IProtocolParticipant) + if (activation.GrainInstance is ILogConsistencyProtocolParticipant) { - await ((IProtocolParticipant)activation.GrainInstance).DeactivateProtocolParticipant(); + await ((ILogConsistencyProtocolParticipant)activation.GrainInstance).DeactivateProtocolParticipant(); } } catch(Exception exc) diff --git a/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs b/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs index 1f49a338d0..6eb733d82f 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolGateway.cs @@ -13,16 +13,16 @@ namespace Orleans.Runtime.LogConsistency { [Reentrant] - internal class ProtocolGateway : SystemTarget, IProtocolGateway + internal class ProtocolGateway : SystemTarget, ILogConsistencyProtocolGateway { public ProtocolGateway(SiloAddress silo) : base(Constants.ProtocolGatewayId, silo) { } - public async Task RelayMessage(GrainId id, IProtocolMessage payload) + public async Task RelayMessage(GrainId id, ILogConsistencyProtocolMessage payload) { - var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(GrainReference.FromGrainId(id)); + var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(GrainReference.FromGrainId(id)); return await g.OnProtocolMessageReceived(payload); } diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs index cc7fa00c6f..95a1d50b55 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -18,7 +18,7 @@ namespace Orleans.Runtime.LogConsistency /// This class allows access to these services to providers that cannot see runtime-internals. /// It also stores grain-specific information like the grain reference, and caches /// - internal class ProtocolServices : IProtocolServices + internal class ProtocolServices : ILogConsistencyProtocolServices { public GrainReference GrainReference { get { return grain.GrainReference; } } @@ -40,7 +40,7 @@ internal ProtocolServices(Grain gr, Logger log, IMultiClusterRegistrationStrateg } - public async Task SendMessage(IProtocolMessage payload, string clusterId) + public async Task SendMessage(ILogConsistencyProtocolMessage payload, string clusterId) { var silo = Silo.CurrentSilo; var mycluster = silo.ClusterId; @@ -51,7 +51,7 @@ public async Task SendMessage(IProtocolMessage payload, string // send the message to ourself if we are the destination cluster if (mycluster == clusterId) { - var g = (IProtocolParticipant)grain; + var g = (ILogConsistencyProtocolParticipant)grain; // we are on the same scheduler, so we can call the method directly return await g.OnProtocolMessageReceived(payload); } @@ -76,7 +76,7 @@ public async Task SendMessage(IProtocolMessage payload, string if (clusterGateway == null) throw new ProtocolTransportException("no active gateways found for cluster"); - var repAgent = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget(Constants.ProtocolGatewayId, clusterGateway); + var repAgent = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget(Constants.ProtocolGatewayId, clusterGateway); // test hook var filter = (oracle as MultiClusterNetwork.MultiClusterOracle).ProtocolMessageFilterForTesting; diff --git a/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs b/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs index 471f73299d..24d237917e 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/IMultiClusterOracle.cs @@ -72,6 +72,6 @@ internal interface IMultiClusterOracle /// /// A test hook for dropping protocol messages between replicated grain instances /// - Func ProtocolMessageFilterForTesting { get; set; } + Func ProtocolMessageFilterForTesting { get; set; } } } diff --git a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs index 60213372d0..732742932a 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracle.cs @@ -126,7 +126,7 @@ public bool UnSubscribeFromMultiClusterConfigurationEvents(GrainReference observ /// - public Func ProtocolMessageFilterForTesting { get; set; } + public Func ProtocolMessageFilterForTesting { get; set; } public async Task Start(ISiloStatusOracle oracle) diff --git a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs index 42f2f4327d..73dda46832 100644 --- a/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs +++ b/src/OrleansRuntime/MultiClusterNetwork/MultiClusterOracleData.cs @@ -114,7 +114,7 @@ public MultiClusterData ApplyDataAndNotify(MultiClusterData data) logger.Verbose2("-NotificationWork: notify IProtocolParticipant {0} of configuration {1}", listener, delta.Configuration); // enqueue conf change event as grain call - var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(listener); + var g = InsideRuntimeClient.Current.InternalGrainFactory.Cast(listener); g.OnMultiClusterConfigurationChange(delta.Configuration).Ignore(); } catch (Exception exc) diff --git a/src/OrleansTestingHost/AppDomainSiloHost.cs b/src/OrleansTestingHost/AppDomainSiloHost.cs index 3e37250746..4369e0da05 100644 --- a/src/OrleansTestingHost/AppDomainSiloHost.cs +++ b/src/OrleansTestingHost/AppDomainSiloHost.cs @@ -175,7 +175,7 @@ internal void UnblockSiloCommunication() simulatedMessageLoss.Clear(); } - internal Func ProtocolMessageFilterForTesting + internal Func ProtocolMessageFilterForTesting { get { diff --git a/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs b/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs index 50814cd7d9..1357caf912 100644 --- a/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs +++ b/test/TesterInternal/GeoClusterTests/TestingClusterHost.cs @@ -415,7 +415,7 @@ public void UnblockAllClusterCommunication(string from) } } - public void SetProtocolMessageFilterForTesting(string origincluster, Func filter) + public void SetProtocolMessageFilterForTesting(string origincluster, Func filter) { var silos = Clusters[origincluster].Silos; foreach (var silo in silos) From 9008b1b1e11432329f206189243ee7235f1f2e8d Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Mon, 19 Dec 2016 16:07:03 -0800 Subject: [PATCH 10/14] fix some access modifiers, remove an unused field, add some comments, remove async suffix --- .../LogConsistency/ILogConsistencyProtocolServices.cs | 4 ++++ src/Orleans/LogConsistency/ILogViewAdaptor.cs | 9 +++++---- src/OrleansEventSourcing/Common/NotificationTracker.cs | 8 ++------ .../Common/PrimaryBasedLogViewAdaptor.cs | 4 ++-- src/OrleansEventSourcing/JournaledGrain.cs | 10 ++++++---- test/TestGrains/LogConsistentGrainVariations.cs | 6 +++--- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs index fb3a726962..59cec5c818 100644 --- a/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs +++ b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs @@ -44,6 +44,10 @@ public interface ILogConsistencyProtocolServices /// bool MultiClusterEnabled { get; } + // If there is no multi-cluster network, + // we use a default multi-cluster configuration containing just one cluster (this one), named "I" + // this is more convenient and consistent than returning null or invalid, + // since it means the log-consistency providers do the right thing (run like a single-cluster configuration) /// /// The id of this cluster. Returns "I" if no multi-cluster network is present. diff --git a/src/Orleans/LogConsistency/ILogViewAdaptor.cs b/src/Orleans/LogConsistency/ILogViewAdaptor.cs index e7f12ebf9b..440a213e9e 100644 --- a/src/Orleans/LogConsistency/ILogViewAdaptor.cs +++ b/src/Orleans/LogConsistency/ILogViewAdaptor.cs @@ -104,13 +104,14 @@ public interface ILogViewUpdate /// Confirm all submitted entries. ///Waits until all previously submitted entries appear in the confirmed prefix of the log. /// - Task ConfirmSubmittedEntriesAsync(); + /// A task that completes after all entries are confirmed. + Task ConfirmSubmittedEntries(); /// - /// Confirm all submitted entries and get the latest log view. + /// Get the latest log view and confirm all submitted entries. ///Waits until all previously submitted entries appear in the confirmed prefix of the log, and forces a refresh of the confirmed prefix. /// - /// - Task SynchronizeNowAsync(); + /// A task that completes after getting the latest version and confirming all entries. + Task Synchronize(); } } diff --git a/src/OrleansEventSourcing/Common/NotificationTracker.cs b/src/OrleansEventSourcing/Common/NotificationTracker.cs index 94efaf970f..6b10d0632a 100644 --- a/src/OrleansEventSourcing/Common/NotificationTracker.cs +++ b/src/OrleansEventSourcing/Common/NotificationTracker.cs @@ -100,20 +100,16 @@ public class NotificationWorker : BatchWorker /// /// Queue messages /// - public INotificationMessage QueuedMessage = null; + private INotificationMessage QueuedMessage = null; /// /// Queue state /// - public NotificationQueueState QueueState = NotificationQueueState.Empty; + private NotificationQueueState QueueState = NotificationQueueState.Empty; /// /// Last exception /// public RecordedConnectionIssue LastConnectionIssue; /// - /// Number of consecutive failures - /// - public int NumConsecutiveFailures; - /// /// Is current task done or not /// public bool Done; diff --git a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs index 214164348a..99126e89a1 100644 --- a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs +++ b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs @@ -769,7 +769,7 @@ public IEnumerable UnresolvedConnectionIssues /// - public async Task SynchronizeNowAsync() + public async Task Synchronize() { if (stats != null) stats.EventCounters["SynchronizeNowCalled"]++; @@ -792,7 +792,7 @@ public IEnumerable UnconfirmedSuffix } /// - public async Task ConfirmSubmittedEntriesAsync() + public async Task ConfirmSubmittedEntries() { if (stats != null) stats.EventCounters["ConfirmSubmittedEntriesCalled"]++; diff --git a/src/OrleansEventSourcing/JournaledGrain.cs b/src/OrleansEventSourcing/JournaledGrain.cs index 5f7f734cda..52fc724885 100644 --- a/src/OrleansEventSourcing/JournaledGrain.cs +++ b/src/OrleansEventSourcing/JournaledGrain.cs @@ -146,22 +146,24 @@ protected virtual void OnConfirmedStateChanged() /// /// Waits until all previously raised events have been confirmed. + /// await this after raising one or more events, to ensure events are persisted before proceeding, or to guarantee strong consistency (linearizability) even if there are multiple instances of this grain /// - /// + /// a task that completes once the events have been confirmed. protected Task ConfirmEvents() { - return LogViewAdaptor.ConfirmSubmittedEntriesAsync(); + return LogViewAdaptor.ConfirmSubmittedEntries(); } /// /// Retrieves the latest state now, and confirms all previously raised events. /// Effectively, this enforces synchronization with the global state. + /// Await this before reading the state to ensure strong consistency (linearizability) even if there are multiple instances of this grain /// - /// + /// a task that completes once the log has been refreshed and the events have been confirmed. protected Task RefreshNow() { - return LogViewAdaptor.SynchronizeNowAsync(); + return LogViewAdaptor.Synchronize(); } diff --git a/test/TestGrains/LogConsistentGrainVariations.cs b/test/TestGrains/LogConsistentGrainVariations.cs index 8d3d348556..1ac5021f1a 100644 --- a/test/TestGrains/LogConsistentGrainVariations.cs +++ b/test/TestGrains/LogConsistentGrainVariations.cs @@ -57,7 +57,7 @@ public class LogConsistentGrainCustomStorage : LogConsistentGrain, { // we use another impl of this grain as the primary. - ILogConsistentGrain storagegrain; + private ILogConsistentGrain storagegrain; private ILogConsistentGrain GetStorageGrain() { @@ -89,8 +89,8 @@ public class LogConsistentGrainCustomStoragePrimaryCluster : LogConsistentGrain, { // we use fake in-memory state as the storage - MyGrainState state; - int version; + private MyGrainState state; + private int version; // simulate an async call during activation. This caused deadlock in earlier version, // so I add it here to catch regressions. From 711a9c664b5abc189c22d52a87b87b122ebc1459 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Wed, 11 Jan 2017 14:55:39 -0800 Subject: [PATCH 11/14] Change Silo.ClusterId to return Pseudo-ID, not null, if no multi-cluster network is present --- .../GlobalSingleInstanceRegistrar.cs | 18 +++++++++--------- .../LogConsistency/ProtocolServices.cs | 6 +++--- .../Messaging/GatewayAcceptor.cs | 4 ++-- src/OrleansRuntime/Silo/Silo.cs | 9 ++++++++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/OrleansRuntime/GrainDirectory/MultiClusterRegistration/GlobalSingleInstanceRegistrar.cs b/src/OrleansRuntime/GrainDirectory/MultiClusterRegistration/GlobalSingleInstanceRegistrar.cs index be57d9481d..2ea2730a33 100644 --- a/src/OrleansRuntime/GrainDirectory/MultiClusterRegistration/GlobalSingleInstanceRegistrar.cs +++ b/src/OrleansRuntime/GrainDirectory/MultiClusterRegistration/GlobalSingleInstanceRegistrar.cs @@ -57,14 +57,14 @@ public async Task RegisterAsync(ActivationAddress address, bool s if (!singleActivation) throw new OrleansException("global single instance protocol is incompatible with using multiple activations"); - var myClusterId = Silo.CurrentSilo.ClusterId; - - if (myClusterId == null) + if (!Silo.CurrentSilo.HasMultiClusterNetwork) { // no multicluster network. Go to owned state directly. return directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.Owned); } + var myClusterId = Silo.CurrentSilo.ClusterId; + // examine the multicluster configuration var config = Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration(); @@ -192,13 +192,13 @@ public Task UnregisterAsync(List addresses, UnregistrationCau if (formerActivationsInThisCluster == null) return TaskDone.Done; + if (!Silo.CurrentSilo.HasMultiClusterNetwork) + return TaskDone.Done; // single cluster - no broadcast required + // we must also remove cached references to former activations in this cluster // from remote clusters; thus, we broadcast the unregistration var myClusterId = Silo.CurrentSilo.ClusterId; - if (myClusterId == null) - return TaskDone.Done; // single cluster - no broadcast required - // target clusters in current configuration, other than this one var remoteClusters = Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration().Clusters .Where(id => id != myClusterId).ToList(); @@ -241,12 +241,12 @@ public Task DeleteAsync(GrainId gid) { directoryPartition.RemoveGrain(gid); + if (!Silo.CurrentSilo.HasMultiClusterNetwork) + return TaskDone.Done; // single cluster - no broadcast required + // broadcast deletion to all other clusters var myClusterId = Silo.CurrentSilo.ClusterId; - if (myClusterId == null) - return TaskDone.Done; // single cluster - no broadcast required - // target ALL clusters, not just clusters in current configuration var remoteClusters = Silo.CurrentSilo.LocalMultiClusterOracle.GetActiveClusters() .Where(id => id != myClusterId).ToList(); diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs index 95a1d50b55..0251de1625 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -36,7 +36,7 @@ internal ProtocolServices(Grain gr, Logger log, IMultiClusterRegistrationStrateg this.RegistrationStrategy = strategy; if (!Silo.CurrentSilo.GlobalConfig.HasMultiClusterNetwork) - PseudoMultiClusterConfiguration = new MultiClusterConfiguration(DateTime.UtcNow, new string[] { PseudoReplicaId }.ToList()); + PseudoMultiClusterConfiguration = new MultiClusterConfiguration(DateTime.UtcNow, new string[] { PseudoClusterId }.ToList()); } @@ -96,7 +96,7 @@ public async Task SendMessage(ILogConsistencyPro // pseudo-configuration to use if there is no actual multicluster network private static MultiClusterConfiguration PseudoMultiClusterConfiguration; - private static string PseudoReplicaId = "I"; + internal static string PseudoClusterId = "I"; public bool MultiClusterEnabled @@ -112,7 +112,7 @@ public string MyClusterId get { if (PseudoMultiClusterConfiguration != null) - return PseudoReplicaId; + return PseudoClusterId; else return Silo.CurrentSilo.ClusterId; } diff --git a/src/OrleansRuntime/Messaging/GatewayAcceptor.cs b/src/OrleansRuntime/Messaging/GatewayAcceptor.cs index ca95a0fa7b..a2da54f8f6 100644 --- a/src/OrleansRuntime/Messaging/GatewayAcceptor.cs +++ b/src/OrleansRuntime/Messaging/GatewayAcceptor.cs @@ -40,7 +40,7 @@ protected override bool RecordOpenedSocket(Socket sock) else { //convert handshake cliendId to a GeoClient ID - if (!string.IsNullOrEmpty(Silo.CurrentSilo.ClusterId)) + if (Silo.CurrentSilo.HasMultiClusterNetwork) { client = GrainId.NewClientId(client.PrimaryKey, Silo.CurrentSilo.ClusterId); } @@ -76,7 +76,7 @@ protected override void HandleMessage(Message msg, Socket receivedOnSocket) gatewayTrafficCounter.Increment(); // return address translation for geo clients (replace sending address cli/* with gcl/*) - if (! string.IsNullOrEmpty(Silo.CurrentSilo.ClusterId) && msg.SendingAddress.Grain.Category != UniqueKey.Category.GeoClient) + if (Silo.CurrentSilo.HasMultiClusterNetwork && msg.SendingAddress.Grain.Category != UniqueKey.Category.GeoClient) { msg.SendingGrain = GrainId.NewClientId(msg.SendingAddress.Grain.PrimaryKey, Silo.CurrentSilo.ClusterId); } diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 32f0c0abbc..de52b11903 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -116,10 +116,17 @@ internal IReadOnlyCollection AllSiloProviders internal IServiceProvider Services { get; } + + /// Gets whether this cluster is configured to be part of a multicluster. + public bool HasMultiClusterNetwork + { + get { return GlobalConfig.HasMultiClusterNetwork; } + } + /// Get the id of the cluster this silo is part of. public string ClusterId { - get { return GlobalConfig.HasMultiClusterNetwork ? GlobalConfig.ClusterId : null; } + get { return GlobalConfig.HasMultiClusterNetwork ? GlobalConfig.ClusterId : ProtocolServices.PseudoClusterId; } } /// SiloAddress for this silo. From 8d4cbdce9992739c0eb6e299e5545f4a94e02d24 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Wed, 11 Jan 2017 16:12:19 -0800 Subject: [PATCH 12/14] fix confllict: pass provider runtime in constructor of LogConsistencyProviderManager --- .../LogConsistencyProviderManager.cs | 14 +++++++------- src/OrleansRuntime/Silo/Silo.cs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs b/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs index 2d8348e9c5..8adca063f9 100644 --- a/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs +++ b/src/OrleansRuntime/LogConsistency/LogConsistencyProviderManager.cs @@ -14,21 +14,21 @@ namespace Orleans.Runtime.LogConsistency internal class LogConsistencyProviderManager : ILogConsistencyProviderManager, ILogConsistencyProviderRuntime { private ProviderLoader providerLoader; - private IProviderRuntime providerRuntime; + private IProviderRuntime runtime; public IGrainFactory GrainFactory { get; private set; } public IServiceProvider ServiceProvider { get; private set; } - public LogConsistencyProviderManager(IGrainFactory grainFactory, IServiceProvider serviceProvider) + public LogConsistencyProviderManager(IGrainFactory grainFactory, IServiceProvider serviceProvider, IProviderRuntime runtime) { GrainFactory = grainFactory; ServiceProvider = serviceProvider; + this.runtime = runtime; } internal Task LoadLogConsistencyProviders(IDictionary configs) { providerLoader = new ProviderLoader(); - providerRuntime = SiloProviderRuntime.Instance; if (!configs.ContainsKey(ProviderCategoryConfiguration.LOG_CONSISTENCY_PROVIDER_CATEGORY_NAME)) return TaskDone.Done; @@ -59,12 +59,12 @@ public IList GetProviders() public void SetInvokeInterceptor(InvokeInterceptor interceptor) { - providerRuntime.SetInvokeInterceptor(interceptor); + runtime.SetInvokeInterceptor(interceptor); } public InvokeInterceptor GetInvokeInterceptor() { - return providerRuntime.GetInvokeInterceptor(); + return runtime.GetInvokeInterceptor(); } public Logger GetLogger(string loggerName) @@ -74,12 +74,12 @@ public Logger GetLogger(string loggerName) public Guid ServiceId { - get { return providerRuntime.ServiceId; } + get { return runtime.ServiceId; } } public string SiloIdentity { - get { return providerRuntime.SiloIdentity; } + get { return runtime.SiloIdentity; } } /// diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 1a0e39eea2..0826caa7d0 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -540,7 +540,7 @@ private void DoStart() if (logger.IsVerbose) { logger.Verbose("Storage provider manager created successfully."); } // Initialize log consistency providers once we have a basic silo runtime environment operating - logConsistencyProviderManager = new LogConsistencyProviderManager(grainFactory, Services); + logConsistencyProviderManager = new LogConsistencyProviderManager(grainFactory, Services, siloProviderRuntime); scheduler.QueueTask( () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) From eca92aab7b9983eae0aeb739ee8da9daf5b46559 Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 12 Jan 2017 10:44:46 -0800 Subject: [PATCH 13/14] remove logging severity overloads from ILogConsistencyProtocolServices --- .../ILogConsistencyProtocolServices.cs | 10 +--- .../Common/NotificationTracker.cs | 11 +++-- .../Common/PrimaryBasedLogViewAdaptor.cs | 49 ++++++++++--------- .../CustomStorage/LogViewAdaptor.cs | 22 ++++----- .../LogStorage/LogViewAdaptor.cs | 25 +++++----- .../StateStorage/LogViewAdaptor.cs | 25 +++++----- .../LogConsistency/ProtocolServices.cs | 49 +++---------------- 7 files changed, 78 insertions(+), 113 deletions(-) diff --git a/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs index 59cec5c818..e0e2432a97 100644 --- a/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs +++ b/src/Orleans/LogConsistency/ILogConsistencyProtocolServices.cs @@ -97,14 +97,8 @@ public interface ILogConsistencyProtocolServices /// The caught exception void CaughtUserCodeException(string callback, string where, Exception e); - /// Output the specified message at Info log level. - void Info(string format, params object[] args); - /// Output the specified message at Verbose log level. - void Verbose(string format, params object[] args); - /// Output the specified message at Verbose2 log level. - void Verbose2(string format, params object[] args); - /// Output the specified message at Verbose3 log level. - void Verbose3(string format, params object[] args); + /// Output the specified message at the specified log level. + void Log(Severity severity, string format, params object[] args); #endregion } diff --git a/src/OrleansEventSourcing/Common/NotificationTracker.cs b/src/OrleansEventSourcing/Common/NotificationTracker.cs index 6b10d0632a..b84edf4a7c 100644 --- a/src/OrleansEventSourcing/Common/NotificationTracker.cs +++ b/src/OrleansEventSourcing/Common/NotificationTracker.cs @@ -1,4 +1,5 @@ using Orleans.LogConsistency; +using Orleans.Runtime; using System; using System.Collections.Generic; using System.Linq; @@ -28,7 +29,7 @@ public NotificationTracker(ILogConsistencyProtocolServices services, IEnumerable foreach (var x in remoteInstances) { - services.Verbose("Now sending notifications to {0}", x); + services.Log(Severity.Verbose, "Now sending notifications to {0}", x); sendWorkers.Add(x, new NotificationWorker(this, x)); } } @@ -64,7 +65,7 @@ public void UpdateNotificationTargets(IReadOnlyList remoteInstances) var removed = sendWorkers.Keys.Except(remoteInstances); foreach (var x in removed) { - services.Verbose("No longer sending notifications to {0}", x); + services.Log(Severity.Verbose, "No longer sending notifications to {0}", x); sendWorkers[x].Done = true; sendWorkers.Remove(x); } @@ -74,7 +75,7 @@ public void UpdateNotificationTargets(IReadOnlyList remoteInstances) { if (x != services.MyClusterId) { - services.Verbose("Now sending notifications to {0}", x); + services.Log(Severity.Verbose, "Now sending notifications to {0}", x); sendWorkers.Add(x, new NotificationWorker(this, x)); } } @@ -198,13 +199,13 @@ protected override async Task Work() await tracker.services.SendMessage(msg, clusterId); // notification was successful - tracker.services.Verbose("Sent notification to cluster {0}: {1}", clusterId, msg); + tracker.services.Log(Severity.Verbose, "Sent notification to cluster {0}: {1}", clusterId, msg); LastConnectionIssue.Resolve(tracker.listener, tracker.services); } catch (Exception e) { - tracker.services.Info("Could not send notification to cluster {0}: {1}", clusterId, e); + tracker.services.Log(Severity.Info, "Could not send notification to cluster {0}: {1}", clusterId, e); LastConnectionIssue.Record( new NotificationFailed() { RemoteCluster = clusterId, Exception = e }, diff --git a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs index 99126e89a1..235da394ff 100644 --- a/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs +++ b/src/OrleansEventSourcing/Common/PrimaryBasedLogViewAdaptor.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using Orleans.Serialization; using Orleans.MultiCluster; +using Orleans.Runtime; namespace Orleans.EventSourcing.Common { @@ -123,7 +124,7 @@ protected virtual void ProcessNotifications() { if (lastVersionNotified > this.GetConfirmedVersion()) { - Services.Verbose("force refresh because of version notification v{0}", lastVersionNotified); + Services.Log(Severity.Verbose, "force refresh because of version notification v{0}", lastVersionNotified); needRefresh = true; } } @@ -186,7 +187,7 @@ protected virtual Task OnConfigurationChange(MultiClusterConfiguration next) /// public virtual async Task PreOnActivate() { - Services.Verbose2("PreActivation Started"); + Services.Log(Severity.Verbose2, "PreActivation Started"); // this flag indicates we have not done an initial load from storage yet // we do not act on this yet, but wait until after user OnActivate has run. @@ -198,18 +199,18 @@ public virtual async Task PreOnActivate() if (latestconf != null) await OnMultiClusterConfigurationChange(latestconf); - Services.Verbose2("PreActivation Complete"); + Services.Log(Severity.Verbose2, "PreActivation Complete"); } public virtual Task PostOnActivate() { - Services.Verbose2("PostActivation Started"); + Services.Log(Severity.Verbose2, "PostActivation Started"); // start worker, if it has not already happened if (needInitialRead) worker.Notify(); - Services.Verbose2("PostActivation Complete"); + Services.Log(Severity.Verbose2, "PostActivation Complete"); return TaskDone.Done; } @@ -217,7 +218,7 @@ public virtual Task PostOnActivate() /// public virtual async Task PostOnDeactivate() { - Services.Verbose2("Deactivation Started"); + Services.Log(Severity.Verbose2, "Deactivation Started"); while (!worker.IsIdle()) { @@ -226,7 +227,7 @@ public virtual async Task PostOnDeactivate() Services.UnsubscribeFromMultiClusterConfigurationChanges(); - Services.Verbose2("Deactivation Complete"); + Services.Log(Severity.Verbose2, "Deactivation Complete"); } @@ -296,7 +297,7 @@ protected async Task EnsureClusterJoinedAsync() { while (!IsMyClusterJoined()) { - Services.Verbose("Waiting for join"); + Services.Log(Severity.Verbose, "Waiting for join"); await Task.Delay(5000); } } @@ -307,7 +308,7 @@ protected async Task GetCaughtUpWithConfigurationAsync(DateTime adminTimestamp) { while (Configuration == null || Configuration.AdminTimestamp < adminTimestamp) { - Services.Verbose("Waiting for config {0}", adminTimestamp); + Services.Log(Severity.Verbose, "Waiting for config {0}", adminTimestamp); await Task.Delay(5000); } @@ -327,7 +328,7 @@ public void Submit(TLogEntry logEntry) if (stats != null) stats.EventCounters["SubmitCalled"]++; - Services.Verbose2("Submit"); + Services.Log(Severity.Verbose2, "Submit"); SubmitInternal(DateTime.UtcNow, logEntry); @@ -342,7 +343,7 @@ public void SubmitRange(IEnumerable logEntries) if (stats != null) stats.EventCounters["SubmitRangeCalled"]++; - Services.Verbose2("SubmitRange"); + Services.Log(Severity.Verbose2, "SubmitRange"); var time = DateTime.UtcNow; @@ -360,7 +361,7 @@ public Task TryAppend(TLogEntry logEntry) if (stats != null) stats.EventCounters["TryAppendCalled"]++; - Services.Verbose2("TryAppend"); + Services.Log(Severity.Verbose2, "TryAppend"); var promise = new TaskCompletionSource(); @@ -379,7 +380,7 @@ public Task TryAppendRange(IEnumerable logEntries) if (stats != null) stats.EventCounters["TryAppendRangeCalled"]++; - Services.Verbose2("TryAppendRange"); + Services.Log(Severity.Verbose2, "TryAppendRange"); var promise = new TaskCompletionSource(); var time = DateTime.UtcNow; @@ -484,7 +485,7 @@ public async Task OnProtocolMessageReceived(ILog if (notificationMessage != null) { - Services.Verbose("NotificationReceived v{0}", notificationMessage.Version); + Services.Log(Severity.Verbose, "NotificationReceived v{0}", notificationMessage.Version); OnNotificationReceived(notificationMessage); @@ -515,7 +516,7 @@ public async Task OnMultiClusterConfigurationChange(MultiCluster.MultiClusterCon if (!MultiClusterConfiguration.OlderThan(oldConfig, newConfig)) return; - Services.Verbose("Processing Configuration {0}", newConfig); + Services.Log(Severity.Verbose, "Processing Configuration {0}", newConfig); await this.OnConfigurationChange(newConfig); // updates Configuration and does any work required @@ -526,7 +527,7 @@ public async Task OnMultiClusterConfigurationChange(MultiCluster.MultiClusterCon if (!needInitialRead && added.Contains(Services.MyClusterId)) { needRefresh = true; - Services.Verbose("Refresh Because of Join"); + Services.Log(Severity.Verbose, "Refresh Because of Join"); worker.Notify(); } @@ -609,13 +610,13 @@ private void CalculateTentativeState() /// internal async Task Work() { - Services.Verbose("<1 ProcessNotifications"); + Services.Log(Severity.Verbose, "<1 ProcessNotifications"); var version = GetConfirmedVersion(); ProcessNotifications(); - Services.Verbose("<2 NotifyViewChanges"); + Services.Log(Severity.Verbose, "<2 NotifyViewChanges"); NotifyViewChanges(ref version); @@ -623,7 +624,7 @@ internal async Task Work() bool haveToRead = needInitialRead || (needRefresh && !haveToWrite); - Services.Verbose("<3 Storage htr={0} htw={1}", haveToRead, haveToWrite); + Services.Log(Severity.Verbose, "<3 Storage htr={0} htw={1}", haveToRead, haveToWrite); try { @@ -654,7 +655,7 @@ internal async Task Work() } - Services.Verbose("<4 Done"); + Services.Log(Severity.Verbose, "<4 Done"); } @@ -774,12 +775,12 @@ public async Task Synchronize() if (stats != null) stats.EventCounters["SynchronizeNowCalled"]++; - Services.Verbose("SynchronizeNowStart"); + Services.Log(Severity.Verbose, "SynchronizeNowStart"); needRefresh = true; await worker.NotifyAndWaitForWorkToBeServiced(); - Services.Verbose("SynchronizeNowComplete"); + Services.Log(Severity.Verbose, "SynchronizeNowComplete"); } /// @@ -797,12 +798,12 @@ public async Task ConfirmSubmittedEntries() if (stats != null) stats.EventCounters["ConfirmSubmittedEntriesCalled"]++; - Services.Verbose("ConfirmSubmittedEntriesStart"); + Services.Log(Severity.Verbose, "ConfirmSubmittedEntriesStart"); if (pending.Count != 0) await worker.WaitForCurrentWorkToBeServiced(); - Services.Verbose("ConfirmSubmittedEntriesEnd"); + Services.Log(Severity.Verbose, "ConfirmSubmittedEntriesEnd"); } /// diff --git a/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs index 87d79a9146..7638ee6a2c 100644 --- a/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/CustomStorage/LogViewAdaptor.cs @@ -144,7 +144,7 @@ protected override async Task ReadAsync() } } - Services.Verbose("read success v{0}", version); + Services.Log(Severity.Verbose, "read success v{0}", version); LastPrimaryIssue.Resolve(Host, Services); @@ -159,7 +159,7 @@ protected override async Task ReadAsync() LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } @@ -193,7 +193,7 @@ protected override async Task WriteAsync() if (writesuccessful) { - Services.Verbose("write ({0} updates) success v{1}", updates.Count, version + updates.Count); + Services.Log(Severity.Verbose, "write ({0} updates) success v{1}", updates.Count, version + updates.Count); // now we update the cached state by applying the same updates // in case we encounter any exceptions we will re-read the whole state from storage @@ -215,7 +215,7 @@ protected override async Task WriteAsync() if (!writesuccessful || !transitionssuccessful) { - Services.Verbose("{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue); + Services.Log(Severity.Verbose, "{0} failed {1}", writesuccessful ? "transitions" : "write", LastPrimaryIssue); while (true) // be stubborn until we can re-read the state from storage { @@ -227,7 +227,7 @@ protected override async Task WriteAsync() version = result.Key; cached = result.Value; - Services.Verbose("read success v{0}", version); + Services.Log(Severity.Verbose, "read success v{0}", version); LastPrimaryIssue.Resolve(Host, Services); @@ -242,7 +242,7 @@ protected override async Task WriteAsync() LastPrimaryIssue.Record(new ReadFromPrimaryFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); } } @@ -329,7 +329,7 @@ protected override void ProcessNotifications() // discard notifications that are behind our already confirmed state while (notifications.Count > 0 && notifications.ElementAt(0).Key < version) { - Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + Services.Log(Severity.Verbose, "discarding notification {0}", notifications.ElementAt(0).Value); notifications.RemoveAt(0); } @@ -352,10 +352,10 @@ protected override void ProcessNotifications() version = updatenotification.Version; - Services.Verbose("notification success ({0} updates) v{1}", updatenotification.Updates.Count, version); + Services.Log(Severity.Verbose, "notification success ({0} updates) v{1}", updatenotification.Updates.Count, version); } - Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + Services.Log(Severity.Verbose2, "unprocessed notifications in queue: {0}", notifications.Count); base.ProcessNotifications(); @@ -364,13 +364,13 @@ protected override void ProcessNotifications() [Conditional("DEBUG")] private void enter_operation(string name) { - Services.Verbose2("/-- enter {0}", name); + Services.Log(Severity.Verbose2, "/-- enter {0}", name); } [Conditional("DEBUG")] private void exit_operation(string name) { - Services.Verbose2("\\-- exit {0}", name); + Services.Log(Severity.Verbose2, "\\-- exit {0}", name); } } diff --git a/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs index 8a02dba715..3e507c798c 100644 --- a/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/LogStorage/LogViewAdaptor.cs @@ -8,6 +8,7 @@ using Orleans.LogConsistency; using Orleans.Storage; using Orleans.EventSourcing.Common; +using Orleans.Runtime; namespace Orleans.EventSourcing.LogStorage { @@ -105,7 +106,7 @@ protected override async Task ReadAsync() await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); - Services.Verbose("read success {0}", GlobalLog); + Services.Log(Severity.Verbose, "read success {0}", GlobalLog); UpdateConfirmedView(); @@ -118,7 +119,7 @@ protected override async Task ReadAsync() LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } @@ -148,7 +149,7 @@ protected override async Task WriteAsync() batchsuccessfullywritten = true; - Services.Verbose("write ({0} updates) success {1}", updates.Length, GlobalLog); + Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalLog); UpdateConfirmedView(); @@ -161,7 +162,7 @@ protected override async Task WriteAsync() if (!batchsuccessfullywritten) { - Services.Verbose("write apparently failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "write apparently failed {0}", LastPrimaryIssue); while (true) // be stubborn until we can read what is there { @@ -172,7 +173,7 @@ protected override async Task WriteAsync() { await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalLog); - Services.Verbose("read success {0}", GlobalLog); + Services.Log(Severity.Verbose, "read success {0}", GlobalLog); UpdateConfirmedView(); @@ -185,14 +186,14 @@ protected override async Task WriteAsync() LastPrimaryIssue.Record(new ReadFromLogStorageFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); } // check if last apparently failed write was in fact successful if (writebit == GlobalLog.StateAndMetaData.GetBit(Services.MyClusterId)) { - Services.Verbose("last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog); + Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalLog); batchsuccessfullywritten = true; } @@ -313,7 +314,7 @@ protected override void ProcessNotifications() // discard notifications that are behind our already confirmed state while (notifications.Count > 0 && notifications.ElementAt(0).Key < GlobalLog.StateAndMetaData.GlobalVersion) { - Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + Services.Log(Severity.Verbose, "discarding notification {0}", notifications.ElementAt(0).Value); notifications.RemoveAt(0); } @@ -333,10 +334,10 @@ protected override void ProcessNotifications() UpdateConfirmedView(); - Services.Verbose("notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalLog); + Services.Log(Severity.Verbose, "notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalLog); } - Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + Services.Log(Severity.Verbose2, "unprocessed notifications in queue: {0}", notifications.Count); base.ProcessNotifications(); @@ -353,7 +354,7 @@ protected override void ProcessNotifications() private void enter_operation(string name) { #if DEBUG - Services.Verbose2("/-- enter {0}", name); + Services.Log(Severity.Verbose2, "/-- enter {0}", name); Debug.Assert(!operation_in_progress); operation_in_progress = true; #endif @@ -363,7 +364,7 @@ private void enter_operation(string name) private void exit_operation(string name) { #if DEBUG - Services.Verbose2("\\-- exit {0}", name); + Services.Log(Severity.Verbose2, "\\-- exit {0}", name); Debug.Assert(operation_in_progress); operation_in_progress = false; #endif diff --git a/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs index 7feec78ffc..a8fd3d3d70 100644 --- a/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs +++ b/src/OrleansEventSourcing/StateStorage/LogViewAdaptor.cs @@ -8,6 +8,7 @@ using Orleans.LogConsistency; using Orleans.Storage; using Orleans.EventSourcing.Common; +using Orleans.Runtime; namespace Orleans.EventSourcing.StateStorage { @@ -80,7 +81,7 @@ protected override async Task ReadAsync() await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); - Services.Verbose("read success {0}", GlobalStateCache); + Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); @@ -91,7 +92,7 @@ protected override async Task ReadAsync() LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); await LastPrimaryIssue.DelayBeforeRetry(); } @@ -127,7 +128,7 @@ protected override async Task WriteAsync() GlobalStateCache = nextglobalstate; - Services.Verbose("write ({0} updates) success {1}", updates.Length, GlobalStateCache); + Services.Log(Severity.Verbose, "write ({0} updates) success {1}", updates.Length, GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); } @@ -138,7 +139,7 @@ protected override async Task WriteAsync() if (!batchsuccessfullywritten) { - Services.Verbose("write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue); + Services.Log(Severity.Verbose, "write apparently failed {0} {1}", nextglobalstate, LastPrimaryIssue); while (true) // be stubborn until we can read what is there { @@ -149,7 +150,7 @@ protected override async Task WriteAsync() { await globalStorageProvider.ReadStateAsync(grainTypeName, Services.GrainReference, GlobalStateCache); - Services.Verbose("read success {0}", GlobalStateCache); + Services.Log(Severity.Verbose, "read success {0}", GlobalStateCache); LastPrimaryIssue.Resolve(Host, Services); @@ -160,7 +161,7 @@ protected override async Task WriteAsync() LastPrimaryIssue.Record(new ReadFromStateStorageFailed() { Exception = e }, Host, Services); } - Services.Verbose("read failed {0}", LastPrimaryIssue); + Services.Log(Severity.Verbose, "read failed {0}", LastPrimaryIssue); } // check if last apparently failed write was in fact successful @@ -169,7 +170,7 @@ protected override async Task WriteAsync() { GlobalStateCache = nextglobalstate; - Services.Verbose("last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache); + Services.Log(Severity.Verbose, "last write ({0} updates) was actually a success {1}", updates.Length, GlobalStateCache); batchsuccessfullywritten = true; } @@ -290,7 +291,7 @@ protected override void ProcessNotifications() // discard notifications that are behind our already confirmed state while (notifications.Count > 0 && notifications.ElementAt(0).Key < GlobalStateCache.StateAndMetaData.GlobalVersion) { - Services.Verbose("discarding notification {0}", notifications.ElementAt(0).Value); + Services.Log(Severity.Verbose, "discarding notification {0}", notifications.ElementAt(0).Value); notifications.RemoveAt(0); } @@ -317,10 +318,10 @@ protected override void ProcessNotifications() GlobalStateCache.ETag = updateNotification.ETag; - Services.Verbose("notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalStateCache); + Services.Log(Severity.Verbose, "notification success ({0} updates) {1}", updateNotification.Updates.Count, GlobalStateCache); } - Services.Verbose2("unprocessed notifications in queue: {0}", notifications.Count); + Services.Log(Severity.Verbose2, "unprocessed notifications in queue: {0}", notifications.Count); base.ProcessNotifications(); @@ -337,7 +338,7 @@ protected override void ProcessNotifications() private void enter_operation(string name) { #if DEBUG - Services.Verbose2("/-- enter {0}", name); + Services.Log(Severity.Verbose2, "/-- enter {0}", name); Debug.Assert(!operation_in_progress); operation_in_progress = true; #endif @@ -347,7 +348,7 @@ private void enter_operation(string name) private void exit_operation(string name) { #if DEBUG - Services.Verbose2("\\-- exit {0}", name); + Services.Log(Severity.Verbose2, "\\-- exit {0}", name); Debug.Assert(operation_in_progress); operation_in_progress = false; #endif diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs index 0251de1625..f7e404bf4b 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -201,8 +201,6 @@ public void CaughtException(string where, Exception e) PseudoMultiClusterConfiguration == null ? "" : (" " + Silo.CurrentSilo.ClusterId), where),e); } - - public void CaughtUserCodeException(string callback, string where, Exception e) { @@ -214,50 +212,19 @@ public void CaughtUserCodeException(string callback, string where, Exception e) where), e); } - - public void Info(string format, params object[] args) - { - log?.Info("{0}{1} {2}", - grain.GrainReference, - PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), - string.Format(format, args)); - } - - public void Verbose(string format, params object[] args) - { - if (log != null && log.IsVerbose) - { - log.Verbose("{0}{1} {2}", - grain.GrainReference, - PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), - string.Format(format, args)); - } - } - - /// Output the specified message at Verbose2 log level. - public void Verbose2(string format, params object[] args) - { - if (log != null && log.IsVerbose2) - { - log.Verbose2("{0}{1} {2}", - grain.GrainReference, - PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), - string.Format(format, args)); - } - } - - /// Output the specified message at Verbose3 log level. - public void Verbose3(string format, params object[] args) + public void Log(Severity severity, string format, params object[] args) { - if (log != null && log.IsVerbose3) + if (log != null && log.SeverityLevel >= severity) { - log.Verbose3("{0}{1} {2}", - grain.GrainReference, - PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), - string.Format(format, args)); + var msg = string.Format("{0}{1} {2}", + grain.GrainReference, + PseudoMultiClusterConfiguration != null ? "" : (" " + Silo.CurrentSilo.ClusterId), + string.Format(format, args)); + log.Log(0, severity, msg, EmptyObjectArray, null); } } + private static readonly object[] EmptyObjectArray = new object[0]; } } From 641f53d63bf32ca2446ce2d26eddc95656907f5b Mon Sep 17 00:00:00 2001 From: Sebastian Burckhardt Date: Thu, 12 Jan 2017 11:14:48 -0800 Subject: [PATCH 14/14] use DeploymentId when creating a pseudo-multicluster-configuration, instead of a made-up identifier --- .../LogConsistency/ProtocolServices.cs | 27 ++++++++----------- src/OrleansRuntime/Silo/Silo.cs | 3 ++- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs index f7e404bf4b..7bdf399ad6 100644 --- a/src/OrleansRuntime/LogConsistency/ProtocolServices.cs +++ b/src/OrleansRuntime/LogConsistency/ProtocolServices.cs @@ -29,17 +29,23 @@ internal class ProtocolServices : ILogConsistencyProtocolServices private Grain grain; // links to the grain that owns this service object + // pseudo-configuration to use if there is no actual multicluster network + private static MultiClusterConfiguration PseudoMultiClusterConfiguration; + internal ProtocolServices(Grain gr, Logger log, IMultiClusterRegistrationStrategy strategy) { this.grain = gr; this.log = log; this.RegistrationStrategy = strategy; - if (!Silo.CurrentSilo.GlobalConfig.HasMultiClusterNetwork) - PseudoMultiClusterConfiguration = new MultiClusterConfiguration(DateTime.UtcNow, new string[] { PseudoClusterId }.ToList()); + if (!Silo.CurrentSilo.HasMultiClusterNetwork) + { + // we are creating a default multi-cluster configuration containing exactly one cluster, this one. + PseudoMultiClusterConfiguration = new MultiClusterConfiguration( + DateTime.UtcNow, new string[] { Silo.CurrentSilo.ClusterId }.ToList()); + } } - public async Task SendMessage(ILogConsistencyProtocolMessage payload, string clusterId) { var silo = Silo.CurrentSilo; @@ -94,11 +100,6 @@ public async Task SendMessage(ILogConsistencyPro } } - // pseudo-configuration to use if there is no actual multicluster network - private static MultiClusterConfiguration PseudoMultiClusterConfiguration; - internal static string PseudoClusterId = "I"; - - public bool MultiClusterEnabled { get @@ -111,10 +112,7 @@ public string MyClusterId { get { - if (PseudoMultiClusterConfiguration != null) - return PseudoClusterId; - else - return Silo.CurrentSilo.ClusterId; + return Silo.CurrentSilo.ClusterId; } } @@ -122,10 +120,7 @@ public MultiClusterConfiguration MultiClusterConfiguration { get { - if (PseudoMultiClusterConfiguration != null) - return PseudoMultiClusterConfiguration; - else - return Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration(); + return PseudoMultiClusterConfiguration ?? Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration(); } } diff --git a/src/OrleansRuntime/Silo/Silo.cs b/src/OrleansRuntime/Silo/Silo.cs index 0826caa7d0..46aa95a593 100644 --- a/src/OrleansRuntime/Silo/Silo.cs +++ b/src/OrleansRuntime/Silo/Silo.cs @@ -35,6 +35,7 @@ using Orleans.Storage; using Orleans.Streams; using Orleans.Timers; +using Orleans.MultiCluster; namespace Orleans.Runtime { @@ -129,7 +130,7 @@ public bool HasMultiClusterNetwork /// Get the id of the cluster this silo is part of. public string ClusterId { - get { return GlobalConfig.HasMultiClusterNetwork ? GlobalConfig.ClusterId : ProtocolServices.PseudoClusterId; } + get { return GlobalConfig.HasMultiClusterNetwork ? GlobalConfig.ClusterId : GlobalConfig.DeploymentId; } } /// SiloAddress for this silo.