Merge branch 'iter-fixes' of https://github.com/microsoft/FASTER into…

… iter-fixes

README.md

@@ -38,6 +38,9 @@ analysis of C# FASTER KV performance in a wiki page
 C# and C++ versions of FASTER are very similar. FASTER Log is also extremely fast, capable of saturating modern
 NVMe SSDs using less than a core of CPU, and scaling well in a multi-threaded setting.
 
+:new: We now support C# async in FASTER KV (and FASTER Log). See the detailed guide at [this link](https://github.com/Microsoft/FASTER/blob/master/cs/README.md) for more information. Also, check out the 
+samples in the playground located [here](https://github.com/Microsoft/FASTER/tree/master/cs/playground).
+
 # Getting Started
 
 Visit our [research website](http://aka.ms/FASTER) for technical details and papers. For FASTER usage and 

cs/FASTER.sln

@@ -46,6 +46,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FASTER.devices.AzureStorage
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterLogSample", "playground\FasterLogSample\FasterLogSample.csproj", "{25C5C6B6-4A8A-46DD-88C1-EB247033FE58}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterKVAsyncSample", "playground\FasterKVAsyncSample\FasterKVAsyncSample.csproj", "{859F76F4-93D8-4D60-BF9A-363E217FA247}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -166,6 +168,12 @@ Global
 		{25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|Any CPU.Build.0 = Release|x64
 		{25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|x64.ActiveCfg = Release|x64
 		{25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|x64.Build.0 = Release|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|x64.ActiveCfg = Debug|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|x64.Build.0 = Debug|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|Any CPU.ActiveCfg = Release|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|x64.ActiveCfg = Release|x64
+		{859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -187,6 +195,7 @@ Global
 		{A6B14415-D316-4955-BE5F-725BB2DEBEBE} = {28800357-C8CE-4CD0-A2AD-D4A910ABB496}
 		{E571E686-01A0-44D5-BFF5-B7678284258B} = {A6B14415-D316-4955-BE5F-725BB2DEBEBE}
 		{25C5C6B6-4A8A-46DD-88C1-EB247033FE58} = {E6026D6A-01C5-4582-B2C1-64751490DABE}
+		{859F76F4-93D8-4D60-BF9A-363E217FA247} = {E6026D6A-01C5-4582-B2C1-64751490DABE}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A0750637-2CCB-4139-B25E-F2CE740DCFAC}

cs/README.md

@@ -1,7 +1,7 @@
 Introduction to FASTER C#
 =========================
 
-FASTER C# works in .NET Framework and .NET core, and can be used in both a single-threaded and concurrent setting. It has been tested to work on both Windows and Linux. It exposes an API that allows one to performs a mix of Reads, Blind Updates (Upserts), and Read-Modify-Write operations. It supports data larger than memory, and accepts an `IDevice` implementation for storing logs on storage. We have provided `IDevice` implementations for local file system, but one may create new devices, for example, to write to remote file systems. Alternatively, one may mount remote storage into the local file system. FASTER  may be used as a high-performance replacement for traditional concurrent data structures such as the .NET ConcurrentDictionary, and additionally supports larger-than-memory data. It also supports checkpointing of the data structure - both incremental and non-incremental.
+FASTER C# works in .NET Framework and .NET core, and can be used in both a single-threaded and concurrent setting. It has been tested to work on both Windows and Linux. It exposes an API that allows one to performs a mix of Reads, Blind Updates (Upserts), and Read-Modify-Write operations. It supports data larger than memory, and accepts an `IDevice` implementation for storing logs on storage. We have provided `IDevice` implementations for local file system and Azure Page Blobs, but one may create new devices as well. We also offer meta-devices that can group device instances into sharded and tiered configurations. FASTER  may be used as a high-performance replacement for traditional concurrent data structures such as the .NET ConcurrentDictionary, and additionally supports larger-than-memory data. It also supports checkpointing of the data structure - both incremental and non-incremental. Operations on FASTER can be issued synchronously or asynchronously, i.e., using the C# `async` interface.
 
 Table of Contents
 -----------
@@ -14,7 +14,7 @@ Table of Contents
 ## Getting FASTER
 
 ### Building From Sources
-Clone the Git repo, open cs/FASTER.sln in VS 2017, and build.
+Clone the Git repo, open cs/FASTER.sln in VS 2019, and build.
 
 ### NuGet
 You can install FASTER binaries using Nuget, from Nuget.org. Right-click on your project, manage NuGet packages, browse for FASTER. Here is a [direct link](https://www.nuget.org/packages/FASTER).
@@ -30,7 +30,7 @@ FASTER supports three basic operations:
 
 ### Constructor
 
-Before instantiating FASTER, you need to create storage devices that FASTER will use. If you are using blittable types, you only need the hybrid log device. If you are also using objects, you need to create a separate object log device.
+Before instantiating FASTER, you need to create storage devices that FASTER will use. If you are using value (blittable) types, you only need one log device. If you are also using objects, you need to create a separate object log device.
 
 ```Csharp
 IDevice log = Devices.CreateLogDevice("C:\\Temp\\hybridlog_native.log");
@@ -55,11 +55,11 @@ There are six basic concepts, provided as generic type arguments when instantiat
 
 ### Callback Functions
 
-The user provides an instance of a type that implements `IFunctions<>`. This type encapsulates all the callbacks, which are described next:
+The user provides an instance of a type that implements `IFunctions<>`, or its corresponding abstract base class `FunctionsBase<>`. This type encapsulates all the callbacks, which are described next:
 
 1. SingleReader and ConcurrentReader: These are used to read from the store values and copy them to Output. Single reader can assume there are no concurrent operations.
 2. SingleWriter and ConcurrentWriter: These are used to write values to the store, from a source value.
-3. Completion callbacks: Called when various operations complete.
+3. Completion callbacks: Called by FASTER when various operations complete.
 4. RMW Updaters: There are three updaters that the user specifies, InitialUpdater, InPlaceUpdater, and CopyUpdater. Together, they are used to implement the RMW operation.
 
 ### Constructor Parameters
@@ -76,15 +76,22 @@ The total in-memory footprint of FASTER is controlled by the following parameter
 
 ### Sessions (Threads)
 
-Once FASTER is instantiated, threads may use FASTER by registering themselves via the concept of a Session, using the call 
+Once FASTER is instantiated, one issues operations to FASTER by creating logical sessions. A session represents a sequence of operations issued to FASTER. There is no concurrency within a session, but different sessions may execute concurrently. Sessions do not need to be affinitized to threads, but if they are, FASTER can leverage the same (covered later). You create a session as follows:
 
-```fht.StartSession();```
+```var session = fht.NewSession();```
 
-At the end, the thread calls:
+You can then perform a sequence of read, upsert, and RMW operations on the session. FASTER supports sync and async versions of operations. Examples:
 
-```fht.StopSession();```
+```cs
+var status = session.Read(ref key, ref input, ref output, ref context);
+await session.ReadAsync(key, input);
+```
+
+At the end, the session is disposed:
+
+```session.Dispose();```
 
-When all threads are done operating on FASTER, you finally dispose the FASTER instance:
+When all sessions are done operating on FASTER, you finally dispose the FASTER instance:
 
 ```fht.Dispose();```
 
@@ -100,16 +107,16 @@ public static void Test()
   var log = Devices.CreateLogDevice("C:\\Temp\\hlog.log");
   var fht = new FasterKV<long, long, long, long, Empty, Funcs>
     (1L << 20, new Funcs(), new LogSettings { LogDevice = log });
-  fht.StartSession();
+  var session = fht.NewSession();
   long key = 1, value = 1, input = 10, output = 0;
-  fht.Upsert(ref key, ref value, Empty.Default, 0);
-  fht.Read(ref key, ref input, ref output, Empty.Default, 0);
+  session.Upsert(ref key, ref value, Empty.Default, 0);
+  session.Read(ref key, ref input, ref output, Empty.Default, 0);
   Debug.Assert(output == value);
-  fht.RMW(ref key, ref input, Empty.Default, 0);
-  fht.RMW(ref key, ref input, Empty.Default, 0);
-  fht.Read(ref key, ref input, ref output, Empty.Default, 0);
+  session.RMW(ref key, ref input, Empty.Default, 0);
+  session.RMW(ref key, ref input, Empty.Default, 0);
+  session.Read(ref key, ref input, ref output, Empty.Default, 0);
   Debug.Assert(output == value + 20);
-  fht.StopSession();
+  session.Dispose();
   fht.Dispose();
   log.Close();
 }
@@ -130,7 +137,7 @@ public class Funcs : IFunctions<long, long, long, long, Empty>
   public void UpsertCompletionCallback(ref long key, ref long value, Empty ctx) { }
   public void ReadCompletionCallback(ref long key, ref long input, ref long output, Empty ctx, Status s) { }
   public void RMWCompletionCallback(ref long key, ref long input, Empty ctx, Status s) { }
-  public void CheckpointCompletionCallback(Guid sessionId, long serialNum) { }
+  public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) { }
 }
 ```
 
@@ -140,17 +147,20 @@ Several example projects are located in [cs/playground](https://github.com/Micro
 
 ## Checkpointing and Recovery
 
-FASTER supports **checkpoint-based recovery**. Every new checkpoint persists (or makes durable) additional user-operations (Read, Upsert or RMW). FASTER allows client threads to keep track of operations that have persisted and those that have not using a session-based API. 
+FASTER supports **checkpoint-based recovery**. Every new checkpoint persists (or makes durable) additional user-operations (Read, Upsert or RMW). FASTER allows clients to keep track of operations that have persisted and those that have not using a session-based API. 
 
-Recall that each FASTER threads starts a session, associated with a unique Guid.
-All FASTER thread operations (Read, Upsert, RMW) carry a monotonic sequence number.
-At any point in time, one may call `Checkpoint` to initiate an asynchronous checkpoint of FASTER.
-After calling `Checkpoint`, each FASTER thread is (eventually) notified of a sequence number, such that all operations until, and no operations after, that sequence number, are guaranteed to be persisted as part of that checkpoint. 
-This sequence number can be used by the FASTER thread to clear any in-memory buffer of operations waiting to be performed.
+Recall that each FASTER client starts a session, associated with a unique session ID (or name). All FASTER session operations (Read, Upsert, RMW) carry a monotonic sequence number (sequence numbers are implicit in case of async calls). At any point in time, one may call `Checkpoint` to initiate an asynchronous checkpoint of FASTER. After calling `Checkpoint`, each FASTER session is (eventually) notified of a commit point. A commit point consists of (1) a sequence number, such that all operations until, and no operations after, that sequence number, are guaranteed to be persisted as part of that checkpoint; (2) an optional exception list of operations that were not part of the commit because they went pending and could not complete before the checkpoint, because the session was not active at the time of checkpointing.
 
-During recovery, threads can continue their session with the same Guid using `ContinueSession`. The function returns the thread-local sequence number until which that session hash been recovered. The new thread may use this information to replay all uncommitted operations since that point.
+The commit point information can be used by the session to clear any in-memory buffer of operations waiting to be performed. During recovery, sessions can continue using `ResumeSession` invoked with the same session ID. The function returns the thread-local sequence number until which that session hash been recovered. The new thread may use this information to replay all uncommitted operations since that point.
+
+With async session operations on FASTER, one may optionally set a boolean parameter `waitForCommit` in the calls. When set, the async call completes only after the operation is made persistent by an asynchronous checkpoint. The user is responsible for performing the checkpoint asynchronously. An async upsert which returns only after the upsert is made durable, is shown below:
+
+```cs
+await session.UpsertAsync(key, value, waitForCommit: true);
+```
+
+Below, we show a simple recovery example with asynchronous checkpointing.
 
-Below, we show a simple recovery example for a single thread. 
 ```Csharp
 public class PersistenceExample 
 {
@@ -180,35 +190,27 @@ public class PersistenceExample
   /* Helper Functions */
   private void RunSession() 
   {
-    Guid guid = fht.StartSession();
-    System.IO.File.WriteAllText(@"C:\\Temp\\session1.txt", guid.ToString());
-
+    using var session = fht.NewSession("s1");
     long seq = 0; // sequence identifier
     
     long key = 1, input = 10;
     while(true) 
     {
       key = (seq % 1L << 20);
-      fht.RMW(ref key, ref input, Empty.Default, seq);
-      seq++;
+      session.RMW(ref key, ref input, Empty.Default, seq++);
     }
-    // fht.StopSession() - outside infinite loop
   }
 
   private void ContinueSession() 
   {
-    string guidText = System.IO.File.ReadAllText(@"C:\\Temp\session1.txt");
-    Guid sessionGuid = Guid.Parse(guidText);
-
-    long seq = fht.ContinueSession(sessionGuid); // recovered seq identifier
-    seq++;
+    using var session = fht.ResumeSession("s1", out CommitPoint cp); // recovered session
+    var seq = cp.UntilSerialNo + 1;
 
     long key = 1, input = 10;
     while(true) 
     {
       key = (seq % 1L << 20);
-      fht.RMW(ref key, ref input, Empty.Default, seq);
-      seq++;
+      session.RMW(ref key, ref input, Empty.Default, seq++);
     }
   }
 
@@ -219,10 +221,8 @@ public class PersistenceExample
       while(true) 
       {
         Thread.Sleep(10000);
-		fht.StartSession();
-        fht.TakeCheckpoint(out Guid token);
-        fht.CompleteCheckpoint(token, true);
-		fht.StopSession();
+        fht.TakeFullCheckpoint(out Guid token);
+        fht.CompleteCheckpointAsync().GetAwaiter().GetResult();
       }
     });
     t.Start();

cs/benchmark/ConcurrentDictionaryBenchmark.cs

@@ -60,11 +60,10 @@ public enum Op : ulong
         Input[] input_;
         Input* input_ptr;
 
-        ConcurrentDictionary<Key, Value> store;
+        readonly ConcurrentDictionary<Key, Value> store;
 
         long total_ops_done = 0;
 
-        const string kKeyWorkload = "a";
         readonly int threadCount;
         readonly int numaStyle;
         readonly string distribution;