Skip to content

Commit

Permalink
Made KV Test DiskWriteScanBasicTest a multiple device test and labele…
Browse files Browse the repository at this point in the history 
…d smoke. Also, h duplicate of CommitAsyncPrevTask so removed the dupe
  • Loading branch information
@darrenge
darrenge committed Jun 30, 2021
1 parent 7b247e3 commit d337ddf
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 93 deletions.
67 changes: 0 additions & 67 deletions cs/test/FasterLogTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,73 +321,6 @@ public void CommitNoSpinWait()
log.Dispose();
}

[Test]
[Category("FasterLog")]
public async ValueTask CommitAsyncPrevTask()
{
CancellationTokenSource cts = new CancellationTokenSource();
CancellationToken token = cts.Token;

var logSettings = new FasterLogSettings { LogDevice = device, LogCommitManager = manager };
log = await FasterLog.CreateAsync(logSettings);

// make it small since launching each on separate threads
int entryLength = 10;
int expectedEntries = 3; // Not entry length because this is number of enqueues called

// Set Default entry data
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}

// Enqueue and AsyncCommit in a separate thread (wait there until commit is done though).
Task currentTask = Task.Run(() => LogWriterAsync(log, entry), token);

// Give all a second or so to queue up and to help with timing issues - shouldn't need but timing issues
Thread.Sleep(2000);

// Commit to the log
currentTask.Wait(4000, token);

// double check to make sure finished - seen cases where timing kept running even after commit done
bool wasCanceled = false;
if (currentTask.Status != TaskStatus.RanToCompletion)
{
wasCanceled = true;
Console.WriteLine("currentTask failed to complete; canceling");
cts.Cancel();
}

// Read the log to make sure all entries are put in
int currentEntry = 0;
using (var iter = log.Scan(0, 100_000_000))
{
while (iter.GetNext(out byte[] result, out _, out _))
{
if (currentEntry < entryLength)
{

Assert.IsTrue(result[currentEntry] == (byte)currentEntry, "Fail - Result[" + currentEntry.ToString() + "]:" + result[0].ToString() + " not match expected:" + currentEntry);

currentEntry++;
}
}
}

// Make sure expected entries is same as current - also makes sure that data verification was not skipped
Assert.AreEqual(expectedEntries, currentEntry);

// NOTE: seeing issues where task is not running to completion on Release builds
// This is a final check to make sure task finished. If didn't then assert
// One note - if made it this far, know that data was Enqueue and read properly, so just
// case of task not stopping
if (currentTask.Status != TaskStatus.RanToCompletion)
{
Assert.Fail($"Final Status check Failure -- Task should be 'RanToCompletion' but current Status is: {currentTask.Status}; wasCanceled = {wasCanceled}");
}
}

[Test]
[Category("FasterLog")]
public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iteratorType)
Expand Down
77 changes: 51 additions & 26 deletions cs/test/GenericLogScanTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,40 +12,65 @@ internal class GenericFASTERScanTests
{
private FasterKV<MyKey, MyValue> fht;
private IDevice log, objlog;
const int totalRecords = 2000;
const int totalRecords = 200;
private string path;

[SetUp]
public void Setup()
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/GenericFASTERScanTests.log", deleteOnClose: true);
objlog = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/GenericFASTERScanTests.obj.log", deleteOnClose: true);
path = TestUtils.MethodTestDir + "/";

// Clean up log files from previous test runs in case they weren't cleaned up
TestUtils.DeleteDirectory(path, wait: true);

fht = new FasterKV<MyKey, MyValue>
(128,
logSettings: new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 15, PageSizeBits = 9 },
checkpointSettings: new CheckpointSettings { CheckPointType = CheckpointType.FoldOver },
serializerSettings: new SerializerSettings<MyKey, MyValue> { keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer() }
);
}

[TearDown]
public void TearDown()
{
fht?.Dispose();
fht = null;
log?.Dispose();
log = null;
objlog?.Dispose();
objlog = null;
//** #142980 - Blob not exist exception in Dispose so use Try \ Catch to make sure tests run without issues
try
{
fht?.Dispose();
fht = null;
log?.Dispose();
log = null;
objlog?.Dispose();
objlog = null;
}
catch { }

TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}

[Test]
[Category("FasterKV")]
public void DiskWriteScanBasicTest()
[Category("Smoke")]
public void DiskWriteScanBasicTest([Values] TestUtils.DeviceType deviceType)
{


#if WINDOWS
//*#*#*# Bug #143131 - fht.Log.Scan failing to get proper value on EmulatedAzure only *#*#*#
if (deviceType == TestUtils.DeviceType.EmulatedAzure)
{
return;
}
#endif


string logfilename = path + "DiskWriteScanBasicTest" + deviceType.ToString() + ".log";
string objlogfilename = path + "DiskWriteScanBasicTest" + deviceType.ToString() + ".obj.log";

log = TestUtils.CreateTestDevice(deviceType, logfilename);
objlog = TestUtils.CreateTestDevice(deviceType, objlogfilename);
fht = new FasterKV<MyKey, MyValue>
(128,
logSettings: new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 15, PageSizeBits = 9, SegmentSizeBits = 22 },
checkpointSettings: new CheckpointSettings { CheckPointType = CheckpointType.FoldOver },
serializerSettings: new SerializerSettings<MyKey, MyValue> { keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer() }
);

using var session = fht.For(new MyFunctions()).NewSession<MyFunctions>();

var s = fht.Log.Subscribe(new LogObserver());
Expand All @@ -65,23 +90,23 @@ public void DiskWriteScanBasicTest()
int val = 0;
while (iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value))
{
Assert.IsTrue(key.key == val);
Assert.IsTrue(value.value == val);
Assert.IsTrue(key.key == val,$"log scan 1: key.key: {key.key} should = val: {val}");
Assert.IsTrue(value.value == val, $"log scan 1: value.key: {value.value} should = val: {val}");
val++;
}
Assert.IsTrue(totalRecords == val);
Assert.IsTrue(totalRecords == val, $"log scan 2: totalRecords: {totalRecords} should = val: {val}");
}

using (var iter = fht.Log.Scan(start, fht.Log.TailAddress, ScanBufferingMode.DoublePageBuffering))
{
int val = 0;
while (iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value))
{
Assert.IsTrue(key.key == val);
Assert.IsTrue(value.value == val);
Assert.IsTrue(key.key == val, $"log scan 2: key.key: {key.key} should = val: {val}");
Assert.IsTrue(value.value == val, $"log scan 2: value.key: {value.value} should = val: {val}");
val++;
}
Assert.IsTrue(totalRecords == val);
Assert.IsTrue(totalRecords == val, $"log scan 2: totalRecords: {totalRecords} should = val: {val}");
}

s.Dispose();
Expand All @@ -93,7 +118,7 @@ class LogObserver : IObserver<IFasterScanIterator<MyKey, MyValue>>

public void OnCompleted()
{
Assert.IsTrue(val == totalRecords);
Assert.IsTrue(val == totalRecords, $"OnCompleted: totalRecords: {totalRecords} should = val: {val}");
}

public void OnError(Exception error)
Expand All @@ -104,8 +129,8 @@ public void OnNext(IFasterScanIterator<MyKey, MyValue> iter)
{
while (iter.GetNext(out _, out MyKey key, out MyValue value))
{
Assert.IsTrue(key.key == val);
Assert.IsTrue(value.value == val);
Assert.IsTrue(key.key == val,$"OnNext: key.key: {key.key} should = val: {val}");
Assert.IsTrue(value.value == val, $"OnNext: value.value: {value.value} should = val: {val}");
val++;
}
}
Expand Down

0 comments on commit d337ddf

Please sign in to comment.