From 9967757b4b2bfeccbc876fa8ba7e6062b8c98796 Mon Sep 17 00:00:00 2001 From: Joao Sousa Date: Thu, 22 Jul 2021 21:23:19 +0100 Subject: [PATCH] Use offset on single kv to avoid scanning the full file --- .../org/example/kv/IndexedKeyValueStore.kt | 16 +++++++++++----- .../kotlin/org/example/kv/KeyValueStore.kt | 7 ++++--- .../org/example/kv/SingleLogKeyValueStore.kt | 4 ++-- .../sequential/SequentialLogMergeStrategy.kt | 5 ++++- .../lsm/sequential/SequentialOpenSegment.kt | 19 ++++++------------- .../kv/lsm/sstable/SortedStringTable.kt | 7 ++++--- .../kotlin/org/example/kv/KeyValueStores.kt | 2 ++ 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/org/example/kv/IndexedKeyValueStore.kt b/src/main/kotlin/org/example/kv/IndexedKeyValueStore.kt index bac81fa..f166949 100644 --- a/src/main/kotlin/org/example/kv/IndexedKeyValueStore.kt +++ b/src/main/kotlin/org/example/kv/IndexedKeyValueStore.kt @@ -26,9 +26,7 @@ private class IndexedKeyValueStore( } } - private fun getRaw(key: K, nullifyTombstone: Boolean): V? { - - val offset = index.getOffset(key) + private fun getRaw(key: K, offset: Long?, nullifyTombstone: Boolean): V? { if (offset == tombstoneIndex) { return if (nullifyTombstone) null else tombstone @@ -43,9 +41,17 @@ private class IndexedKeyValueStore( ?.value } - override fun getWithTombstone(key: K) = getRaw(key, false) + override fun getWithTombstone(key: K, offset: Long?) = getRaw( + key, + offset ?: index.getOffset(key), + false + ) - override fun get(key: K) = getRaw(key, true) + override fun get(key: K, offset: Long?) = getRaw( + key, + offset ?: index.getOffset(key), + true + ) override fun delete(key: K) { logKV.delete(key) diff --git a/src/main/kotlin/org/example/kv/KeyValueStore.kt b/src/main/kotlin/org/example/kv/KeyValueStore.kt index ac9bebc..5a1c668 100644 --- a/src/main/kotlin/org/example/kv/KeyValueStore.kt +++ b/src/main/kotlin/org/example/kv/KeyValueStore.kt @@ -23,8 +23,6 @@ interface LogBasedKeyValueStore: TombstoneKeyValueStore { fun appendAll(entries: Map): Sequence - fun get(key: K, offset: Long): V? - fun getWithOffset(key: K): ValueWithOffset? fun useEntries(block: (Sequence>) -> R): R = log.useEntries(0) { block(it) } @@ -38,8 +36,11 @@ interface LogBasedKeyValueStore: TombstoneKeyValueStore { interface TombstoneKeyValueStore: KeyValueStore { - fun getWithTombstone(key: K): V? + fun getWithTombstone(key: K, offset: Long? = null): V? + + fun get(key: K, offset: Long?): V? + override fun get(key: K) = get(key, null) } data class ValueWithOffset(val offset: Long, val value: V) diff --git a/src/main/kotlin/org/example/kv/SingleLogKeyValueStore.kt b/src/main/kotlin/org/example/kv/SingleLogKeyValueStore.kt index f0b61ef..4b2527b 100644 --- a/src/main/kotlin/org/example/kv/SingleLogKeyValueStore.kt +++ b/src/main/kotlin/org/example/kv/SingleLogKeyValueStore.kt @@ -32,13 +32,13 @@ private class SingleLogKeyValueStore(override val log: Log } } - override fun get(key: K, offset: Long) = getWithTombstone(key) + override fun get(key: K, offset: Long?) = getWithTombstone(key, offset) ?.takeUnless { possiblyArrayEquals(it, tombstone) } override fun getWithOffset(key: K) = log.useEntriesWithOffset { it.findLastKey(key) } ?.takeUnless { possiblyArrayEquals(it.value, tombstone) } - override fun getWithTombstone(key: K): V? = log.useEntries(0) { it.findLastKey(key) } + override fun getWithTombstone(key: K, offset: Long?): V? = log.useEntries(offset ?: 0L) { it.findLastKey(key) } private fun Sequence>.findLastKey(key: K): V? = this .findLast { possiblyArrayEquals(key, it.key) }?.value diff --git a/src/main/kotlin/org/example/kv/lsm/sequential/SequentialLogMergeStrategy.kt b/src/main/kotlin/org/example/kv/lsm/sequential/SequentialLogMergeStrategy.kt index 1494588..6526f47 100644 --- a/src/main/kotlin/org/example/kv/lsm/sequential/SequentialLogMergeStrategy.kt +++ b/src/main/kotlin/org/example/kv/lsm/sequential/SequentialLogMergeStrategy.kt @@ -1,6 +1,9 @@ -package org.example.kv.lsm +package org.example.kv.lsm.sequential import mu.KotlinLogging +import org.example.kv.lsm.Segment +import org.example.kv.lsm.SegmentFactory +import org.example.kv.lsm.SegmentMergeStrategy import org.example.size.SizeCalculator import java.util.* diff --git a/src/main/kotlin/org/example/kv/lsm/sequential/SequentialOpenSegment.kt b/src/main/kotlin/org/example/kv/lsm/sequential/SequentialOpenSegment.kt index b1cfaaa..53ed463 100644 --- a/src/main/kotlin/org/example/kv/lsm/sequential/SequentialOpenSegment.kt +++ b/src/main/kotlin/org/example/kv/lsm/sequential/SequentialOpenSegment.kt @@ -1,22 +1,14 @@ -package org.example.kv.lsm +package org.example.kv.lsm.sequential import org.example.kv.LogBasedKeyValueStore import org.example.kv.LogBasedKeyValueStoreFactory +import org.example.kv.TombstoneKeyValueStore +import org.example.kv.lsm.* import org.example.log.LogFactory private class SequentialOpenSegment(private val keyValueStore: LogBasedKeyValueStore, private val segmentThreshold: Long = 1024 * 1024): - OpenSegment { - - override fun put(key: K, value: V) = keyValueStore.put(key, value) - - override fun getWithTombstone(key: K): V? = keyValueStore.getWithTombstone(key) - - override fun get(key: K): V? = keyValueStore.get(key) - - override fun delete(key: K) = keyValueStore.delete(key) - - override fun clear() = keyValueStore.clear() + OpenSegment, TombstoneKeyValueStore by keyValueStore { override fun closeSegment(): Segment = Segment(keyValueStore, segmentThreshold) @@ -27,7 +19,8 @@ class SequentialSegmentManager(private val segmentDirectory: SegmentDirect private val logFactory: LogFactory>, private val keyValueStoreFactory: LogBasedKeyValueStoreFactory, segmentThreshold: Long, - mergeStrategy: SequentialLogMergeStrategy): + mergeStrategy: SequentialLogMergeStrategy +): SegmentManager(segmentDirectory, logFactory, keyValueStoreFactory, mergeStrategy, segmentThreshold) { override fun createOpenSegment(): OpenSegment = segmentDirectory diff --git a/src/main/kotlin/org/example/kv/lsm/sstable/SortedStringTable.kt b/src/main/kotlin/org/example/kv/lsm/sstable/SortedStringTable.kt index da2cef5..1faed55 100644 --- a/src/main/kotlin/org/example/kv/lsm/sstable/SortedStringTable.kt +++ b/src/main/kotlin/org/example/kv/lsm/sstable/SortedStringTable.kt @@ -32,8 +32,9 @@ private class SortedMapStringTable, V>(private val memTable: So } } - override fun get(key: K): V? = when(dumped) { - true -> dump.get(key) + override fun get(key: K, offset: Long?): V? = when(dumped) { + true -> dump.get(key, offset) + // offset is ignored here. Is this problematic? false -> memTable[key] } @@ -69,7 +70,7 @@ private class SortedMapStringTable, V>(private val memTable: So } } - override fun getWithTombstone(key: K): V? = get(key) + override fun getWithTombstone(key: K, offset: Long?): V? = get(key) override fun isFull(): Boolean = size >= segmentThreshold diff --git a/src/test/kotlin/org/example/kv/KeyValueStores.kt b/src/test/kotlin/org/example/kv/KeyValueStores.kt index 9f61437..cf97a81 100644 --- a/src/test/kotlin/org/example/kv/KeyValueStores.kt +++ b/src/test/kotlin/org/example/kv/KeyValueStores.kt @@ -9,6 +9,8 @@ import org.example.TestInstance import org.example.TestResources import org.example.encoder.Encoders import org.example.kv.lsm.* +import org.example.kv.lsm.sequential.SequentialLogMergeStrategy +import org.example.kv.lsm.sequential.SequentialSegmentManager import org.example.kv.lsm.sstable.SSTableMergeStrategy import org.example.kv.lsm.sstable.SSTableSegmentManager import org.example.log.LogEncoderFactory