Skip to content

Commit

Permalink
remove re-export of RunFsPaths
Browse files Browse the repository at this point in the history
There is no need for a module re-export and we don't do it anywhere else.
Also, wanting to know the file paths to where a run stores its data
breaks the abstraction boundary and is only needed for tests or
benchmarks. As such, it's good to be more explicit about importing them.
  • Loading branch information
mheinzel committed May 6, 2024
1 parent 8abfb67 commit a682e2e
Show file tree
Hide file tree
Showing 8 changed files with 38 additions and 31 deletions.
3 changes: 2 additions & 1 deletion bench/micro/Bench/Database/LSMTree/Internal/Lookup.hs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import Database.LSMTree.Internal.Lookup (BatchSize (..),
prepLookups)
import Database.LSMTree.Internal.Run (Run)
import qualified Database.LSMTree.Internal.Run as Run
import Database.LSMTree.Internal.RunFsPaths (RunFsPaths (..))
import Database.LSMTree.Internal.Serialise
import qualified Database.LSMTree.Internal.WriteBuffer as WB
import Prelude hiding (getContents)
Expand Down Expand Up @@ -128,7 +129,7 @@ lookupsInBatchesEnv Config {..} = do
let hasFS = FS.ioHasFS (FS.MountPoint benchTmpDir)
hasBlockIO <- FS.ioHasBlockIO hasFS ioctxps
let wb = WB.WB storedKeys
fsps = Run.RunFsPaths 0
fsps = RunFsPaths 0
r <- Run.fromWriteBuffer hasFS fsps wb
let nentriesReal = unNumEntries $ Run.runNumEntries r
assert (nentriesReal == nentries) $ pure ()
Expand Down
10 changes: 6 additions & 4 deletions bench/micro/Bench/Database/LSMTree/Internal/WriteBuffer.hs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import Database.LSMTree.Extras.UTxO
import Database.LSMTree.Internal.Entry
import Database.LSMTree.Internal.Run (Run)
import qualified Database.LSMTree.Internal.Run as Run
import Database.LSMTree.Internal.RunFsPaths (RunFsPaths (..),
activeRunsDir)
import Database.LSMTree.Internal.Serialise
import Database.LSMTree.Internal.WriteBuffer (WriteBuffer)
import qualified Database.LSMTree.Internal.WriteBuffer as WB
Expand Down Expand Up @@ -133,20 +135,20 @@ benchWriteBuffer conf@Config{name} =
writeBufferEnvCleanup

-- We'll remove the files on every run, so we can re-use the same run number.
getPaths :: IO Run.RunFsPaths
getPaths = pure (Run.RunFsPaths 0)
getPaths :: IO RunFsPaths
getPaths = pure (RunFsPaths 0)

-- Simply remove the whole active directory.
cleanupPaths :: FS.HasFS IO FS.HandleIO -> IO ()
cleanupPaths hasFS = FS.removeDirectoryRecursive hasFS Run.activeRunsDir
cleanupPaths hasFS = FS.removeDirectoryRecursive hasFS activeRunsDir

insert :: InputKOps -> WriteBuffer
insert (NormalInputs kops) =
List.foldl' (\wb (k, e) -> WB.addEntryNormal k e wb) WB.empty kops
insert (MonoidalInputs kops mappendVal) =
List.foldl' (\wb (k, e) -> WB.addEntryMonoidal mappendVal k e wb) WB.empty kops

flush :: FS.HasFS IO FS.HandleIO -> Run.RunFsPaths -> WriteBuffer -> IO (Run (FS.Handle (FS.HandleIO)))
flush :: FS.HasFS IO FS.HandleIO -> RunFsPaths -> WriteBuffer -> IO (Run (FS.Handle (FS.HandleIO)))
flush = Run.fromWriteBuffer

data InputKOps
Expand Down
9 changes: 4 additions & 5 deletions src/Database/LSMTree/Internal/Run.hs
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,9 @@
-- not exhaustive.
--
module Database.LSMTree.Internal.Run (
-- * Paths
module FsPaths
-- * Run
, Run (..)
Run (..)
, RunFsPaths
, sizeInPages
, addReference
, removeReference
Expand Down Expand Up @@ -73,7 +72,7 @@ import qualified Database.LSMTree.Internal.IndexCompact as Index
import qualified Database.LSMTree.Internal.RawBytes as RB
import Database.LSMTree.Internal.RunBuilder (RunBuilder)
import qualified Database.LSMTree.Internal.RunBuilder as Builder
import Database.LSMTree.Internal.RunFsPaths as FsPaths
import Database.LSMTree.Internal.RunFsPaths
import Database.LSMTree.Internal.Serialise
import Database.LSMTree.Internal.WriteBuffer (WriteBuffer)
import qualified Database.LSMTree.Internal.WriteBuffer as WB
Expand Down Expand Up @@ -206,7 +205,7 @@ openFromDisk fs runRunFsPaths = do
=<< CRC.readChecksumsFile fs (runChecksumsPath runRunFsPaths)

-- verify checksums of files we don't read yet
let paths = runFsPaths runRunFsPaths
let paths = pathsForRunFiles runRunFsPaths
checkCRC (forRunKOps expectedChecksums) (forRunKOps paths)
checkCRC (forRunBlob expectedChecksums) (forRunBlob paths)

Expand Down
4 changes: 2 additions & 2 deletions src/Database/LSMTree/Internal/RunBuilder.hs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ new fs runBuilderFsPaths numEntries estimatedNumPages = do
runBuilderBlobOffset <- newIORef 0

FS.createDirectoryIfMissing fs False activeRunsDir
runBuilderHandles <- traverse (makeHandle fs) (runFsPaths runBuilderFsPaths)
runBuilderHandles <- traverse (makeHandle fs) (pathsForRunFiles runBuilderFsPaths)

let builder = RunBuilder {..}
writeIndexHeader fs builder
Expand Down Expand Up @@ -169,7 +169,7 @@ unsafeFinalise fs builder@RunBuilder {..} = do
close :: HasFS IO h -> RunBuilder (FS.Handle h) -> IO ()
close fs RunBuilder {..} = do
traverse_ (closeHandle fs) runBuilderHandles
traverse_ (FS.removeFile fs) (runFsPaths runBuilderFsPaths)
traverse_ (FS.removeFile fs) (pathsForRunFiles runBuilderFsPaths)

{-------------------------------------------------------------------------------
Helpers
Expand Down
25 changes: 13 additions & 12 deletions src/Database/LSMTree/Internal/RunFsPaths.hs
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
module Database.LSMTree.Internal.RunFsPaths (
RunFsPaths (..)
, runFsPaths
, pathsForRunFiles
, runKOpsPath
, runBlobPath
, runFilterPath
, runIndexPath
, runChecksumsPath
, activeRunsDir
-- * Checksums
, runChecksumFileNames
, checksumFileNamesForRunFiles
, toChecksumsFile
, fromChecksumsFile
-- * ForRunFiles abstraction
Expand Down Expand Up @@ -44,20 +44,21 @@ import qualified Database.LSMTree.Internal.CRC32C as CRC
newtype RunFsPaths = RunFsPaths { runNumber :: Int }
deriving (Show, NFData)

runFsPaths :: RunFsPaths -> ForRunFiles FsPath
runFsPaths fsPaths = fmap (runFilePathWithExt fsPaths) runFileExts
-- | Paths to all files associated with this run, except 'runChecksumsPath'.
pathsForRunFiles :: RunFsPaths -> ForRunFiles FsPath
pathsForRunFiles fsPaths = fmap (runFilePathWithExt fsPaths) runFileExts

runKOpsPath :: RunFsPaths -> FsPath
runKOpsPath = forRunKOps . runFsPaths
runKOpsPath = forRunKOps . pathsForRunFiles

runBlobPath :: RunFsPaths -> FsPath
runBlobPath = forRunBlob . runFsPaths
runBlobPath = forRunBlob . pathsForRunFiles

runFilterPath :: RunFsPaths -> FsPath
runFilterPath = forRunFilter . runFsPaths
runFilterPath = forRunFilter . pathsForRunFiles

runIndexPath :: RunFsPaths -> FsPath
runIndexPath = forRunIndex . runFsPaths
runIndexPath = forRunIndex . pathsForRunFiles

runChecksumsPath :: RunFsPaths -> FsPath
runChecksumsPath = flip runFilePathWithExt "checksums"
Expand All @@ -81,14 +82,14 @@ runFileExts = ForRunFiles {
Checksums
-------------------------------------------------------------------------------}

runChecksumFileNames :: ForRunFiles CRC.ChecksumsFileName
runChecksumFileNames = fmap (CRC.ChecksumsFileName . BS.pack) runFileExts
checksumFileNamesForRunFiles :: ForRunFiles CRC.ChecksumsFileName
checksumFileNamesForRunFiles = fmap (CRC.ChecksumsFileName . BS.pack) runFileExts

toChecksumsFile :: ForRunFiles CRC.CRC32C -> CRC.ChecksumsFile
toChecksumsFile = Map.fromList . toList . liftA2 (,) runChecksumFileNames
toChecksumsFile = Map.fromList . toList . liftA2 (,) checksumFileNamesForRunFiles

fromChecksumsFile :: CRC.ChecksumsFile -> Either String (ForRunFiles CRC.CRC32C)
fromChecksumsFile file = for runChecksumFileNames $ \name ->
fromChecksumsFile file = for checksumFileNamesForRunFiles $ \name ->
case Map.lookup name file of
Just crc -> Right crc
Nothing -> Left ("key not found: " <> show name)
Expand Down
3 changes: 2 additions & 1 deletion test/Test/Database/LSMTree/Internal/Lookup.hs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import Database.LSMTree.Internal.RawOverflowPage
import Database.LSMTree.Internal.RawPage
import qualified Database.LSMTree.Internal.Run as Run
import Database.LSMTree.Internal.RunAcc as Run
import Database.LSMTree.Internal.RunFsPaths (RunFsPaths (..))
import Database.LSMTree.Internal.Serialise
import Database.LSMTree.Internal.Serialise.Class
import qualified Database.LSMTree.Internal.WriteBuffer as WB
Expand Down Expand Up @@ -301,7 +302,7 @@ prop_roundtripFromWriteBufferLookupIO dats =
pure $ opaqueifyBlobs (V.fromList model) === opaqueifyBlobs (V.zip (V.fromList lookupss) real)
where
mkRuns hasFS = first V.fromList . unzip <$> sequence [
(,wb) <$> Run.fromWriteBuffer hasFS (Run.RunFsPaths i) wb
(,wb) <$> Run.fromWriteBuffer hasFS (RunFsPaths i) wb
| (i, dat) <- zip [0..] (getSmallList dats)
, let wb = WB.WB (runData dat)
]
Expand Down
14 changes: 8 additions & 6 deletions test/Test/Database/LSMTree/Internal/Merge.hs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import Database.LSMTree.Extras.Generators (KeyForIndexCompact,
import qualified Database.LSMTree.Internal.Entry as Entry
import qualified Database.LSMTree.Internal.Merge as Merge
import qualified Database.LSMTree.Internal.Run as Run
import Database.LSMTree.Internal.RunFsPaths (RunFsPaths (..),
pathsForRunFiles)
import Database.LSMTree.Internal.Serialise
import Database.LSMTree.Internal.WriteBuffer (WriteBuffer)
import qualified Database.LSMTree.Internal.WriteBuffer as WB
Expand Down Expand Up @@ -81,7 +83,7 @@ prop_MergeDistributes fs level stepSize (fmap unTypedWriteBuffer -> wbs) = do
.&&. lhsKOpsFile === rhsKOpsFile
.&&. lhsBlobFile === rhsBlobFile
where
flush n = Run.fromWriteBuffer fs (Run.RunFsPaths n)
flush n = Run.fromWriteBuffer fs (RunFsPaths n)

stats = tabulate "value size" (map (showPowersOf10 . sizeofValue) vals)
. label (if any isLargeKOp kops then "has large k/op" else "no large k/op")
Expand All @@ -98,12 +100,12 @@ prop_CloseMerge ::
[TypedWriteBuffer KeyForIndexCompact SerialisedValue SerialisedBlob] ->
IO Property
prop_CloseMerge fs level (Positive stepSize) (fmap unTypedWriteBuffer -> wbs) = do
let path0 = Run.RunFsPaths 0
let path0 = RunFsPaths 0
runs <- sequenceA $ zipWith flush [10..] wbs
mergeToClose <- makeInProgressMerge path0 runs
traverse_ (Merge.close fs) mergeToClose

filesExist <- traverse (FS.doesFileExist fs) (Run.runFsPaths path0)
filesExist <- traverse (FS.doesFileExist fs) (pathsForRunFiles path0)

-- cleanup
traverse_ (Run.removeReference fs) runs
Expand All @@ -112,7 +114,7 @@ prop_CloseMerge fs level (Positive stepSize) (fmap unTypedWriteBuffer -> wbs) =
counterexample ("run files exist: " <> show filesExist) $
isJust mergeToClose ==> all not filesExist
where
flush n = Run.fromWriteBuffer fs (Run.RunFsPaths n)
flush n = Run.fromWriteBuffer fs (RunFsPaths n)

makeInProgressMerge path runs =
Merge.new fs level mappendValues path runs >>= \case
Expand Down Expand Up @@ -140,8 +142,8 @@ mergeRuns ::
StepSize ->
IO (Run.Run (FS.Handle h))
mergeRuns fs level n runs (Positive stepSize) = do
Merge.new fs level mappendValues (Run.RunFsPaths n) runs >>= \case
Nothing -> Run.fromWriteBuffer fs (Run.RunFsPaths n) WB.empty
Merge.new fs level mappendValues (RunFsPaths n) runs >>= \case
Nothing -> Run.fromWriteBuffer fs (RunFsPaths n) WB.empty
Just m -> go m
where
go m =
Expand Down
1 change: 1 addition & 0 deletions test/Test/Database/LSMTree/Internal/Run.hs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import Database.LSMTree.Internal.RawOverflowPage
(rawOverflowPageRawBytes)
import Database.LSMTree.Internal.RawPage
import Database.LSMTree.Internal.Run
import Database.LSMTree.Internal.RunFsPaths (RunFsPaths (..))
import qualified Database.LSMTree.Internal.RunReader as Reader
import Database.LSMTree.Internal.Serialise
import qualified Database.LSMTree.Internal.WriteBuffer as WB
Expand Down

0 comments on commit a682e2e

Please sign in to comment.