Skip to content

Commit

Permalink
re-org moconi/marconi-mamba test folders
Browse files Browse the repository at this point in the history
Moved various property-check generators under test-helper sub-project.
Re org the test test modules to match the corresponding module name
under test
  • Loading branch information
kayvank committed Feb 6, 2023
1 parent 1cba31f commit 6882b98
Show file tree
Hide file tree
Showing 21 changed files with 1,010 additions and 317 deletions.
4 changes: 2 additions & 2 deletions marconi-mamba/examples/json-rpc-server/src/Main.hs
Expand Up @@ -15,8 +15,8 @@ import Control.Concurrent.STM (atomically)
import Control.Lens.Operators ((^.))
import Options.Applicative (Parser, execParser, help, helper, info, long, metavar, short, strOption, (<**>))

import Marconi.Api.Index.Utxo (writeTMVar')
import Marconi.Api.Types (UtxoIndexerEnv, queryEnv, uiIndexer)
import Marconi.Api.UtxoIndexersQuery qualified as UIQ
import Marconi.Bootstrap (bootstrapHttp, bootstrapJsonRpc)
import Marconi.CLI (multiString)
import Marconi.Index.Utxo qualified as Utxo
Expand Down Expand Up @@ -56,5 +56,5 @@ mocUtxoIndexer dbpath env =
Utxo.open dbpath (Utxo.Depth 4) >>= callback >> innerLoop
where
callback :: Utxo.UtxoIndexer -> IO ()
callback = atomically . UIQ.writeTMVar' (env ^. uiIndexer)
callback = atomically . writeTMVar' (env ^. uiIndexer)
innerLoop = threadDelay 1000000 >> innerLoop -- create some latency
16 changes: 10 additions & 6 deletions marconi-mamba/marconi-mamba.cabal
Expand Up @@ -45,9 +45,9 @@ library
hs-source-dirs: src
exposed-modules:
Marconi.Api.HttpServer
Marconi.Api.Index.Utxo
Marconi.Api.Routes
Marconi.Api.Types
Marconi.Api.UtxoIndexersQuery
Marconi.Bootstrap
Marconi.MambaCli

Expand Down Expand Up @@ -134,7 +134,7 @@ executable examples-json-rpc-client
--------------------
build-depends: marconi:{marconi, json-rpc}

------------------------
--------------
-- Non-IOG dependencies
------------------------
build-depends:
Expand Down Expand Up @@ -187,22 +187,26 @@ test-suite marconi-mamba-test
type: exitcode-stdio-1.0
main-is: Spec.hs
hs-source-dirs: test
other-modules: Spec.UtxoIndexersQuery
other-modules: Test.Marconi.Api.Index.Utxo

--------------------------
-- Other IOG dependencies
--------------------------
build-depends: cardano-api:{cardano-api, gen}

--------------------
-- Local components
--------------------
build-depends:
, marconi
, marconi-mamba
, marconi:{marconi, marconi-test-lib}
, rewindable-index

------------------------
-- Non-IOG dependencies
------------------------
build-depends:
, base >=4.9 && <5
, cardano-api:{cardano-api, gen}
, base >=4.9 && <5
, containers
, hedgehog
, lens
Expand Down
19 changes: 11 additions & 8 deletions marconi-mamba/src/Marconi/Api/HttpServer.hs
Expand Up @@ -5,9 +5,10 @@
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}

module Marconi.Api.HttpServer(
bootstrap
) where
module Marconi.Api.HttpServer
( bootstrap
, marconiApp
) where

import Control.Lens ((^.))
import Control.Monad.IO.Class (liftIO)
Expand All @@ -17,21 +18,23 @@ import Data.Text (Text, pack)
import Data.Time (defaultTimeLocale, formatTime, getCurrentTime)
import Network.Wai.Handler.Warp (runSettings)
import Servant.API (NoContent (NoContent), (:<|>) ((:<|>)))
import Servant.Server (Handler, Server, serve)
import Servant.Server (Application, Handler, Server, serve)

import Cardano.Api ()
import Marconi.Api.Index.Utxo qualified as Q.Utxo
import Marconi.Api.Routes (API)
import Marconi.Api.Types (HasJsonRpcEnv (httpSettings, queryEnv), JsonRpcEnv, QueryExceptions, UtxoIndexerEnv,
UtxoReport)
import Marconi.Api.UtxoIndexersQuery qualified as Q.Utxo
import Marconi.JsonRpc.Types (JsonRpcErr (JsonRpcErr, errorCode, errorData, errorMessage), parseErrorCode)
import Marconi.Server.Types ()

-- | bootstraps the he http server
bootstrap :: JsonRpcEnv -> IO ()
bootstrap env = runSettings
(env ^. httpSettings)
(serve (Proxy @API) (server (env ^. queryEnv ) ) )
bootstrap env = runSettings (env ^. httpSettings) $ marconiApp (env ^. queryEnv)

-- | Implement marconi API and pruduce the Wai Application
marconiApp :: UtxoIndexerEnv -> Application
marconiApp env = serve (Proxy @API) (server env)

server
:: UtxoIndexerEnv -- ^ Utxo Environment to access Utxo Storage running on the marconi thread
Expand Down
136 changes: 136 additions & 0 deletions marconi-mamba/src/Marconi/Api/Index/Utxo.hs
@@ -0,0 +1,136 @@
module Marconi.Api.Index.Utxo
( bootstrap
, findByCardanoAddress
, findByAddress
, findAll
, reportQueryAddresses
, Utxo.UtxoRow(..)
, Utxo.UtxoIndexer
, reportQueryCardanoAddresses
, reportBech32Addresses
, withQueryAction
, writeTMVar
, writeTMVar'
) where
import Control.Concurrent.Async (forConcurrently)
import Control.Concurrent.STM (atomically)
import Control.Concurrent.STM.TMVar (TMVar, newEmptyTMVar, putTMVar, takeTMVar, tryTakeTMVar)
import Control.Exception (bracket)
import Control.Lens ((^.))
import Control.Monad.STM (STM)
import Data.Functor ((<&>))
import Data.List.NonEmpty qualified as NonEmpty
import Data.Text (Text, intercalate, pack, unpack)

import Cardano.Api qualified as C
import Marconi.Api.Types (HasUtxoIndexerEnv (uiIndexer, uiQaddresses),
QueryExceptions (AddressNotInListError, QueryError),
UtxoIndexerEnv (UtxoIndexerEnv, _uiIndexer, _uiQaddresses),
UtxoIndexerWrapper (UtxoIndexerWrapper, unWrap), UtxoReport (UtxoReport))
import Marconi.Index.Utxo qualified as Utxo
import Marconi.Types (TargetAddresses)
import RewindableIndex.Storable qualified as Storable

-- | Bootstraps the utxo query environment.
-- The module is responsible for accessing SQLite for quries.
-- The main issue we try to avoid here is mixing inserts and quries in SQLite to avoid locking the database
bootstrap
:: TargetAddresses -- ^ user provided target addresses
-> IO UtxoIndexerEnv -- ^ returns Query runtime environment
bootstrap targetAddresses = do
ix <- atomically (newEmptyTMVar :: STM (TMVar Utxo.UtxoIndexer) )
pure $ UtxoIndexerEnv
{ _uiIndexer = UtxoIndexerWrapper ix
, _uiQaddresses = targetAddresses
}
-- | finds reports for all user-provided addresses.
-- TODO consider sqlite streaming, https://hackage.haskell.org/package/sqlite-simple-0.4.18.2/docs/Database-SQLite-Simple.html#g:14
--
findAll
:: UtxoIndexerEnv -- ^ Query run time environment
-> IO [UtxoReport] -- ^ set of corresponding TxOutRefs
findAll env = forConcurrently addresses f
where
addresses = NonEmpty.toList (env ^. uiQaddresses)
f :: C.Address C.ShelleyAddr -> IO UtxoReport
f addr = (findByCardanoAddress env . C.toAddressAny $ addr) <&> UtxoReport (pack . show $ addr)

-- | Query utxos by Cardano Address
-- To Cardano error may occure
findByCardanoAddress
:: UtxoIndexerEnv -- ^ Query run time environment
-> C.AddressAny -- ^ Cardano address to query
-> IO [Utxo.UtxoRow]
findByCardanoAddress = withQueryAction

-- | Retrieve a Set of TxOutRefs associated with the given Cardano Era address
-- We return an empty Set if no address is found
findByAddress
:: UtxoIndexerEnv -- ^ Query run time environment
-> Text -- ^ Bech32 Address
-> IO (Either QueryExceptions UtxoReport) -- ^ To Plutus address conversion error may occure
findByAddress env addressText =
let
f :: Either C.Bech32DecodeError (C.Address C.ShelleyAddr) -> IO (Either QueryExceptions UtxoReport)
f (Right address)
| address `elem` (env ^. uiQaddresses) = -- allow for targetAddress search only
(pure . C.toAddressAny $ address)
>>= findByCardanoAddress env
<&> Right . UtxoReport addressText
| otherwise = pure . Left . AddressNotInListError . QueryError $
unpack addressText <> " not in the provided target addresses"
f (Left e) = pure . Left $ QueryError (unpack addressText
<> " generated error: "
<> show e)
in
f $ C.deserialiseFromBech32 C.AsShelleyAddress addressText

-- | Execute the query function
-- We must stop the utxo inserts before doing the query
withQueryAction
:: UtxoIndexerEnv -- ^ Query run time environment
-> C.AddressAny -- ^ Cardano address to query
-> IO [Utxo.UtxoRow]
withQueryAction env address =
let
utxoIndexer = unWrap $ env ^. uiIndexer
action :: Utxo.UtxoIndexer -> IO [Utxo.UtxoRow]
action indexer = do
Utxo.UtxoResult rows <- Storable.query Storable.QEverything indexer (Utxo.UtxoAddress address)
pure rows
in
bracket
(atomically . takeTMVar $ utxoIndexer)
(atomically . putTMVar utxoIndexer)
action

-- | report target addresses
-- Used by JSON-RPC
reportQueryAddresses
:: UtxoIndexerEnv
-> IO [C.Address C.ShelleyAddr]
reportQueryAddresses env
= pure
. NonEmpty.toList
$ (env ^. uiQaddresses )

reportQueryCardanoAddresses
:: UtxoIndexerEnv
-> Text
reportQueryCardanoAddresses = intercalate ", " . reportBech32Addresses

reportBech32Addresses
:: UtxoIndexerEnv
-> [Text]
reportBech32Addresses env
= NonEmpty.toList
. fmap C.serialiseAddress
$ (env ^. uiQaddresses )

-- | Non-blocking write of a new value to a 'TMVar'
-- Puts if empty. Replaces if populated.
writeTMVar :: TMVar a -> a -> STM ()
writeTMVar t new = tryTakeTMVar t >> putTMVar t new

writeTMVar' :: UtxoIndexerWrapper-> Utxo.UtxoIndexer -> STM ()
writeTMVar' (UtxoIndexerWrapper t) = writeTMVar t
6 changes: 3 additions & 3 deletions marconi-mamba/src/Marconi/Api/Types.hs
Expand Up @@ -55,14 +55,14 @@ newtype UtxoIndexerWrapper = UtxoIndexerWrapper

data UtxoIndexerEnv = UtxoIndexerEnv
{ _uiIndexer :: UtxoIndexerWrapper
, _uiQaddresses :: TargetAddresses -- ^ user provided addresses to filter
, _uiQaddresses :: !TargetAddresses -- ^ user provided addresses to filter
}
makeClassy ''UtxoIndexerEnv

-- | JSON-RPC configuration
data JsonRpcEnv = JsonRpcEnv
{ _httpSettings :: Settings -- ^ HTTP server setting
, _queryEnv :: UtxoIndexerEnv -- ^ used for query sqlite
, _queryEnv :: !UtxoIndexerEnv -- ^ used for query sqlite
}
makeClassy ''JsonRpcEnv

Expand Down Expand Up @@ -101,7 +101,7 @@ instance ToJSON Utxo.Utxo where

data UtxoReport = UtxoReport
{ urAddress :: Text
, urReport :: [Utxo.UtxoRow]
, urReport :: ![Utxo.UtxoRow]
} deriving (Eq, Ord, Generic)

instance ToJSON UtxoReport where
Expand Down
2 changes: 1 addition & 1 deletion marconi-mamba/src/Marconi/Bootstrap.hs
Expand Up @@ -19,9 +19,9 @@ import Cardano.BM.Trace (logError)
import Cardano.BM.Tracing (defaultConfigStdout)
import Cardano.Streaming (ChainSyncEventException (NoIntersectionFound), withChainSyncEventStream)
import Marconi.Api.HttpServer qualified as Http
import Marconi.Api.Index.Utxo (UtxoIndexer, bootstrap, writeTMVar')
import Marconi.Api.Types (CliArgs (CliArgs), HasJsonRpcEnv (queryEnv), HasUtxoIndexerEnv (uiIndexer),
JsonRpcEnv (JsonRpcEnv, _httpSettings, _queryEnv), RpcPortNumber)
import Marconi.Api.UtxoIndexersQuery (UtxoIndexer, bootstrap, writeTMVar')
import Marconi.Indexers (mkIndexerStream, startIndexers, utxoWorker)
import Marconi.Types (TargetAddresses)

Expand Down
4 changes: 2 additions & 2 deletions marconi-mamba/test/Spec.hs
Expand Up @@ -2,11 +2,11 @@ module Main (main) where

import Test.Tasty (TestTree, defaultMain, testGroup)

import Spec.UtxoIndexersQuery qualified
import Test.Marconi.Api.Index.Utxo qualified

main :: IO ()
main = defaultMain tests

tests :: TestTree
tests = testGroup "marconi-mamba"
[Spec.UtxoIndexersQuery.tests]
[Test.Marconi.Api.Index.Utxo.tests]
9 changes: 0 additions & 9 deletions marconi-mamba/test/Spec/Golden/Cli/marconi.help

This file was deleted.

20 changes: 0 additions & 20 deletions marconi-mamba/test/Spec/Golden/Cli/marconi___help.help

This file was deleted.

7 changes: 0 additions & 7 deletions marconi-mamba/test/Spec/Golden/Cli/marconi___socket.help

This file was deleted.

0 comments on commit 6882b98

Please sign in to comment.