Skip to content

Commit

Permalink
Merge branch 'master' into SCP-1751-keep-simulation-state
Browse files Browse the repository at this point in the history
  • Loading branch information
sjoerdvisscher committed Oct 12, 2021
2 parents 1b5e1e8 + 1f20481 commit c1cdefe
Show file tree
Hide file tree
Showing 128 changed files with 12,857 additions and 8,947 deletions.
39 changes: 34 additions & 5 deletions freer-extras/freer-extras.cabal
Expand Up @@ -30,21 +30,50 @@ library
hs-source-dirs: src
exposed-modules:
Control.Monad.Freer.Extras
Control.Monad.Freer.Extras.Modify
Control.Monad.Freer.Extras.Beam
Control.Monad.Freer.Extras.Log
Control.Monad.Freer.Extras.Stream
Control.Monad.Freer.Extras.Modify
Control.Monad.Freer.Extras.Pagination
Control.Monad.Freer.Extras.State
Control.Monad.Freer.Extras.Beam
Control.Monad.Freer.Extras.Stream
build-depends:
aeson -any,
base >=4.7 && <5,
lens -any,
beam-core -any,
beam-sqlite -any,
containers -any,
data-default -any,
freer-simple -any,
iohk-monitoring -any,
lens -any,
mtl -any,
openapi3 -any,
prettyprinter -any,
sqlite-simple -any,
streaming -any,
text -any,

test-suite freer-extras-test
import: lang
type: exitcode-stdio-1.0
main-is: Spec.hs
hs-source-dirs: test
other-modules:
Control.Monad.Freer.Extras.BeamSpec
Control.Monad.Freer.Extras.PaginationSpec
build-depends:
hedgehog -any,
tasty -any,
tasty-hedgehog -any,
build-depends:
base >=4.7 && <5,
beam-core -any,
beam-migrate -any,
beam-sqlite -any,
containers -any,
contra-tracer -any,
freer-extras -any,
freer-simple -any,
lens -any,
semigroups -any,
sqlite-simple -any,
iohk-monitoring -any,
55 changes: 49 additions & 6 deletions freer-extras/src/Control/Monad/Freer/Extras/Beam.hs
Expand Up @@ -22,29 +22,39 @@ import Cardano.BM.Data.Tracer (ToObject (..))
import Cardano.BM.Trace (Trace, logDebug)
import Control.Concurrent (threadDelay)
import Control.Exception (try)
import Control.Monad (guard)
import Control.Monad.Freer (Eff, LastMember, Member, type (~>))
import Control.Monad.Freer.Error (Error, throwError)
import Control.Monad.Freer.Extras.Pagination (Page (..), PageQuery (..), PageSize (..))
import Control.Monad.Freer.Reader (Reader, ask)
import Control.Monad.Freer.TH (makeEffect)
import Data.Aeson (FromJSON, ToJSON)
import Data.Foldable (traverse_)
import qualified Data.List.NonEmpty as L
import Data.Maybe (isJust, listToMaybe)
import Data.Text (Text)
import qualified Data.Text as Text
import Database.Beam (Beamable, DatabaseEntity, FromBackendRow, Identity,
MonadIO (liftIO), SqlDelete, SqlInsert, SqlSelect, SqlUpdate,
TableEntity, insertValues, runDelete, runInsert,
runSelectReturningList, runSelectReturningOne, runUpdate)
import Database.Beam.Backend.SQL (BeamSqlBackendCanSerialize)
MonadIO (liftIO), Q, QBaseScope, QExpr, SqlDelete, SqlInsert,
SqlSelect, SqlUpdate, TableEntity, asc_, filter_,
insertValues, limit_, orderBy_, runDelete, runInsert,
runSelectReturningList, runSelectReturningOne, runUpdate,
select, val_, (>.))
import Database.Beam.Backend.SQL (BeamSqlBackendCanSerialize, HasSqlValueSyntax)
import Database.Beam.Backend.SQL.BeamExtensions (BeamHasInsertOnConflict (anyConflict, insertOnConflict, onConflictDoNothing))
import Database.Beam.Query.Internal (QNested)
import Database.Beam.Schema.Tables (FieldsFulfillConstraint)
import Database.Beam.Sqlite (Sqlite, SqliteM, runBeamSqliteDebug)
import Database.Beam.Sqlite.Syntax (SqliteValueSyntax)
import qualified Database.SQLite.Simple as Sqlite
import GHC.Generics (Generic)
import Prettyprinter (Pretty (..), colon, (<+>))

type BeamableSqlite table = (Beamable table, FieldsFulfillConstraint (BeamSqlBackendCanSerialize Sqlite) table)

data BeamError =
type BeamThreadingArg = QNested (QNested QBaseScope)

newtype BeamError =
SqlError Text
deriving stock (Eq, Show, Generic)
deriving anyclass (FromJSON, ToJSON, ToObject)
Expand All @@ -53,7 +63,7 @@ instance Pretty BeamError where
pretty = \case
SqlError s -> "SqlError (via Beam)" <> colon <+> pretty s

data BeamLog =
newtype BeamLog =
SqlLog String
deriving stock (Eq, Show, Generic)
deriving anyclass (FromJSON, ToJSON, ToObject)
Expand Down Expand Up @@ -92,6 +102,13 @@ data BeamEffect r where
=> SqlSelect Sqlite a
-> BeamEffect [a]

-- | Select using Seek Pagination.
SelectPage
:: (FromBackendRow Sqlite a, HasSqlValueSyntax SqliteValueSyntax a)
=> PageQuery a
-> Q Sqlite db BeamThreadingArg (QExpr Sqlite BeamThreadingArg a)
-> BeamEffect (Page a)

SelectOne
:: FromBackendRow Sqlite a
=> SqlSelect Sqlite a
Expand Down Expand Up @@ -122,6 +139,32 @@ handleBeam trace eff = runBeam trace $ execute eff
UpdateRows q -> runUpdate q
DeleteRows q -> runDelete q
SelectList q -> runSelectReturningList q
SelectPage pageQuery@PageQuery { pageQuerySize = PageSize ps, pageQueryLastItem } q -> do
let ps' = fromIntegral ps

-- Fetch the first @PageSize + 1@ elements after the last query
-- element. The @+1@ allows to us to know if there is a next page
-- or not.
items <- runSelectReturningList
$ select
$ limit_ (ps' + 1)
$ orderBy_ asc_
$ filter_ (\qExpr -> maybe (val_ True)
(\lastItem -> qExpr >. val_ lastItem)
pageQueryLastItem
) q

let lastItemM = guard (length items > fromIntegral ps)
>> L.nonEmpty items
>>= listToMaybe . L.tail . L.reverse
let newPageQuery = fmap (PageQuery (PageSize ps) . Just) lastItemM

pure $
Page
{ currentPageQuery = pageQuery
, nextPageQuery = newPageQuery
, pageItems = if isJust lastItemM then init items else items
}
SelectOne q -> runSelectReturningOne q
Combined effs -> traverse_ execute effs

Expand Down
107 changes: 107 additions & 0 deletions freer-extras/src/Control/Monad/Freer/Extras/Pagination.hs
@@ -0,0 +1,107 @@
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DerivingStrategies #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE QuantifiedConstraints #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE UndecidableInstances #-}
{-
Pagination allows to return a subset of results through pages. Once the first
page was requested, we can request the next ones until we get empty results.
There are multiple strategies for implementation pagination, such as Offset
Pagination and Seek Pagination. Offset Pagination is the easiest to implement
and use. However, it is not performant for large offset values and it is not
consistent if new items are inserted in the database while we are querying.
For these reasons, we decided to use Seek Pagination which doesn't suffer from
those drawbacks. Seek Pagination works as follows. For a given page request, we
need to provide the number of items per page and last element we queried (can
be Nothing). We suppose the elements are ordered in ascending order.
Here's a simple illustrative example:
* Suppose we have the following items in the database [1..100].
* We first request the 50 first items.
* We obtain the first page containing [1..50].
* To request the next page, we request 50 items after the last item of the
previous page (which is 50).
* We obtain the second page containing [51..100].
* Since we don't know this was the last page, we would request the next 50
items after the last item (which is 100).
* We obtain a page with no elements, thus we don't need to query for more pages.
-}
module Control.Monad.Freer.Extras.Pagination
( PageQuery(..)
, Page(..)
, PageSize(..)
, pageOf
) where

import Control.Monad (guard)
import Data.Aeson (FromJSON, ToJSON)
import Data.Default (Default (..))
import qualified Data.List.NonEmpty as L
import Data.Maybe (isJust, listToMaybe)
import qualified Data.OpenApi as OpenApi
import Data.Set (Set)
import qualified Data.Set as Set
import GHC.Generics (Generic)
import Numeric.Natural (Natural)

-- | Query parameters for pagination.
data PageQuery a = PageQuery
{ pageQuerySize :: PageSize -- ^ Number of items per page.
, pageQueryLastItem :: Maybe a -- ^ Last item of the queried page.
}
deriving stock (Eq, Ord, Show, Generic, Functor)
deriving anyclass (ToJSON, FromJSON, OpenApi.ToSchema)

instance Default (PageQuery a) where
def = PageQuery def Nothing

newtype PageSize = PageSize { getPageSize :: Natural }
deriving stock (Eq, Ord, Show, Generic)
deriving anyclass (ToJSON, FromJSON, OpenApi.ToSchema)
deriving newtype (Num)

instance Default PageSize where
def = PageSize 50

-- | Part of a collection.
data Page a = Page
{ currentPageQuery :: PageQuery a
-- ^ The 'PageQuery' which was used to request this 'Page'.
, nextPageQuery :: Maybe (PageQuery a)
-- ^ The 'PageQuery' to use to request the next 'Page'. Nothing if we requested the last page.
, pageItems :: [a]
-- ^ Items in the current 'Page'.
}
deriving stock (Eq, Ord, Show, Generic, Functor)
deriving anyclass (ToJSON, FromJSON)

-- | Given a 'Set', request the 'Page' with the given 'PageQuery'.
pageOf
:: (Eq a)
=> PageQuery a -- ^ Pagination query parameters.
-> Set a
-> Page a
pageOf pageQuery@PageQuery { pageQuerySize = PageSize ps, pageQueryLastItem } items =
let ps' = fromIntegral ps
-- The extract the @PageSize + 1@ next elements after the last query
-- element. The @+1@ allows to now if there is a next page or not.
pageItems = case pageQueryLastItem of
Nothing -> take (ps' + 1) $ Set.toList items
Just i -> take (ps' + 1) $ drop 1 $ dropWhile ((/=) i) $ Set.toList items

nextLastItem = guard (length items > fromIntegral ps)
>> L.nonEmpty pageItems
>>= listToMaybe . L.tail . L.reverse
in Page
{ currentPageQuery = pageQuery
, nextPageQuery = fmap (PageQuery (PageSize ps) . Just) nextLastItem
, pageItems = if isJust nextLastItem then init pageItems else pageItems
}

0 comments on commit c1cdefe

Please sign in to comment.