[CAD-779] Add a database backend.

.gitignore

@@ -2,3 +2,4 @@
 dist-newstyle/
 *~
 tags
+result*

README.md

@@ -90,3 +90,69 @@ You need an HTTP server to serve it from and simply point to the application por
 Run the application, go to the local port http://localhost:3000/swagger.json and copy the content into https://editor.swagger.io/
 Voila! You got it, the spec is there.
 
+## How to run
+
+### Create DB
+
+You first need to create the database. You can provide your own path, the example will use the default location. We need the PostgreSQL database and we create it with:
+```
+PGPASSFILE=config/pgpass ./scripts/postgresql-setup.sh --createdb
+```
+Or if it needs to be recreated:
+```
+PGPASSFILE=config/pgpass ./scripts/postgresql-setup.sh --recreatedb
+```
+
+After that we need to run the migrations (if there are any):
+```
+PGPASSFILE=config/pgpass stack run smash-exe -- run-migrations --mdir ./schema
+```
+
+And after that we can run additional migration scripts if they need to be created:
+```
+PGPASSFILE=config/pgpass stack run smash-exe -- create-migration --mdir ./schema
+```
+
+To show all tables:
+```
+\dt
+```
+
+To show details about specific table:
+```
+\d+ TABLE_NAME
+```
+
+For example:
+```
+\d+ block
+```
+
+Dumping the schema:
+```
+pg_dump -c -s --no-owner cexplorer > cexplorer.sql
+```
+
+## Inserting pool metadata
+
+
+This is an example (we got the hash from Blake2 256):
+```
+stack exec smash-exe -- insert-pool --filepath test_pool.json --poolhash "cbdfc4f21feb0a414b2b9471fa56b0ebd312825e63db776d68cc3fa0ca1f5a2f"
+```
+
+## Test script
+
+An example of how the whole thing works.
+```
+PGPASSFILE=config/pgpass ./scripts/postgresql-setup.sh --recreatedb
+PGPASSFILE=config/pgpass stack run smash-exe -- run-migrations --mdir ./schema
+PGPASSFILE=config/pgpass stack run smash-exe -- create-migration --mdir ./schema
+PGPASSFILE=config/pgpass stack run smash-exe -- run-migrations --mdir ./schema
+
+PGPASSFILE=config/pgpass stack run smash-exe -- insert-pool --filepath test_pool.json --poolhash "cbdfc4f21feb0a414b2b9471fa56b0ebd312825e63db776d68cc3fa0ca1f5a2f"
+
+PGPASSFILE=config/pgpass stack run smash-exe -- run-app
+```
+
+After the server is running, you can check the hash on http://localhost:3100/api/v1/metadata/cbdfc4f21feb0a414b2b9471fa56b0ebd312825e63db776d68cc3fa0ca1f5a2f to see it return the JSON metadata.

app/Main.hs

@@ -4,5 +4,135 @@ import           Cardano.Prelude
 
 import           Lib
 
+import           DB
+
+import           Control.Applicative (optional)
+
+import           Data.Monoid ((<>))
+
+import           Options.Applicative (Parser, ParserInfo, ParserPrefs)
+import qualified Options.Applicative as Opt
+
+
 main :: IO ()
-main = runApp defaultConfiguration
+main = do
+    Opt.customExecParser p opts >>= runCommand
+  where
+    opts :: ParserInfo Command
+    opts = Opt.info (Opt.helper <*> pVersion <*> pCommand)
+      ( Opt.fullDesc
+      <> Opt.header "SMASH - Manage the Stakepool Metadata Aggregation Server"
+      )
+
+    p :: ParserPrefs
+    p = Opt.prefs Opt.showHelpOnEmpty
+
+-- -----------------------------------------------------------------------------
+
+data Command
+  = CreateMigration MigrationDir
+  | RunMigrations MigrationDir (Maybe LogFileDir)
+  | RunApplication
+  | InsertPool FilePath Text
+
+runCommand :: Command -> IO ()
+runCommand cmd =
+  case cmd of
+    CreateMigration mdir -> doCreateMigration mdir
+    RunMigrations mdir mldir -> runMigrations (\pgConfig -> pgConfig) False mdir mldir
+    RunApplication -> runApp defaultConfiguration
+    InsertPool poolMetadataJsonPath poolHash -> do
+        putTextLn "Inserting pool metadata!"
+        result <- runPoolInsertion poolMetadataJsonPath poolHash
+        either
+            (\err -> putTextLn $ "Error occured. " <> renderLookupFail err)
+            (\_ -> putTextLn "Insertion completed!")
+            result
+
+doCreateMigration :: MigrationDir -> IO ()
+doCreateMigration mdir = do
+  mfp <- createMigration mdir
+  case mfp of
+    Nothing -> putTextLn "No migration needed."
+    Just fp -> putTextLn $ toS ("New migration '" ++ fp ++ "' created.")
+
+-------------------------------------------------------------------------------
+
+pVersion :: Parser (a -> a)
+pVersion =
+  Opt.infoOption "cardano-db-tool version 0.1.0.0"
+    (  Opt.long "version"
+    <> Opt.short 'v'
+    <> Opt.help "Print the version and exit"
+    )
+
+pCommand :: Parser Command
+pCommand =
+  Opt.subparser
+    ( Opt.command "create-migration"
+        ( Opt.info pCreateMigration
+          $ Opt.progDesc "Create a database migration (only really used by devs)."
+          )
+    <> Opt.command "run-migrations"
+        ( Opt.info pRunMigrations
+          $ Opt.progDesc "Run the database migrations (which are idempotent)."
+          )
+    <> Opt.command "run-app"
+        ( Opt.info pRunApp
+          $ Opt.progDesc "Run the actual application."
+          )
+    <> Opt.command "insert-pool"
+        ( Opt.info pInsertPool
+          $ Opt.progDesc "Inserts the pool into the database (utility)."
+          )
+    )
+  where
+    pCreateMigration :: Parser Command
+    pCreateMigration =
+      CreateMigration <$> pMigrationDir
+
+    pRunMigrations :: Parser Command
+    pRunMigrations =
+      RunMigrations <$> pMigrationDir <*> optional pLogFileDir
+
+    -- Empty right now but we might add some params over time. Like ports and stuff?
+    pRunApp :: Parser Command
+    pRunApp =
+      pure RunApplication
+
+    -- Empty right now but we might add some params over time.
+    pInsertPool :: Parser Command
+    pInsertPool =
+      InsertPool <$> pFilePath <*> pPoolHash
+
+pFilePath :: Parser FilePath
+pFilePath =
+  Opt.strOption
+    (  Opt.long "filepath"
+    <> Opt.help "The JSON metadata filepath location."
+    <> Opt.completer (Opt.bashCompleter "directory")
+    )
+
+pPoolHash :: Parser Text
+pPoolHash =
+  Opt.strOption
+    (  Opt.long "poolhash"
+    <> Opt.help "The JSON metadata Blake2 256 hash."
+    )
+
+pMigrationDir :: Parser MigrationDir
+pMigrationDir =
+  MigrationDir <$> Opt.strOption
+    (  Opt.long "mdir"
+    <> Opt.help "The directory containing the migrations."
+    <> Opt.completer (Opt.bashCompleter "directory")
+    )
+
+pLogFileDir :: Parser LogFileDir
+pLogFileDir =
+  LogFileDir <$> Opt.strOption
+    (  Opt.long "ldir"
+    <> Opt.help "The directory to write the log to."
+    <> Opt.completer (Opt.bashCompleter "directory")
+    )
+

cabal.project

@@ -1,21 +1,118 @@
-index-state: 2020-01-31T00:00:00Z
+index-state: 2020-04-01T00:00:00Z
 
 packages:
-    .
+    ./.
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-prelude
-  tag: fe76ec64f6b45259cc407a6d840dad79ee6063b6
-  --sha256: 1w39806djx7vmps47vvc72i20w4dkam3l8687kkw5ih1isvl671x
+  tag: e0257be9d745a04f85ab8287a48a9c193acafec8
+  --sha256: 09cxfa0vzny8xhjskc6khrfwsnajjawas6j574i2qfd71yd8mrrg
 
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-prelude
-  tag: fe76ec64f6b45259cc407a6d840dad79ee6063b6
-  --sha256: 1w39806djx7vmps47vvc72i20w4dkam3l8687kkw5ih1isvl671x
+  tag: e0257be9d745a04f85ab8287a48a9c193acafec8
+  --sha256: 09cxfa0vzny8xhjskc6khrfwsnajjawas6j574i2qfd71yd8mrrg
   subdir: test
 
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: contra-tracer
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: iohk-monitoring
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: plugins/backend-aggregation
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: plugins/backend-ekg
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: plugins/backend-monitoring
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: plugins/backend-trace-forwarder
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: plugins/scribe-systemd
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/iohk-monitoring-framework
+  tag: 71df15f7b888d2671f619cc632080aaaaca48087
+  --sha256: 02shwi4qppf88biq2y9amfzkzif9j0fdj0d77zdjrfjacldvc6ci
+  subdir: tracer-transformers
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/ouroboros-network
+  tag: 16bca08140fb37746538edff9fe77220acf91d55
+  --sha256: 1m6z6qfy6y5rvlj0isrzf3wcm1vsmlzvhmn988dp14r6fycljvca
+  subdir: Win32-network
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-crypto
+  tag: 2547ad1e80aeabca2899951601079408becbc92c
+  --sha256: 1p2kg2w02q5w1cvqzhfhqmxviy4xrzada3mmb096j2n6hfr20kri
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: 4a457f44e68132ce2bd978ab45a3188e64327abc
+  --sha256: 1yzbhkil119hkcb36ykwrchxk3zqmshpp57an17zjjqh7n2nw392
+  subdir: binary
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: 4a457f44e68132ce2bd978ab45a3188e64327abc
+  --sha256: 1yzbhkil119hkcb36ykwrchxk3zqmshpp57an17zjjqh7n2nw392
+  subdir: binary/test
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: 4a457f44e68132ce2bd978ab45a3188e64327abc
+  --sha256: 1yzbhkil119hkcb36ykwrchxk3zqmshpp57an17zjjqh7n2nw392
+  subdir: cardano-crypto-class
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: 4a457f44e68132ce2bd978ab45a3188e64327abc
+  --sha256: 1yzbhkil119hkcb36ykwrchxk3zqmshpp57an17zjjqh7n2nw392
+  subdir: slotting
+
 constraints:
   ip < 1.5,
   hedgehog >= 1.0,
@@ -27,3 +124,15 @@ allow-newer: katip:Win32
 
 package comonad
   flags: -test-doctests
+
+package cardano-prelude
+  tests: False
+
+package iohk-monitoring
+  tests: False
+
+package cardano-binary
+  tests: False
+
+package cardano-crypto-class
+  tests: False

config/pgpass

@@ -0,0 +1 @@
+/var/run/postgresql:5432:smash:*:*

default.nix

@@ -35,6 +35,10 @@ let
       tests = collectChecks haskellPackages;
     };
 
+    nixosTests = import ./nix/nixos/tests {
+      inherit pkgs;
+    };
+
     shell = import ./shell.nix {
       inherit pkgs;
       withHoogle = true;

nix/default.nix

@@ -6,24 +6,23 @@
 let
   sources = import ./sources.nix { inherit pkgs; }
     // sourcesOverride;
-  iohKNix = import sources.iohk-nix {};
+  iohkNix = import sources.iohk-nix {};
   haskellNix = import sources."haskell.nix";
   # use our own nixpkgs if it exists in our sources,
   # otherwise use iohkNix default nixpkgs.
   nixpkgs = if (sources ? nixpkgs)
     then (builtins.trace "Not using IOHK default nixpkgs (use 'niv drop nixpkgs' to use default for better sharing)"
       sources.nixpkgs)
-    else (builtins.trace "Using IOHK default nixpkgs"
-      iohKNix.nixpkgs);
+    else iohkNix.nixpkgs;
 
   # for inclusion in pkgs:
   overlays =
     # Haskell.nix (https://github.com/input-output-hk/haskell.nix)
     haskellNix.overlays
     # haskell-nix.haskellLib.extra: some useful extra utility functions for haskell.nix
-    ++ iohKNix.overlays.haskell-nix-extra
+    ++ iohkNix.overlays.haskell-nix-extra
     # iohkNix: nix utilities and niv:
-    ++ iohKNix.overlays.iohkNix
+    ++ iohkNix.overlays.iohkNix
     # our own overlays:
     ++ [
       (pkgs: _: with pkgs; {