feat(#1234): add reddis support for storing metadata validation results

.envrc.example

@@ -33,6 +33,8 @@ export NGINX_BASIC_AUTH=<NGINX_BASIC_AUTH>
 # the IP addresses that bypass the basic auth
 export IP_ADDRESS_BYPASSING_BASIC_AUTH1=<IP_ADDRESS_BYPASSING_BASIC_AUTH1>
 export IP_ADDRESS_BYPASSING_BASIC_AUTH2=<IP_ADDRESS_BYPASSING_BASIC_AUTH2>
+# the Redis Password
+export REDIS_PASSWORD=<REDIS_PASSWORD>
 
 # the domain name for the target environment
 case "$ENVIRONMENT" in

.github/backend-config.json

@@ -14,5 +14,16 @@
       "port"     : 5432
   },
   "port" : 8080,
-  "host" : "localhost"
+  "host" : "localhost",
+  "cachedurationseconds": 20,
+  "sentrydsn": "",
+  "sentryenv": "",
+  "metadatavalidationhost": "http://metadata-validation",
+  "metadatavalidationport": "3000",
+  "metadatavalidationmaxconcurrentrequests": 10,
+  "redisconfig" : {
+      "host"    : "http://redis",
+      "port"    : 8094,
+      "password": ""
+  }
 }

.github/workflows/build-and-deploy-beta.yml

@@ -57,6 +57,7 @@ jobs:
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
       IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
       PDF_API_URL: ${{ secrets.PDF_API_URL}}
     steps:
       - name: Checkout code

.github/workflows/build-and-deploy-dev.yml

@@ -57,6 +57,7 @@ jobs:
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
       IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
       PDF_API_URL: ${{ secrets.PDF_API_URL }}
     steps:
       - name: Checkout code

.github/workflows/build-and-deploy-staging.yml

@@ -60,6 +60,7 @@ jobs:
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
       IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'false' || inputs.isProposalDiscussionForumEnabled == 'enabled'}}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
       PDF_API_URL: ${{ secrets.PDF_API_URL}}
     steps:
       - name: Checkout code

.github/workflows/build-and-deploy-test.yml

@@ -60,6 +60,7 @@ jobs:
       TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com"
       USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }}
       IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'false' || inputs.isProposalDiscussionForumEnabled == 'enabled'}}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
       PDF_API_URL: ${{ secrets.PDF_API_URL}}
     steps:
       - name: Checkout code

CHANGELOG.md

@@ -22,6 +22,7 @@ changes.
 
 - Set Proposal Discussion Forum Pillar enabled by default [Issue 1501](https://github.com/IntersectMBO/govtool/issues/1501)
 - bump @intersect.mbo/pdf-ui to v0.3.0
+- `redis` config fields changed [Issue 1234](https://github.com/IntersectMBO/govtool/issues/1234)
 
 ## [sancho-v1.0.9](https://github.com/IntersectMBO/govtool/releases/tag/sancho-v1.0.9) 2024-07-16
 

govtool/backend/app/Main.hs

@@ -8,6 +8,7 @@
 
 module Main where
 
+import           Control.Concurrent                     (forkIO)
 import           Control.Concurrent.QSem                (newQSem)
 import           Control.Exception                      (Exception, SomeException, fromException, throw)
 import           Control.Lens.Operators                 ((.~))
@@ -63,6 +64,7 @@ import           VVA.API
 import           VVA.API.Types
 import           VVA.CommandLine
 import           VVA.Config
+import           VVA.Metadata (startFetchProcess)
 import           VVA.Types                              (AppEnv (..),
                                                          AppError (CriticalError, InternalError, NotFoundError, ValidationError),
                                                          CacheEnv (..))
@@ -127,6 +129,13 @@ startApp vvaConfig = do
   vvaTlsManager <- newManager tlsManagerSettings
   qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig)
   let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem}
+
+  _ <- forkIO $ do
+     result <- runReaderT (runExceptT startFetchProcess) appEnv
+     case result of
+        Left e -> throw e
+        Right _ -> return ()
+
   server' <- mkVVAServer appEnv
   runSettings settings server'
 

govtool/backend/example-config.json

@@ -14,5 +14,10 @@
     "metadatavalidationenabled": true,
     "metadatavalidationhost": "localhost",
     "metadatavalidationport": 3001,
-    "metadatavalidationmaxconcurrentrequests": 10
+    "metadatavalidationmaxconcurrentrequests": 10,
+    "redisconfig" : {
+      "host"    : "localhost",
+      "port"    : 8094,
+      "password": null
+  }
 }

govtool/backend/sql/get-voting-anchors.sql

@@ -0,0 +1,3 @@
+select id, url, encode(data_hash, 'hex'), type::text
+from voting_anchor
+where voting_anchor.id > ?

govtool/backend/src/VVA/Config.hs

@@ -21,6 +21,9 @@ module VVA.Config
     , loadVVAConfig
       -- * Data type conversions
     , getDbSyncConnectionString
+    , getRedisHost
+    , getRedisPort
+    , getRedisPassword
     , getMetadataValidationEnabled
     , getMetadataValidationHost
     , getMetadataValidationPort
@@ -66,6 +69,14 @@ data DBConfig
 instance DefaultConfig DBConfig where
   configDef = DBConfig "localhost" "cexplorer" "postgres" "test" 9903
 
+data RedisInternalConfig
+  = RedisInternalConfig
+      { redisInternalConfigHost     :: Text
+      , redisInternalConfigPort     :: Int
+      , redisInternalConfigPassword :: Maybe Text
+      }
+  deriving (FromConfig, Generic, Show)
+
 -- | Internal, backend-dependent representation of configuration for DEX.  This
 -- data type should not be exported from this module.
 data VVAConfigInternal
@@ -90,6 +101,8 @@ data VVAConfigInternal
       , vVAConfigInternalMetadataValidationPort                  :: Int
         -- | Maximum number of concurrent metadata requests
       , vVAConfigInternalMetadataValidationMaxConcurrentRequests :: Int
+        -- | Redis config
+      , vVAConfigInternalRedisConfig :: RedisInternalConfig
       }
   deriving (FromConfig, Generic, Show)
 
@@ -105,9 +118,18 @@ instance DefaultConfig VVAConfigInternal where
         vVAConfigInternalMetadataValidationEnabled = True,
         vVAConfigInternalMetadataValidationHost = "localhost",
         vVAConfigInternalMetadataValidationPort = 3001,
-        vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10
+        vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10,
+        vVAConfigInternalRedisConfig = RedisInternalConfig "localhost" 6379 Nothing
       }
 
+data RedisConfig
+  = RedisConfig
+      { redisHost :: Text
+      , redisPort :: Int
+      , redisPassword :: Maybe Text
+      }
+  deriving (Generic, Show, ToJSON)
+
 -- | DEX configuration.
 data VVAConfig
   = VVAConfig
@@ -131,6 +153,8 @@ data VVAConfig
       , metadataValidationPort                  :: Int
         -- | Maximum number of concurrent metadata requests
       , metadataValidationMaxConcurrentRequests :: Int
+        -- | Redis config
+      , redisConfig :: RedisConfig
       }
   deriving (Generic, Show, ToJSON)
 
@@ -175,7 +199,12 @@ convertConfig VVAConfigInternal {..} =
       metadataValidationEnabled = vVAConfigInternalMetadataValidationEnabled,
       metadataValidationHost = vVAConfigInternalMetadataValidationHost,
       metadataValidationPort = vVAConfigInternalMetadataValidationPort,
-      metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests
+      metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests,
+      redisConfig = RedisConfig
+        { redisHost = redisInternalConfigHost vVAConfigInternalRedisConfig,
+          redisPort = redisInternalConfigPort vVAConfigInternalRedisConfig,
+          redisPassword = redisInternalConfigPassword vVAConfigInternalRedisConfig
+        }
     }
 
 -- | Load configuration from a file specified on the command line.  Load from
@@ -214,6 +243,23 @@ getServerHost ::
   m Text
 getServerHost = asks (serverHost . getter)
 
+-- | Access redis host
+getRedisHost ::
+  (Has VVAConfig r, MonadReader r m) =>
+  m Text
+getRedisHost = asks (redisHost . redisConfig . getter)
+
+-- | Access redis port
+getRedisPort ::
+  (Has VVAConfig r, MonadReader r m) =>
+  m Int
+getRedisPort = asks (redisPort . redisConfig .  getter)
+
+getRedisPassword ::
+  (Has VVAConfig r, MonadReader r m) =>
+  m (Maybe Text)
+getRedisPassword = asks (redisPassword . redisConfig . getter)
+
 -- | Access MetadataValidationService enabled
 getMetadataValidationEnabled ::
   (Has VVAConfig r, MonadReader r m) =>