fix: listener silent fail on replica

CHANGELOG.md

@@ -37,6 +37,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #3424, Admin `/live` and `/ready` now differentiates a failure as 500 status - @steve-chavez
     + 503 status is still given when postgREST is in a recovering state
  - #3478, Media Types are parsed case insensitively - @develop7
+ - #2781, Fix listener silently failing on read replica - @steve-chavez
 
 ### Deprecated
 

cabal.project.freeze

@@ -1 +1 @@
-index-state: hackage.haskell.org 2024-04-15T20:28:44Z
+index-state: hackage.haskell.org 2024-05-17T23:41:49Z

nix/overlays/haskell-packages.nix

@@ -38,7 +38,7 @@ let
       # Notes:
       # - When adding a new package version here, update cabal.
       #   + Update postgrest.cabal with the package version
-      #   + Update cabal.project.freeze. Just set it to the current timestamp then run `cabal build`. It will tell you the correct timestamp for the index state.
+      #   + Update the index-state in cabal.project.freeze. Run `cabal update` which should return the latest index state.
       # - When adding a new package version here, you have to update stack.
       #   + To update stack.yaml add:
       #   extra-deps:
@@ -67,8 +67,8 @@ let
       hasql-notifications = lib.dontCheck (prev.callHackageDirect
         {
           pkg = "hasql-notifications";
-          ver = "0.2.1.1";
-          sha256 = "sha256-oPhKA/pSQGJvgQyhsi7CVr9iDT7uWpKUz0iJfXsaxXo=";
+          ver = "0.2.2.0";
+          sha256 = "sha256-73OQ9/su2qvO7HavF3xuuNWLXSXyB9reBUQDaHys06I=";
         }
         { }
       );

nix/tools/withTools.nix

@@ -106,6 +106,7 @@ let
 
             export PGREPLICAHOST="$replica_host"
             export PGREPLICASLOT="$replica_slot"
+            export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST"
           fi
 
           # shellcheck disable=SC2317

postgrest.cabal

@@ -109,7 +109,7 @@ library
                     , gitrev                    >= 1.2 && < 1.4
                     , hasql                     >= 1.6.1.1 && < 1.7
                     , hasql-dynamic-statements  >= 0.3.1 && < 0.4
-                    , hasql-notifications       >= 0.2.1.1 && < 0.3
+                    , hasql-notifications       >= 0.2.2.0 && < 0.3
                     , hasql-pool                >= 1.0.1 && < 1.1
                     , hasql-transaction         >= 1.0.1 && < 1.1
                     , heredoc                   >= 0.2 && < 0.3

src/PostgREST/AppState.hs

@@ -542,23 +542,25 @@
     dbOrError <- acquire $ toUtf8 (addFallbackAppName prettyVersion configDbUri)
     case dbOrError of
       Right db -> do
-        observer $ DBListenerStart dbChannel
         SQL.listen db $ SQL.toPgIdentifier dbChannel
+        observer $ DBListenStart dbChannel
         SQL.waitForNotifications handleNotification db
 
       Left err -> do
-        observer $ DBListenerFail dbChannel err
+        observer $ DBListenFail dbChannel (Left err)
         exitFailure
   where
     handleFinally dbChannel False err = do
-      observer $ DBListenerFailRecoverObs False dbChannel err
+      observer $ DBListenFail dbChannel (Right err)
       killThread mainThreadId
     handleFinally dbChannel True err = do
       -- if the thread dies, we try to recover
-      observer $ DBListenerFailRecoverObs True dbChannel err
+      observer $ DBListenFail dbChannel (Right err)
       -- assume the pool connection was also lost, call the connection worker
       connectionWorker appState
+
       -- retry the listener
+      observer DBListenRetry
       listener appState conf
 
     handleNotification channel msg =

src/PostgREST/Observation.hs

@@ -40,9 +40,9 @@
   | SchemaCacheLoadedObs Double
   | ConnectionRetryObs Int
   | ConnectionPgVersionErrorObs SQL.UsageError
-  | DBListenerStart Text
-  | DBListenerFail Text SQL.ConnectionError
-  | DBListenerFailRecoverObs Bool Text (Either SomeException ())
+  | DBListenStart Text
+  | DBListenFail Text (Either SQL.ConnectionError (Either SomeException ()))
+  | DBListenRetry
   | DBListenerGotSCacheMsg ByteString
   | DBListenerGotConfigMsg ByteString
   | ConfigReadErrorObs SQL.UsageError
@@ -97,12 +97,16 @@
     "Attempting to reconnect to the database in " <> (show delay::Text) <> " seconds..."
   ConnectionPgVersionErrorObs usageErr ->
     jsonMessage usageErr
-  DBListenerStart channel -> do
+  DBListenStart channel -> do
     "Listening for notifications on the " <> show channel <> " channel"
-  DBListenerFail channel err -> do
-    "Could not listen for notifications on the " <> channel <> " channel. " <> show err
-  DBListenerFailRecoverObs recover channel err ->
-    "Could not listen for notifications on the " <> channel <> " channel. " <> showListenerError err <> (if recover then " Retrying listening for notifications.." else mempty)
+  DBListenFail channel listenErr ->
+    "Failed listening for notifications on the " <> show channel <> " channel. " <> (
+      case listenErr of
+        Left err  -> show err
+        Right err -> showListenerError err
+    )
+  DBListenRetry ->
+    "Retrying listening for notifications..."
   DBListenerGotSCacheMsg channel ->
     "Received a schema cache reload message on the " <> show channel <> " channel"
   DBListenerGotConfigMsg channel ->

stack.yaml

@@ -11,4 +11,5 @@ nix:
 
 extra-deps:
   - fuzzyset-0.2.4
+  - hasql-notifications-0.2.2.0
   - hasql-pool-1.0.1

stack.yaml.lock

@@ -11,6 +11,13 @@ packages:
       size: 574
   original:
     hackage: fuzzyset-0.2.4
+- completed:
+    hackage: hasql-notifications-0.2.2.0@sha256:a4e591ef3f06647b056567d3b66948c4a85371f05deb5434edb6ce190f7c845d,2021
+    pantry-tree:
+      sha256: bd7192a5e82ef6dbac711c3433408a0330c8db1cd3482be1ccd4fbd0a63bc2f6
+      size: 452
+  original:
+    hackage: hasql-notifications-0.2.2.0
 - completed:
     hackage: hasql-pool-1.0.1@sha256:3cfb4c7153a6c536ac7e126c17723e6d26ee03794954deed2d72bcc826d05a40,2302
     pantry-tree: