Reliable persistence

CHANGELOG.md

@@ -14,6 +14,8 @@ changes.
   layer to improve reliability in the face of transient connection
   issues
 
+- Persist network messages on disk in order to gracefully handle crashes
+
 ## [0.13.0] - 2023-10-03
 
 - **BREAKING** Update to plutus 1.9. This changes the script hashes.

hydra-node/exe/hydra-node/Main.hs

@@ -101,8 +101,9 @@ main = do
                 RunOptions{apiHost, apiPort} = opts
             apiPersistence <- createPersistenceIncremental $ persistenceDir <> "/server-output"
             withAPIServer apiHost apiPort party apiPersistence (contramap APIServer tracer) chain pparams (putEvent . ClientEvent) $ \server -> do
+
               -- Network
-              withNetwork tracer (connectionMessages server) signingKey otherParties host port peers nodeId putNetworkEvent $ \hn -> do
+              withNetwork tracer persistenceDir (connectionMessages server) signingKey otherParties host port peers nodeId putNetworkEvent $ \hn -> do
                 -- Main loop
                 runHydraNode (contramap Node tracer) $
                   HydraNode

hydra-node/json-schemas/logs.yaml

@@ -587,13 +587,22 @@ definitions:
         additionalProperties: false
         required:
           - tag
-          - failedToFindMessage
+          - missingMsgIndex
+          - sentMessagesLength
+          - knownAckForUs
+          - messageAckForUs
         properties:
           tag:
             type: string
             enum: ["ReliabilityFailedToFindMsg"]
-          failedToFindMessage:
-            type: string
+          missingMsgIndex:
+            type: number
+          sentMessagesLength:
+            type: number
+          knownAckForUs:
+            type: number
+          messageAckForUs:
+            type: number
       - title: ReliabilityMissingPartyIndex
         description: >-
           Logged when party index could not be found. NOTE: This should never happen.
@@ -608,6 +617,28 @@ definitions:
             enum: ["ReliabilityMissingPartyIndex"]
           missingParty:
             $ref: "api.yaml#/components/schemas/Party"
+      - title: ReceivedMalformedAcks
+        description: >-
+          Logged when we receive a message from a party but its vector clock is malformed (not having the same length as the number of parties in a Head)
+        type: object
+        additionalProperties: false
+        required:
+          - tag
+          - fromParty
+          - partyAcks
+          - numberOfParties
+        properties:
+          tag:
+            type: string
+            enum: ["ReceivedMalformedAcks"]
+          fromParty:
+            $ref: "api.yaml#/components/schemas/Party"
+          partyAcks:
+            type: array
+            items:
+              type: number
+          numberOfParties:
+            type: number
 
 
   # NOTE: We cannot simply describe 'ServerOutput' and re-use it in api.yaml, so