Merge pull request #389 from mebassett/feature/tx-metadata-binary

adds txMetadata as cbor into a bytea column on the tx_metadata table.

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs

@@ -27,7 +27,8 @@ import           Control.Monad.IO.Class (liftIO)
 import           Control.Monad.Trans.Control (MonadBaseControl)
 import           Control.Monad.Trans.Reader (ReaderT)
 
-import           Cardano.Api.MetaData (TxMetadataValue (..))
+import           Cardano.Api.MetaData (TxMetadataValue (..), makeTransactionMetadata)
+import           Cardano.Api.Typed (SerialiseAsCBOR (..))
 
 import qualified Cardano.Crypto.Hash as Crypto
 
@@ -562,22 +563,28 @@ insertTxMetadata tracer txId metadata =
         -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
     insert (key, md) = do
       let jsonbs = LBS.toStrict $ Aeson.encode (metadataValueToJsonNoSchema md)
+          singleKeyCBORMetadata = serialiseToCBOR $ makeTransactionMetadata $ Map.singleton key md
       ejson <- liftIO $ safeDecodeUtf8 jsonbs
-      case ejson of
-        Left err ->
-          liftIO . logWarning tracer $ mconcat
-            [ "insertTxMetadata: Could not decode to UTF8: ", textShow err ]
-        Right json -> do
-          -- See https://github.com/input-output-hk/cardano-db-sync/issues/297
-          if containsUnicodeNul json
-            then liftIO $ logWarning tracer "insertTxMetadata: dropped due to a Unicode NUL character."
-            else
-              void . lift . DB.insertTxMetadata $
-                DB.TxMetadata
-                  { DB.txMetadataKey = DbWord64 key
-                  , DB.txMetadataJson = json
-                  , DB.txMetadataTxId = txId
-                  }
+      mjson <- case ejson of
+                 Left err -> do
+                   liftIO . logWarning tracer $ mconcat
+                      [ "insertTxMetadata: Could not decode to UTF8: ", textShow err ]
+                   return Nothing
+                 Right json ->
+                   -- See https://github.com/input-output-hk/cardano-db-sync/issues/297
+                   if containsUnicodeNul json
+                     then do
+                       liftIO $ logWarning tracer "insertTxMetadata: dropped due to a Unicode NUL character."
+                       return Nothing
+                     else
+                       return $ Just json
+      void . lift . DB.insertTxMetadata $
+        DB.TxMetadata
+          { DB.txMetadataKey = DbWord64 key
+          , DB.txMetadataJson = mjson
+          , DB.txMetadataBytes = singleKeyCBORMetadata
+          , DB.txMetadataTxId = txId
+          }
 
 safeDecodeUtf8 :: ByteString -> IO (Either Text.UnicodeException Text)
 safeDecodeUtf8 bs

cardano-db/src/Cardano/Db/Schema.hs

@@ -245,7 +245,8 @@ share
 
   TxMetadata
     key                 DbWord64            sqltype=word64type
-    json                Text                sqltype=jsonb
+    json                Text Maybe          sqltype=jsonb
+    bytes               ByteString          sqltype=bytea
     txId                TxId
     UniqueTxMetadata    key txId
 

schema/migration-2-0002-20201027.sql

@@ -0,0 +1,20 @@
+-- Persistent generated migration.
+
+CREATE FUNCTION migrate() RETURNS void AS $$
+DECLARE
+  next_version int ;
+BEGIN
+  SELECT stage_two + 1 INTO next_version FROM schema_version ;
+  IF next_version = 2 THEN
+    EXECUTE 'ALTER TABLE "tx_metadata" ALTER COLUMN "json" DROP NOT NULL' ;
+    EXECUTE 'ALTER TABLE "tx_metadata" ADD COLUMN "bytes" bytea NOT NULL' ;
+    -- Hand written SQL statements can be added here.
+    UPDATE schema_version SET stage_two = 2 ;
+    RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ;
+  END IF ;
+END ;
+$$ LANGUAGE plpgsql ;
+
+SELECT migrate() ;
+
+DROP FUNCTION migrate() ;