feat(prism-agent): integrate DID secret storage with Vault (#543)

* feat(prism-agent): integrate vault with DIDSecretStorage

* chore: resolve squash conflict

* ci: vault test container on runner

* test

* chore: pr cleanup

* fix aquireRelease leakage

* tests: conditionally override container host and port

* tests: fix container host name

* tests: fix aquireRelease on test container

* chore: minor cleanup

* chore: resolve rebase conflict

* docs: add doc on agent config guide

* Revert "docs: add doc on agent config guide"

This reverts commit 0100ef7.

* chore: change default devMode config to false

* chore: improve error message when WALLET_SEED parsing fail

* chore: use released agent version in local .env

build.sbt

@@ -55,7 +55,7 @@ lazy val V = new {
 
   val typesafeConfig = "1.4.2"
   val protobuf = "3.1.9"
-  val testContainersScalaPostgresql = "0.40.11"
+  val testContainersScala = "0.40.16"
 
   val doobie = "1.0.0-RC2"
   val quill = "4.6.0"
@@ -74,6 +74,10 @@ lazy val V = new {
   val bouncyCastle = "1.70"
 
   val jsonSchemaValidator = "1.0.83"
+
+  // https://github.com/jopenlibs/vault-java-driver/issues/36
+  // v5.4.0 is not available on Maven yet.
+  val vaultDriver = "5.3.0"
 }
 
 /** Dependencies */
@@ -106,7 +110,8 @@ lazy val D = new {
   val typesafeConfig: ModuleID = "com.typesafe" % "config" % V.typesafeConfig
   val scalaPbGrpc: ModuleID = "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion
   // TODO we are adding test stuff to the main dependencies
-  val testcontainers: ModuleID = "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScalaPostgresql
+  val testcontainersPostgres: ModuleID = "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScala
+  val testcontainersVault: ModuleID = "com.dimafeng" %% "testcontainers-scala-vault" % V.testContainersScala
 
   val doobiePostgres: ModuleID = "org.tpolecat" %% "doobie-postgres" % V.doobie
   val doobieHikari: ModuleID = "org.tpolecat" %% "doobie-hikari" % V.doobie
@@ -128,20 +133,16 @@ lazy val D = new {
 }
 
 lazy val D_Shared = new {
-
-  lazy val dependencies: Seq[ModuleID] = Seq(D.typesafeConfig, D.scalaPbGrpc, D.testcontainers)
+  lazy val dependencies: Seq[ModuleID] = Seq(D.typesafeConfig, D.scalaPbGrpc, D.testcontainersPostgres, D.testcontainersVault)
 }
 
 lazy val D_Connect = new {
 
   private lazy val logback = "ch.qos.logback" % "logback-classic" % V.logback % Test
 
-  private lazy val testcontainers =
-    "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScalaPostgresql % Test
-
   // Dependency Modules
   private lazy val baseDependencies: Seq[ModuleID] =
-    Seq(D.zio, D.zioTest, D.zioTestSbt, D.zioTestMagnolia, testcontainers, logback)
+    Seq(D.zio, D.zioTest, D.zioTestSbt, D.zioTestMagnolia, D.testcontainersPostgres, logback)
 
   // Project Dependencies
   lazy val coreDependencies: Seq[ModuleID] =
@@ -196,9 +197,6 @@ lazy val D_Pollux = new {
   val quillDoobie = "io.getquill" %% "quill-doobie" %
     V.quill exclude ("org.scala-lang.modules", "scala-java8-compat_3")
 
-  val testcontainers =
-    "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScalaPostgresql % Test
-
   // We have to exclude bouncycastle since for some reason bitcoinj depends on bouncycastle jdk15to18
   // (i.e. JDK 1.5 to 1.8), but we are using JDK 11
   val prismCrypto = "io.iohk.atala" % "prism-crypto-jvm" % V.prismSdk excludeAll
@@ -229,10 +227,10 @@ lazy val D_Pollux = new {
     D.zioCatsInterop,
     D.doobiePostgres,
     D.doobieHikari,
+    D.testcontainersPostgres,
     flyway,
     quillDoobie,
     quillJdbcZio,
-    testcontainers
   )
 
   // Project Dependencies
@@ -302,8 +300,8 @@ lazy val D_PrismAgent = new {
     "io.getquill" %% "quill-jdbc-zio" % V.quill exclude ("org.scala-lang.modules", "scala-java8-compat_3")
 
   val flyway = "org.flywaydb" % "flyway-core" % V.flyway
-  val testcontainers_scala_postgresql =
-    "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScalaPostgresql % Test
+
+  val vaultDriver = "io.github.jopenlibs" % "vault-java-driver" % V.vaultDriver
 
   // Dependency Modules
   val baseDependencies: Seq[ModuleID] = Seq(
@@ -333,11 +331,11 @@ lazy val D_PrismAgent = new {
     )
 
   val postgresDependencies: Seq[ModuleID] =
-    Seq(quillDoobie, quillJdbcZio, postgresql, flyway, testcontainers_scala_postgresql)
+    Seq(quillDoobie, quillJdbcZio, postgresql, flyway, D.testcontainersPostgres)
 
   // Project Dependencies
   lazy val keyManagementDependencies: Seq[ModuleID] =
-    baseDependencies ++ bouncyDependencies ++ D.doobieDependencies ++ Seq(D.zioCatsInterop, D.zioMock)
+    baseDependencies ++ bouncyDependencies ++ D.doobieDependencies ++ Seq(D.zioCatsInterop, D.zioMock, vaultDriver)
 
   lazy val serverDependencies: Seq[ModuleID] =
     baseDependencies ++ tapirDependencies ++ postgresDependencies ++ Seq(D.zioMock)

connect/lib/sql-doobie/src/test/scala/io/iohk/atala/connect/sql/repository/JdbcConnectionRepositorySpec.scala

@@ -47,7 +47,7 @@ object JdbcConnectionRepositorySpec extends ZIOSpecDefault {
   override def spec: Spec[TestEnvironment with Scope, Any] =
     (suite("JDBC Connection Repository test suite")(
       ConnectionRepositorySpecSuite.testSuite
-    ) @@ TestAspect.before(
+    ) @@ TestAspect.sequential @@ TestAspect.before(
       ZIO.serviceWithZIO[Migrations](_.migrate)
     )).provide(testEnvironmentLayer)
 }

infrastructure/dev/build.sh

@@ -2,25 +2,13 @@
 
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 REPO_HOME="${SCRIPT_DIR}/../.."
 
-LIBS="shared iris/client/scala-client castor/lib pollux/lib mercury/mercury-library"
-SERVICES="mercury/mercury-mediator prism-agent/service iris/service"
-
-echo "--------------------------------------"
-echo "Publishing libraries"
-echo "--------------------------------------"
-
-for LIB in ${LIBS}; do
-  (cd ${REPO_HOME}/${LIB}; sbt "clean;publishLocal")
-done
-
 echo "--------------------------------------"
 echo "Building service docker images"
 echo "--------------------------------------"
 
-for SERVICE in ${SERVICES}; do
-  (cd ${REPO_HOME}/${SERVICE}; sbt docker:publishLocal)
-done
+cd ${REPO_HOME}
+sbt "clean; docker:publishLocal"

infrastructure/dev/clean.sh

@@ -2,19 +2,9 @@
 
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 # Set working directory
 cd ${SCRIPT_DIR}
 cd ../../
-
-echo "--------------------------------------"
-echo "Cleaning libraries"
-echo "--------------------------------------"
-
-cd shared;sbt "clean;cleanFiles";cd -
-cd iris/client/scala-client;sbt "clean;cleanFiles";cd -
-cd castor/lib;sbt "clean;cleanFiles";cd -
-cd pollux/vc-jwt;sbt "clean;cleanFiles";cd -
-cd pollux/lib;sbt "clean;cleanFiles";cd -
-cd mercury/mercury-library;sbt "clean;cleanFiles";cd -
+sbt "clean;cleanFiles"

infrastructure/dev/get-versions.sh

@@ -1,18 +1,15 @@
 #!/usr/bin/env bash
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 # Set working directory
 cd ${SCRIPT_DIR}
 
-export PRISM_AGENT_VERSION=$(cd ../../prism-agent/service && sbt "project server" -Dsbt.supershell=false -error "print version")
+export PRISM_AGENT_VERSION=$(cd ../../ && sbt "project agent" -Dsbt.supershell=false -error "print version")
 echo "prism-agent server version: ${PRISM_AGENT_VERSION}"
 
-export MERCURY_MEDIATOR_VERSION=$(cd ../../mercury/mercury-mediator && sbt "project mediator" -Dsbt.supershell=false -error "print version")
+export MERCURY_MEDIATOR_VERSION=$(cd ../../ && sbt "project mediator" -Dsbt.supershell=false -error "print version")
 echo "mercury-mediator version: ${MERCURY_MEDIATOR_VERSION}"
 
-export IRIS_SERVICE_VERSION=$(cd ../../iris/service && sbt "project server" -Dsbt.supershell=false -error "print version")
-echo "iris server version: ${IRIS_SERVICE_VERSION}"
-
 export PRISM_NODE_VERSION=v2.1.1
 echo "prism node version: ${PRISM_NODE_VERSION}"

infrastructure/dev/run.sh

@@ -2,77 +2,76 @@
 
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 # Set working directory
 cd ${SCRIPT_DIR}
 
 source get-versions.sh
+export VAULT_DEV_ROOT_TOKEN_ID="root"
 
-Help()
-{
-   # Display Help
-   echo "Run an instance of the ATALA bulding-block stack locally"
-   echo
-   echo "Syntax: run.sh [-n/--name NAME|-p/--port PORT|-b/--background|-w/--wait|--debug|-h/--help]"
-   echo "options:"
-   echo "-n/--name          Name of this instance - defaults to dev."
-   echo "-p/--port          Port to run this instance on - defaults to 80."
-   echo "-b/--background    Run in docker-compose daemon mode in the background."
-   echo "-w/--wait          Wait until all containers are healthy (only in the background)."
-   echo "--debug            Run additional services for debug using docker-compose debug profile."
-   echo "-h/--help          Print this help text."
-   echo
+Help() {
+	# Display Help
+	echo "Run an instance of the ATALA bulding-block stack locally"
+	echo
+	echo "Syntax: run.sh [-n/--name NAME|-p/--port PORT|-b/--background|-w/--wait|--debug|-h/--help]"
+	echo "options:"
+	echo "-n/--name          Name of this instance - defaults to dev."
+	echo "-p/--port          Port to run this instance on - defaults to 80."
+	echo "-b/--background    Run in docker-compose daemon mode in the background."
+	echo "-w/--wait          Wait until all containers are healthy (only in the background)."
+	echo "--debug            Run additional services for debug using docker-compose debug profile."
+	echo "-h/--help          Print this help text."
+	echo
 }
 
 POSITIONAL_ARGS=()
 
 while [[ $# -gt 0 ]]; do
-  case $1 in
-    -n|--name)
-      NAME="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    -p|--port)
-      PORT="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    -b|--background)
-      BACKGROUND="-d"
-      shift # past argument
-      ;;
-    -w|--wait)
-      WAIT="--wait"
-      shift # past argument
-      ;;
-    -h|--help)
-      Help
-      exit
-      ;;
-    --debug)
-      DEBUG="--profile debug"
-      shift # past argument
-      ;;
-    -*|--*)
-      echo "Unknown option $1"
-      Help
-      exit 1
-      ;;
-    *)
-      POSITIONAL_ARGS+=("$1") # save positional arg
-      shift # past argument
-      ;;
-  esac
+	case $1 in
+	-n | --name)
+		NAME="$2"
+		shift # past argument
+		shift # past value
+		;;
+	-p | --port)
+		PORT="$2"
+		shift # past argument
+		shift # past value
+		;;
+	-b | --background)
+		BACKGROUND="-d"
+		shift # past argument
+		;;
+	-w | --wait)
+		WAIT="--wait"
+		shift # past argument
+		;;
+	-h | --help)
+		Help
+		exit
+		;;
+	--debug)
+		DEBUG="--profile debug"
+		shift # past argument
+		;;
+	-* | --*)
+		echo "Unknown option $1"
+		Help
+		exit 1
+		;;
+	*)
+		POSITIONAL_ARGS+=("$1") # save positional arg
+		shift                   # past argument
+		;;
+	esac
 done
 
 set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
 
-
 if [[ -n $1 ]]; then
-    echo "Last line of file specified as non-opt/last argument:"
-    tail -1 "$1"
+	echo "Last line of file specified as non-opt/last argument:"
+	tail -1 "$1"
 fi
 
 NAME="${NAME:=dev}"
@@ -86,5 +85,5 @@ echo "Starting stack using docker compose"
 echo "--------------------------------------"
 
 PORT=${PORT} docker compose \
-  -p ${NAME} \
-  -f ${SCRIPT_DIR}/../shared/docker-compose.yml ${DEBUG} up ${BACKGROUND} ${WAIT}
+	-p ${NAME} \
+	-f ${SCRIPT_DIR}/../shared/docker-compose.yml ${DEBUG} up ${BACKGROUND} ${WAIT}

infrastructure/local/.env

@@ -1,6 +1,7 @@
 MERCURY_MEDIATOR_VERSION=0.3.0
 IRIS_SERVICE_VERSION=0.1.0
-PRISM_AGENT_VERSION=0.61.0
+PRISM_AGENT_VERSION=1.4.0
 PRISM_NODE_VERSION=v2.1.1
 PORT=80
 NETWORK=prism
+VAULT_DEV_ROOT_TOKEN_ID=root

infrastructure/multi/run-e2e-tests-local.sh

@@ -2,28 +2,31 @@
 
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 echo "--------------------------------------"
 echo "Starting issuer using local/run.sh"
 echo "--------------------------------------"
 
-${SCRIPT_DIR}/../local/run.sh -p 8080 -n issuer -b
+${SCRIPT_DIR}/../local/run.sh -p 8080 -n issuer -w
 
 echo "--------------------------------------"
 echo "Starting holder using local/run.sh"
 echo "--------------------------------------"
 
-${SCRIPT_DIR}/../local/run.sh -p 8090 -n holder -b
+${SCRIPT_DIR}/../local/run.sh -p 8090 -n holder -w
 
 echo "--------------------------------------"
 echo "Starting verifier using local/run.sh"
 echo "--------------------------------------"
 
-${SCRIPT_DIR}/../local/run.sh -p 8100 -n verifier -b
+${SCRIPT_DIR}/../local/run.sh -p 8100 -n verifier -w
 
 echo "--------------------------------------"
 echo "Run e2e tests"
 echo "--------------------------------------"
 
-(cd ${SCRIPT_DIR}/../../tests/e2e-tests/; ./gradlew test reports)
+(
+	cd ${SCRIPT_DIR}/../../tests/e2e-tests/
+	./gradlew test reports
+)