hyperledger-labs · prakashngit · Apr 29, 2024 · Apr 11, 2024 · Apr 1, 2024 · Apr 2, 2024
diff --git a/cmake/Jupyter.cmake b/cmake/Jupyter.cmake
@@ -14,6 +14,9 @@
 
 
 SET(JUPYTEXT "${PDO_INSTALL_ROOT}/bin/jupytext" CACHE STRING "Path to the Jupytext command")
+IF(NOT EXISTS ${JUPYTEXT})
+  MESSAGE(FATAL_ERROR "${JUPYTEXT} not found; please install with '${PDO_INSTALL_ROOT}/bin/pip install jupytext'")
+ENDIF()
 
 FUNCTION(CONVERT_JUPYTEXT NOTEBOOKS)
   SET(RESULT)

diff --git a/digital-asset-contract/test/run_tests.sh b/digital-asset-contract/test/run_tests.sh
@@ -26,8 +26,7 @@ source ${PDO_HOME}/bin/lib/common.sh
 check_python_version
 
 if ! command -v pdo-shell &> /dev/null ; then
-    yell unable to locate pdo-shell
-    exit 1
+    die unable to locate pdo-shell
 fi
 
 # -----------------------------------------------------------------
@@ -72,14 +71,21 @@ while true ; do
     esac
 done
 
-F_SERVICE_GROUPS_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_groups.toml
+F_SERVICE_SITE_FILE=${PDO_HOME}/etc/sites/${F_SERVICE_HOST}.toml
+if [ ! -f ${F_SERVICE_SITE_FILE} ] ; then
+    die unable to locate the service information file ${F_SERVICE_SITE_FILE}; \
+        please copy the site.toml file from the service host
+fi
+
+F_SERVICE_GROUPS_DB_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_groups_db
 F_SERVICE_DB_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_db
 
 _COMMON_=("--logfile ${F_LOGFILE}" "--loglevel ${F_LOGLEVEL}")
 _COMMON_+=("--ledger ${F_LEDGER_URL}")
-_COMMON_+=("--bind service_host ${F_SERVICE_HOST}")
-_COMMON_+=("--service-groups ${F_SERVICE_GROUPS_FILE}")
+_COMMON_+=("--groups-db ${F_SERVICE_GROUPS_DB_FILE}")
 _COMMON_+=("--service-db ${F_SERVICE_DB_FILE}")
+SHORT_OPTS=${_COMMON_[@]}
+
 _COMMON_+=("--context-file ${F_CONTEXT_FILE}")
 OPTS=${_COMMON_[@]}
 
@@ -114,7 +120,7 @@ fi
 
 # -----------------------------------------------------------------
 function cleanup {
-    rm -f ${F_SERVICE_GROUPS_FILE}
+    rm -f ${F_SERVICE_GROUPS_DB_FILE} ${F_SERVICE_GROUPS_DB_FILE}-lock
     rm -f ${F_SERVICE_DB_FILE} ${F_SERVICE_DB_FILE}-lock
     rm -f ${F_CONTEXT_FILE}
     for key_file in ${F_KEY_FILES[@]} ; do
@@ -125,13 +131,18 @@ function cleanup {
 trap cleanup EXIT
 
 # -----------------------------------------------------------------
-# reset the eservice database file for the test and create the groups
+# create the service and groups databases from a site file; the site
+# file is assumed to exist in ${PDO_HOME}/etc/sites/${SERVICE_HOST}.toml
+#
+# by default, the groups will include all available services from the
+# service host
 # -----------------------------------------------------------------
 yell create the service and groups database for host ${F_SERVICE_HOST}
-try ${PDO_HOME}/bin/pdo-create-service-groups.psh \
-    --service-db ${F_SERVICE_DB_FILE} \
-    --bind service_host ${F_SERVICE_HOST} \
-    --bind group_file ${F_SERVICE_GROUPS_FILE}
+try pdo-service-db import ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE}
+try pdo-eservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-pservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-sservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default \
+             --replicas 1 --duration 60
 
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------

diff --git a/digital-asset-contract/test/script_test.sh b/digital-asset-contract/test/script_test.sh
@@ -26,8 +26,7 @@ source ${PDO_HOME}/bin/lib/common.sh
 check_python_version
 
 if ! command -v pdo-shell &> /dev/null ; then
-    yell unable to locate pdo-shell
-    exit 1
+    die unable to locate pdo-shell
 fi
 
 # -----------------------------------------------------------------
@@ -73,14 +72,21 @@ while true ; do
     esac
 done
 
-F_SERVICE_GROUPS_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_groups.toml
+F_SERVICE_SITE_FILE=${PDO_HOME}/etc/sites/${F_SERVICE_HOST}.toml
+if [ ! -f ${F_SERVICE_SITE_FILE} ] ; then
+    die unable to locate the service information file ${F_SERVICE_SITE_FILE}; \
+        please copy the site.toml file from the service host
+fi
+
+F_SERVICE_GROUPS_DB_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_groups_db
 F_SERVICE_DB_FILE=${SOURCE_ROOT}/test/${F_SERVICE_HOST}_db
 
 _COMMON_=("--logfile ${F_LOGFILE}" "--loglevel ${F_LOGLEVEL}")
 _COMMON_+=("--ledger ${F_LEDGER_URL}")
-_COMMON_+=("--bind service_host ${F_SERVICE_HOST}")
-_COMMON_+=("--service-groups ${F_SERVICE_GROUPS_FILE}")
+_COMMON_+=("--groups-db ${F_SERVICE_GROUPS_DB_FILE}")
 _COMMON_+=("--service-db ${F_SERVICE_DB_FILE}")
+SHORT_OPTS=${_COMMON_[@]}
+
 _COMMON_+=("--context-file ${F_CONTEXT_FILE}")
 OPTS=${_COMMON_[@]}
 
@@ -115,7 +121,7 @@ fi
 
 # -----------------------------------------------------------------
 function cleanup {
-    rm -f ${F_SERVICE_GROUPS_FILE}
+    rm -f ${F_SERVICE_GROUPS_DB_FILE} ${F_SERVICE_GROUPS_DB_FILE}-lock
     rm -f ${F_SERVICE_DB_FILE} ${F_SERVICE_DB_FILE}-lock
     rm -f ${F_CONTEXT_FILE}
     for key_file in ${F_KEY_FILES[@]} ; do
@@ -126,13 +132,18 @@ function cleanup {
 trap cleanup EXIT
 
 # -----------------------------------------------------------------
-# reset the eservice database file for the test and create the groups
-# -----------------------------------------------------------------
-yell create the service groups database for host ${F_SERVICE_HOST}
-try ${PDO_HOME}/bin/pdo-create-service-groups.psh \
-    --service-db ${F_SERVICE_DB_FILE} \
-    --bind service_host ${F_SERVICE_HOST} \
-    --bind group_file ${F_SERVICE_GROUPS_FILE}
+# create the service and groups databases from a site file; the site
+# file is assumed to exist in ${PDO_HOME}/etc/sites/${SERVICE_HOST}.toml
+#
+# by default, the groups will include all available services from the
+# service host
+# -----------------------------------------------------------------
+yell create the service and groups database for host ${F_SERVICE_HOST}
+try pdo-service-db import ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE}
+try pdo-eservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-pservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-sservice create_from_site ${SHORT_OPTS} --file ${F_SERVICE_SITE_FILE} --group default \
+             --replicas 1 --duration 60
 
 # -----------------------------------------------------------------
 # setup the contexts that will be used later for the tests

diff --git a/docker/test_jupyter.yaml b/docker/test_jupyter.yaml
@@ -69,4 +69,4 @@ services:
     network_mode: "host"
     volumes:
       - ./xfer/:/project/pdo/xfer/
-    entrypoint: /project/pdo/tools/run_jupyter_tests.sh -i localhost -l http://127.0.0.1:6600
+    entrypoint: /project/pdo/tools/run_jupyter_tests.sh -i localhost -l http://127.0.0.1:6600 -s localhost
diff --git a/docker/tools/run_contracts_tests.sh b/docker/tools/run_contracts_tests.sh
@@ -60,6 +60,11 @@ export NO_PROXY=$PDO_HOSTNAME,$NO_PROXY
 
 check_pdo_runtime_env
 
+# -----------------------------------------------------------------
+# activate the virtual environment
+# -----------------------------------------------------------------
+. ${PDO_INSTALL_ROOT}/bin/activate
+
 # -----------------------------------------------------------------
 yell copy ledger keys
 # -----------------------------------------------------------------
@@ -75,23 +80,28 @@ while [ ! -f ${XFER_DIR}/ccf/keys/networkcert.pem ]; do
 done
 try cp ${XFER_DIR}/ccf/keys/networkcert.pem ${PDO_LEDGER_KEY_ROOT}/
 
-# for now the site.toml is just a way to notify
-# that the services are running; in the future
-# the client should be able to incorporate this
-# file and begin to use the information, again
-# in theory this should be taken care of by the
-# health checks in the docker compose configuration
+# site.toml is a way to notify that the services are running;
+# in addition, the client uses the information to load the
+# services and groups database; while we create an initial
+# default database from this information, each test is expected
+# to manage the database appropriately
+F_SERVICE_SITE_FILE=${PDO_HOME}/etc/sites/${F_SERVICE_HOST}.toml
+mkdir -p $(dirname ${F_SERVICE_SITE_FILE})
+
 while [ ! -f ${XFER_DIR}/services/etc/site.toml ]; do
     say "waiting for site configuration"
     sleep 5
 done
 
-try cp ${XFER_DIR}/services/etc/site.toml ${PDO_HOME}/etc/site.toml
+try cp ${XFER_DIR}/services/etc/site.toml ${F_SERVICE_SITE_FILE}
+try pdo-service-db import --file ${F_SERVICE_SITE_FILE}
+try pdo-eservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-pservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-sservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default --replicas 1 --duration 60
 
 # -----------------------------------------------------------------
 yell run the service test suite
 # -----------------------------------------------------------------
-. ${PDO_INSTALL_ROOT}/bin/activate
 try make -C /project/pdo/contracts \
     TEST_LOG_LEVEL=${F_LOGLEVEL} TEST_SERVICE_HOST=${F_SERVICE_HOST} TEST_LEDGER=${F_LEDGER_URL} \
     test
diff --git a/docker/tools/run_guardian_tests.sh b/docker/tools/run_guardian_tests.sh
@@ -75,19 +75,14 @@ while [ ! -f ${XFER_DIR}/ccf/keys/networkcert.pem ]; do
 done
 try cp ${XFER_DIR}/ccf/keys/networkcert.pem ${PDO_LEDGER_KEY_ROOT}/
 
-# for now the site.toml is just a way to notify
-# that the services are running; in the future
-# the client should be able to incorporate this
-# file and begin to use the information, again
-# in theory this should be taken care of by the
-# health checks in the docker compose configuration
+# site.toml is a way to notify that the services are running
+# the guardian service does not require any site specific configuration
+# so the services and groups are not created
 while [ ! -f ${XFER_DIR}/services/etc/site.toml ]; do
     say "waiting for site configuration"
     sleep 5
 done
 
-try cp ${XFER_DIR}/services/etc/site.toml ${PDO_HOME}/etc/site.toml
-
 # -----------------------------------------------------------------
 function cleanup {
     yell "shutdown guardian and storage service"

diff --git a/docker/tools/run_jupyter_tests.sh b/docker/tools/run_jupyter_tests.sh
@@ -22,12 +22,13 @@ source ${PDO_HOME}/bin/lib/common.sh
 # Process command line arguments
 # -----------------------------------------------------------------
 F_LEDGER_URL=
+F_SERVICE_HOST=
 F_INTERFACE=localhost
 F_PORT=8888
 
-F_USAGE='-i|--interface [hostname] -1|--ledger [url] -p|--port [port]'
-SHORT_OPTS='i:l:p:'
-LONG_OPTS='interface:,ledger:,port:'
+F_USAGE='-i|--interface [hostname] -1|--ledger [url] -p|--port [port] -s|--service-host [hostname]'
+SHORT_OPTS='i:l:p:s:'
+LONG_OPTS='interface:,ledger:,port:,service-host:'
 
 TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@")
 if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${F_USAGE}" >&2 ; exit 1 ; fi
@@ -38,27 +39,37 @@ while true ; do
         -i|--interface) F_INTERFACE="$2" ; shift 2 ;;
         -l|--ledger) F_LEDGER_URL="$2" ; shift 2 ;;
         -p|--port) F_PORT="$2" ; shift 2 ;;
+        -s|--service-host) F_SERVICE_HOST="$2" ; shift 2 ;;
         --help) echo "Usage: ${SCRIPT_NAME} ${F_USAGE}"; exit 0 ;;
     	--) shift ; break ;;
     	*) echo "Internal error!" ; exit 1 ;;
     esac
 done
 
+if [ -z "${F_SERVICE_HOST}" ]; then
+    die Missing required parameter: service-host
+fi
+
 if [ -z "${F_LEDGER_URL}" ]; then
     die Missing required parameter: ledger
 fi
 
 # The client only uses PDO_HOSTNAME to create the initial configuration
 # file. Since the notebook will override this for each operation, we
 # just leave it set to localhost
-export PDO_HOSTNAME=localhost
+export PDO_HOSTNAME=${F_SERVICE_HOST}
 export PDO_LEDGER_URL=${F_LEDGER_URL}
 
 export no_proxy=$PDO_HOSTNAME,$no_proxy
 export NO_PROXY=$PDO_HOSTNAME,$NO_PROXY
 
 check_pdo_runtime_env
 
+# -----------------------------------------------------------------
+# activate the virtual environment
+# -----------------------------------------------------------------
+. ${PDO_INSTALL_ROOT}/bin/activate
+
 # -----------------------------------------------------------------
 yell copy ledger keys
 # -----------------------------------------------------------------
@@ -74,18 +85,23 @@ while [ ! -f ${XFER_DIR}/ccf/keys/networkcert.pem ]; do
 done
 try cp ${XFER_DIR}/ccf/keys/networkcert.pem ${PDO_LEDGER_KEY_ROOT}/
 
-# for now the site.toml is just a way to notify
-# that the services are running; in the future
-# the client should be able to incorporate this
-# file and begin to use the information, again
-# in theory this should be taken care of by the
-# health checks in the docker compose configuration
+# site.toml is a way to notify that the services are running;
+# in addition, the client uses the information to load the
+# services and groups database; we create an initial
+# default database from this information
+F_SERVICE_SITE_FILE=${PDO_HOME}/etc/sites/${F_SERVICE_HOST}.toml
+mkdir -p $(dirname ${F_SERVICE_SITE_FILE})
+
 while [ ! -f ${XFER_DIR}/services/etc/site.toml ]; do
     say "waiting for site configuration"
     sleep 5
 done
 
-try cp ${XFER_DIR}/services/etc/site.toml ${PDO_HOME}/etc/site.toml
+try cp ${XFER_DIR}/services/etc/site.toml ${F_SERVICE_SITE_FILE}
+try pdo-service-db import --file ${F_SERVICE_SITE_FILE}
+try pdo-eservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-pservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default
+try pdo-sservice create_from_site --file ${F_SERVICE_SITE_FILE} --group default --replicas 1 --duration 60
 
 # -----------------------------------------------------------------
 # Handle the configuration of the services
@@ -97,8 +113,6 @@ try ${PDO_INSTALL_ROOT}/bin/pdo-configure-users -t ${PDO_SOURCE_ROOT}/build/temp
 # -----------------------------------------------------------------
 yell start the jupyter server
 # -----------------------------------------------------------------
-. ${PDO_INSTALL_ROOT}/bin/activate
-
 export PDO_JUPYTER_ROOT=${PDO_INSTALL_ROOT}/opt/pdo/notebooks
 
 cd ${PDO_JUPYTER_ROOT}

diff --git a/docs/install.md b/docs/install.md
@@ -78,17 +78,19 @@ virtual environment in the directory `${PDO_INSTALL_ROOT}`.
 make -C ${PDO_SOURCE_ROOT}/build client
 ```
 
-Finally, activate the PDO Python virtual environment:
-
-```
-source ${PDO_INSTALL_ROOT}/bin/activate
-```
-
 More information about PDO client installation is found in the
 [PDO documentation](../private-data-objects/docs/client_install.md).
 
 ## <a name="contracts">Build and Install the PDO Contracts Packages</a>
 
+Activate the PDO Python virtual environment and install the
+Python packages necessary to build contracts:
+
+```bash
+source ${PDO_INSTALL_ROOT}/bin/activate
+${PDO_INSTALL_ROOT}/bin/pip install jupytext
+```
+
 Finally, build and install the Python packages associated with the PDO contract
 families. The packages will be built in the directory
 `${PDO_CONTRACTS_ROOT}/build/dist`.
@@ -104,3 +106,10 @@ on ${SERVICE_HOST}, you can test the installation with the following:
 ```
 TEST_SERVICE_HOST=${SERVICE_HOST} make -C ${PDO_CONTRACTS_ROOT} test
 ```
+
+If you wish to use [Jupyter notebooks](jupyter.md) to interact with
+PDO contracts, you will need to install several additional packages:
+
+```bash
+${PDO_INSTALL_ROOT}/bin/pip install jupyterlab papermill ipywidgets
+```