Release 2021.10.0

blackducksoftware · Nov 4, 2021 · b6ba31c · b6ba31c
1 parent 5674489
commit b6ba31c
Show file tree

Hide file tree

Showing 35 changed files with 264 additions and 199 deletions.
diff --git a/README.containers.md b/README.containers.md
@@ -513,8 +513,8 @@ This container is also able to be started as a random UID as long as it is also
 
 ## Container Description
 
-This container will be used to facilitate upload information to the binary analysis worker. It will expose ports within the Docker network, but not outside the Docker network.
-This container is currently only used if Binary Analysis is enabled.
+This container will be used to facilitate upload information to the binary analysis worker as well as to transfer data between containers of the Blackduck system during rapid scanning and full scanning modes. It will expose ports within the Docker network, but not outside the Docker network.
+This container will be running by default.
 
 ## Scalability
 
@@ -525,6 +525,10 @@ There should only be a single instance of this container. It currently cannot be
 This container will need to connect to these other containers/services:
 
 * cfssl
+* scan
+* matchengine
+* bomengine
+* bdba-worker
 
 The container will need to expose port 5671 to other containers that will link to it.
 

diff --git a/README.md b/README.md
@@ -2,9 +2,9 @@
 
 This repository contains orchestration files and documentation for deploying Black Duck Docker containers. 
 
-## Location of Black Duck 2021.8.4 archive:
+## Location of Black Duck 2021.10.0 archive:
 
-https://github.com/blackducksoftware/hub/archive/v2021.8.4.tar.gz
+https://github.com/blackducksoftware/hub/archive/v2021.10.0.tar.gz
 
 NOTE:
 

diff --git a/docker-swarm/README.md b/docker-swarm/README.md
@@ -499,18 +499,16 @@ Note: The username of the certificate used for authentication must exist in the
 
 
 
-# Black Duck Reporting Database
+# Black Duck Reporting Schema
 
-Hub ships with a reporting database. The database port will be exposed to the docker host for connections to the reporting user and reporting database.
+Black Duck includes a reporting schema, called `reporting`, in the `bds_hub` database.
 
 Details:
 
-* Exposed Port: 55436
 * Reporting User Name: blackduck_reporter
-* Reporting Database: bds_hub_report
 * Reporting User Password: initially unset
 
-Before connecting to the reporting database you'll need to set the password for the reporting user. There is a script included in './bin' of the docker-compose directory called 'hub_reportdb_changepassword.sh'. 
+Before connecting to the reporting schema you'll need to set the password for the reporting user. There is a script included in './bin' of the docker-compose directory called 'hub_reportdb_changepassword.sh'. 
 
 To run this script you must:
 
@@ -525,13 +523,6 @@ To run the change password command:
 
 Where 'blackduck' is the new password. This script can also be used to change the password for the reporting user after it has been set.
 
-Once the password is set you should now be able to connect to the reporting database. An example of this with 'psql' is:
-
-```
-psql -U blackduck_reporter -p 55436 -h localhost -W bds_hub_report
-```
-
-This should also work for external connections to the database.
 
 # Scaling Black Duck
 

diff --git a/docker-swarm/bin/bd_get_source_upload_master_key.sh b/docker-swarm/bin/bd_get_source_upload_master_key.sh
@@ -48,11 +48,11 @@ function get_master_key() {
 
     docker exec -i ${container} \
     curl -f --header "X-SEAL-KEY: $seal_key" \
-                           https://uploadcache:9444/api/internal/master-key \
-                           --cert /opt/blackduck/hub/blackduck-upload-cache/security/blackduck-upload-cache-server.crt \
-                           --key /opt/blackduck/hub/blackduck-upload-cache/security/blackduck-upload-cache-server.key \
-                           --cacert /opt/blackduck/hub/blackduck-upload-cache/security/root.crt \
-                           > ${host_path}/${MASTER_KEY_FILE_NAME}
+        https://uploadcache:9444/api/internal/master-key \
+        --cert /opt/blackduck/hub/blackduck-upload-cache/security/blackduck-upload-cache-server.crt \
+        --key /opt/blackduck/hub/blackduck-upload-cache/security/blackduck-upload-cache-server.key \
+        --cacert /opt/blackduck/hub/blackduck-upload-cache/security/root.crt \
+        > ${host_path}/${MASTER_KEY_FILE_NAME}
 
     exitCode=$?
     [ ${exitCode} -ne 0 ] && fail "Unable to get the master key [Container: ${container} | Host path: ${host_path}]"

diff --git a/docker-swarm/bin/hub_add_replication_user.sh b/docker-swarm/bin/hub_add_replication_user.sh
@@ -3,7 +3,7 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.1}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.3}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 
 function fail() {
@@ -14,9 +14,9 @@ function fail() {
 }
 
 function set_container_id() {
-	container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
-								 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
-	return 0
+    container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
+                                 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
+    return 0
 }
 
 # There should be one argument: password

diff --git a/docker-swarm/bin/hub_create_data_dump.sh b/docker-swarm/bin/hub_create_data_dump.sh
@@ -7,7 +7,7 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.1}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.3}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 
 database_name=""
@@ -22,11 +22,15 @@ function fail() {
 }
 
 function set_container_id() {
-	container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
-								 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
+    container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
+                                 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
     return 0
 }
 
+# Returns
+#   0 - database exists
+#   1 - database is bds_hub_report and doesn't exist
+# exits with status "7" if database is not bds_hub_report and does not exist
 function determine_database_readiness() {
     container=$1
     database=$2
@@ -37,11 +41,19 @@ function determine_database_readiness() {
     sleep_count=0
     until [ "$(docker exec -i -u postgres ${container} psql -A -t -c "select count(*) from pg_database where datname = '${database}'" postgres 2> /dev/null)" -eq 1 ] ; do
          sleep_count=$(( ${sleep_count} + 1 ))
-         [ ${sleep_count} -gt ${TIMEOUT} ] && fail "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds." 7
+         if [ ${sleep_count} -gt ${TIMEOUT} ] ; then
+             if [ "${database}" = "bds_hub_report" ] ; then
+                 echo "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds."
+                 return 1
+             else
+                 fail "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds." 7
+             fi
+         fi
          sleep 1
     done
 
     echo "Database is ready [Container: ${container} | Database: ${database}]."
+    return 0
 }
 
 function create_globals() {
@@ -71,19 +83,6 @@ function create_dump() {
     echo "Created database dump [Container: ${container} | Host path: ${host_path} | Database: ${database}]."
 }
 
-function manage_all_databases() {
-    container=$1
-    local_path=$2
-
-    echo "Attempting to manage all databases [Container: ${container} | Path: ${local_path}]."
-
-    manage_globals ${container} ${local_path}
-    manage_database ${container} "bds_hub" ${local_path}
-    manage_database ${container} "bds_hub_report" ${local_path}
-
-    echo "Managed all databases [Container: ${container} | Path: ${local_path}]."
-}
-
 function manage_globals() {
     container=$1
     local_path=$2
@@ -95,17 +94,24 @@ function manage_globals() {
     echo "Managed globals [Container: ${container} | Path: ${local_path}]."
 }
 
+# Returns
+#   0 - database was dumped
+#   1 - database was skipped
 function manage_database() {
     container=$1
     database=$2
     local_path=$3
 
     echo "Attempting to manage database [Container: ${container} | Database: ${database} | Path: ${local_path}]."
 
-    determine_database_readiness ${container} ${database}
-    create_dump ${container} ${local_path} ${database}
-
-    echo "Managed database [Container: ${container} | Database: ${database} | Path: ${local_path}]."
+    if determine_database_readiness ${container} ${database} ; then
+        create_dump ${container} ${local_path} ${database}
+        echo "Managed database [Container: ${container} | Database: ${database} | Path: ${local_path}]."
+        return 0
+    else
+        echo "Skipped database [Container: ${container} | Database: ${database} | Path: ${local_path}]."
+        return 1
+    fi
 }
 
 # There should be two arguments: database name and destination of the path with the name of the dump file.
@@ -183,18 +189,17 @@ then
     # Manage all databases
     echo "Attempting to manage all databases."
 
-    manage_all_databases ${container_id} ${local_absolute_path}
+    manage_globals ${container_id} ${local_absolute_path}
+    manage_database ${container_id} "bds_hub" ${local_absolute_path}
+    manage_database ${container_id} "bds_hub_report" ${local_absolute_path}
 
     echo "Successfully created all database files."
-    echo "Globals SQL file: ${local_absolute_path}/globals.sql"
-    echo "bds_hub database dump file: ${local_absolute_path}/bds_hub.dump"
-    echo "bds_hub_report database dump file: ${local_absolute_path}/bds_hub_report.dump"
 else 
     # Manage a specific database
     echo "Attempting to manage a specific database: ${database_name}."
 
-    manage_database ${container_id} ${database_name} ${local_absolute_path}
-
-    echo "Successfully created database dump file: ${local_absolute_path}/${database_name}.dump"
+    if manage_database ${container_id} ${database_name} ${local_absolute_path} ; then
+        echo "Successfully created database dump file: ${local_absolute_path}/${database_name}.dump"
+    fi
 fi
 
diff --git a/docker-swarm/bin/hub_db_migrate.sh b/docker-swarm/bin/hub_db_migrate.sh
@@ -14,21 +14,21 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.1}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-9.6-1.3}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 SCHEMA_NAME=${HUB_POSTGRES_SCHEMA:-st}
 function fail() {
-	message=$1
-	exit_status=$2
+    message=$1
+    exit_status=$2
 
     echo "${message}"
-	exit ${exit_status}
+    exit ${exit_status}
 }
 
 function set_container_id() {
-	container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
-								 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
-	return 0
+    container_id=( `docker ps -q -f label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION} \
+                                 -f label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}` )
+    return 0
 }
 
 function determine_database_name_validity() {
@@ -113,6 +113,10 @@ function determine_postgresql_readiness() {
     echo "Determined PostgreSQL readiness."
 }
 
+# Returns
+#   0 - database exists
+#   1 - database is bds_hub_report and doesn't exist
+# exits with status "7" if database is not bds_hub_report and does not exist
 function determine_database_readiness() {
     container=$1
     database=$2
@@ -122,12 +126,20 @@ function determine_database_readiness() {
     # Determine if a specific database is ready.
     sleep_count=0
     until [ "$(docker exec -i -u postgres ${container} psql -A -t -c "select count(*) from pg_database where datname = '${database}'" postgres 2> /dev/null)" -eq 1 ] ; do
-        sleep_count=$(( ${sleep_count} + 1 ))
-        [ ${sleep_count} -gt ${TIMEOUT} ] && fail "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds." 8
-        sleep 1
+         sleep_count=$(( ${sleep_count} + 1 ))
+         if [ ${sleep_count} -gt ${TIMEOUT} ] ; then
+             if [ "${database}" = "bds_hub_report" ] ; then
+                 echo "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds."
+                 return 1
+             else
+                 fail "Database ${database} in container ${container} not ready after ${TIMEOUT} seconds." 7
+             fi
+         fi
+         sleep 1
     done
 
     echo "Database is ready [Container: ${container} | Database: ${database}]."
+    return 0
 }
 
 function determine_database_emptiness() {
@@ -192,14 +204,6 @@ EOF
     fi
 }
 
-function validate_database() {
-    container=$1
-    database=$2
-
-    determine_database_readiness ${container} ${database}
-    determine_database_emptiness ${container} ${database}
-}
-
 function migrate_database() {
     container=$1
     database=$2
@@ -209,34 +213,21 @@ function migrate_database() {
     cleanup_database ${container} ${database}
 }
 
-function manage_all_databases() {
-    container=$1
-    directorypath=$2
-
-    echo "Attempting to manage all databases [Container: ${container} | Directory path: ${directorypath}]." 
-
-    validate_database ${container} "bds_hub"
-    validate_database ${container} "bds_hub_report"
-
-    restore_globals ${container} "${directorypath}/globals.sql"
-
-    migrate_database ${container} "bds_hub" "${directorypath}/bds_hub.dump"
-    migrate_database ${container} "bds_hub_report" "${directorypath}/bds_hub_report.dump"
-
-    echo "Managed all databases [Container: ${container} | Directory path: ${directorypath}]."
-}
-
 function manage_database() {
     container=$1
     database=$2
     dump=$3
 
     echo "Attempting to manage database [Container: ${container} | Database: ${database} | Dump: ${dump}]."
 
-    validate_database ${container} ${database}
-    migrate_database ${container} ${database} ${dump}
-
-    echo "Managed database [Container: ${container} | Database: ${database} | Dump: ${dump}]."
+    if determine_database_readiness ${container} ${database} ; then
+        determine_file_validity ${dump}
+        determine_database_emptiness ${container} ${database}
+        migrate_database ${container} ${database} ${dump}
+        echo "Managed database [Container: ${container} | Database: ${database} | Dump: ${dump}]."
+    else
+        echo "Skipped database [Container: ${container} | Database: ${database} | Dump: ${dump}]."
+	fi
 }
 
 # There are two usage options.
@@ -253,17 +244,22 @@ then
     # All databases.
     directory_path="$1"
 
-    determine_file_validity "${directory_path}/globals.sql"
-    determine_file_validity "${directory_path}/bds_hub.dump"
-    determine_file_validity "${directory_path}/bds_hub_report.dump"
-
     determine_docker_path_validity
     determine_docker_daemon_validity
     determine_container_readiness
     determine_singular_container
     determine_postgresql_readiness ${container_id}
 
-    manage_all_databases ${container_id} ${directory_path}
+    echo "Attempting to manage all databases [Container: ${container} | Directory path: ${directorypath}]." 
+
+    determine_file_validity "${directory_path}/globals.sql"
+    restore_globals ${container} "${directorypath}/globals.sql"
+
+    manage_database ${container_id} "bds_hub" "${directory_path}/bds_hub.dump"
+    manage_database ${container_id} "bds_hub_report" "${directory_path}/bds_hub_report.dump"
+
+    echo "Managed all databases [Container: ${container} | Directory path: ${directorypath}]."
+
 elif [ $# -eq "2" ];
 then 
     # Database and a database dump file.
@@ -272,7 +268,6 @@ then
 
     determine_database_name_validity ${database_name}
 
-    determine_file_validity ${dump_file}
     determine_docker_path_validity
     determine_docker_daemon_validity
     determine_container_readiness