From 9415dda3374dd6937662955fe9d36ed432b0d6a3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 19 Nov 2019 16:44:00 +0000 Subject: [PATCH 01/11] Add dev script to generate full SQL schemas. --- scripts-dev/generate_full_schema.sh | 186 ++++++++++++++++++ scripts/synapse_port_db | 5 +- .../main/schema/full_schemas/README.md | 13 ++ .../main/schema/full_schemas/README.txt | 19 -- 4 files changed, 203 insertions(+), 20 deletions(-) create mode 100755 scripts-dev/generate_full_schema.sh create mode 100644 synapse/storage/data_stores/main/schema/full_schemas/README.md delete mode 100644 synapse/storage/data_stores/main/schema/full_schemas/README.txt diff --git a/scripts-dev/generate_full_schema.sh b/scripts-dev/generate_full_schema.sh new file mode 100755 index 000000000000..94c61173a8f9 --- /dev/null +++ b/scripts-dev/generate_full_schema.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# +# This script generates SQL files for creating a brand new Synapse DB with the latest +# schema, on both SQLite3 and Postgres. +# +# It does so by having Synapse generate an up-to-date SQLite DB, then running +# synapse_port_db to convert it to Postgres. It then dumps the contents of both. + +POSTGRES_HOST="localhost" +POSTGRES_DB_NAME="synapse_full_schema" +SQLITE_DB_FILE_NAME="full_schema_sqlite.db" + +SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" +POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" +OUTPUT_DIR=$(pwd) + +usage() { + echo "Usage: $0 -p [-c] [-v] [-o] [-h]" + echo + echo "-p " + echo " Username to connect to local postgres instance. The password will be requested" + echo " during script execution." + echo "-c" + echo " Whether to enable coverage tracking. Useful for CI runs." + echo "-v" + echo " Suppress warning about requiring the use of a virtualenv." + echo "-o" + echo " Directory to output full schema files to. Defaults to the current directory." + echo "-h" + echo " Display this help text." +} + +while getopts "p:cvo:h" opt; do + case $opt in + p) + POSTGRES_USERNAME=$OPTARG + ;; + c) + COVERAGE=1 + ;; + v) + NO_VIRTUALENV=1 + ;; + o) + command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1) + OUTPUT_DIR="$(realpath "$OPTARG")" + ;; + h) + usage + exit + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit + ;; + esac +done + +# Check that the script is running with a virtualenv enabled +if [ -z ${NO_VIRTUALENV+x} ] && [ -z ${VIRTUAL_ENV+x} ]; then + echo "It is highly recommended to run this script with a virtualenv activated. Exiting now." + echo "If you wish to suppress this warning, please run with the -v option." + exit 1 +fi + +if [ -z ${POSTGRES_USERNAME+x} ]; then + echo "No postgres username supplied" + usage + exit 1 +fi + +read -rsp "Postgres password for '$POSTGRES_USERNAME':" POSTGRES_PASSWORD +echo "" + +set -xe + +# cd to root of the synapse directory +cd "$(dirname "$0")/.." + +# Install required dependencies +echo "Installing dependencies..." +if [ -z ${COVERAGE+x} ]; then + # No coverage needed + pip install psycopg2 +else + # Coverage desired + pip install psycopg2 coverage coverage-enable-subprocess +fi + +# Install Synapse itself. This won't update any libraries. +pip install -e . + +# Create temporary SQLite and Postgres homeserver db configs and key file +KEY_FILE=$(mktemp) +SQLITE_CONFIG=$(mktemp) +POSTGRES_CONFIG=$(mktemp) +cat > "$SQLITE_CONFIG" < "$POSTGRES_CONFIG" < "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" + +echo "Dumping Postgres schema to '$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." +pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" + +echo "Cleaning up temporary files and databases..." +rm "$SQLITE_DB_FILE_NAME" +rm "$POSTGRES_CONFIG" +rm "$SQLITE_CONFIG" +rm "$KEY_FILE" +dropdb $POSTGRES_DB_NAME + +# Remove last pesky instance of this table from the output +sed -i '/applied_module_schemas/d' "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" + +echo "Done! Files dumped to: $OUTPUT_DIR" diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 0d3321682c6e..f24b8ffe6741 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -782,7 +782,10 @@ class Porter(object): def _setup_state_group_id_seq(self): def r(txn): txn.execute("SELECT MAX(id) FROM state_groups") - next_id = txn.fetchone()[0] + 1 + curr_id = txn.fetchone()[0] + if not curr_id: + return + next_id = curr_id + 1 txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,)) return self.postgres_store.runInteraction("setup_state_group_id_seq", r) diff --git a/synapse/storage/data_stores/main/schema/full_schemas/README.md b/synapse/storage/data_stores/main/schema/full_schemas/README.md new file mode 100644 index 000000000000..f3e922c15982 --- /dev/null +++ b/synapse/storage/data_stores/main/schema/full_schemas/README.md @@ -0,0 +1,13 @@ +# Building full schema dumps + +These schemas need to be made from a database that has had all background updates run. + +To do so, use `scripts-dev/generate_full_schema.sh`. This will +produce `full.sql.postgres ` and `full.sql.sqlite` files. + +Ensure postgres is installed and your user has the ability to run bash commands +such as `createdb`. + +``` +/scripts-dev/generate_full_schema.sh -p postgres_username -o output_dir/ +``` diff --git a/synapse/storage/data_stores/main/schema/full_schemas/README.txt b/synapse/storage/data_stores/main/schema/full_schemas/README.txt deleted file mode 100644 index d3f640134408..000000000000 --- a/synapse/storage/data_stores/main/schema/full_schemas/README.txt +++ /dev/null @@ -1,19 +0,0 @@ -Building full schema dumps -========================== - -These schemas need to be made from a database that has had all background updates run. - -Postgres --------- - -$ pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner $DATABASE_NAME| sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > full.sql.postgres - -SQLite ------- - -$ sqlite3 $DATABASE_FILE ".schema" > full.sql.sqlite - -After ------ - -Delete the CREATE statements for "sqlite_stat1", "schema_version", "applied_schema_deltas", and "applied_module_schemas". \ No newline at end of file From f6eff4bc2521362443190f1699e2027e58607862 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 20 Nov 2019 18:24:19 +0000 Subject: [PATCH 02/11] Add changelog --- changelog.d/6394.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6394.feature diff --git a/changelog.d/6394.feature b/changelog.d/6394.feature new file mode 100644 index 000000000000..1a0e8845ad69 --- /dev/null +++ b/changelog.d/6394.feature @@ -0,0 +1 @@ +Add a develop script to generate full SQL schemas. \ No newline at end of file From a6e0c40f2b930bf494466f499d3996854805cda6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 2 Dec 2019 17:38:40 +0000 Subject: [PATCH 03/11] Apply suggestions from code review Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- scripts-dev/generate_full_schema.sh | 4 ++-- .../storage/data_stores/main/schema/full_schemas/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts-dev/generate_full_schema.sh b/scripts-dev/generate_full_schema.sh index 94c61173a8f9..aec6050b2d73 100755 --- a/scripts-dev/generate_full_schema.sh +++ b/scripts-dev/generate_full_schema.sh @@ -7,7 +7,7 @@ # synapse_port_db to convert it to Postgres. It then dumps the contents of both. POSTGRES_HOST="localhost" -POSTGRES_DB_NAME="synapse_full_schema" +POSTGRES_DB_NAME="synapse_full_schema.$$" SQLITE_DB_FILE_NAME="full_schema_sqlite.db" SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" @@ -21,7 +21,7 @@ usage() { echo " Username to connect to local postgres instance. The password will be requested" echo " during script execution." echo "-c" - echo " Whether to enable coverage tracking. Useful for CI runs." + echo " Enable coverage tracking. Useful for CI runs." echo "-v" echo " Suppress warning about requiring the use of a virtualenv." echo "-o" diff --git a/synapse/storage/data_stores/main/schema/full_schemas/README.md b/synapse/storage/data_stores/main/schema/full_schemas/README.md index f3e922c15982..8af2f700a462 100644 --- a/synapse/storage/data_stores/main/schema/full_schemas/README.md +++ b/synapse/storage/data_stores/main/schema/full_schemas/README.md @@ -9,5 +9,5 @@ Ensure postgres is installed and your user has the ability to run bash commands such as `createdb`. ``` -/scripts-dev/generate_full_schema.sh -p postgres_username -o output_dir/ +./scripts-dev/generate_full_schema.sh -p postgres_username -o output_dir/ ``` From 9b6e8e237aee6fab12cfb7893137f61d583927f7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 2 Dec 2019 18:21:13 +0000 Subject: [PATCH 04/11] Address review comments --- scripts-dev/generate_full_schema.sh | 87 +++++++++++++++++------------ 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/scripts-dev/generate_full_schema.sh b/scripts-dev/generate_full_schema.sh index aec6050b2d73..bbb6247772b4 100755 --- a/scripts-dev/generate_full_schema.sh +++ b/scripts-dev/generate_full_schema.sh @@ -14,31 +14,39 @@ SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" OUTPUT_DIR=$(pwd) +REQUIRED_DEPS=("matrix-synapse" "psycopg2") + usage() { - echo "Usage: $0 -p [-c] [-v] [-o] [-h]" + echo "Usage: $0 -p -o [-c] [-n] [-h]" echo echo "-p " echo " Username to connect to local postgres instance. The password will be requested" echo " during script execution." echo "-c" - echo " Enable coverage tracking. Useful for CI runs." - echo "-v" + echo " CI mode. Enables coverage tracking and prints every command that the script runs." + echo "-n" echo " Suppress warning about requiring the use of a virtualenv." - echo "-o" - echo " Directory to output full schema files to. Defaults to the current directory." + echo "-o " + echo " Directory to output full schema files to." echo "-h" echo " Display this help text." } -while getopts "p:cvo:h" opt; do +while getopts "p:cno:h" opt; do case $opt in p) POSTGRES_USERNAME=$OPTARG ;; c) + # Print all commands that are being executed + set -x + + # Modify required dependencies for coverage + REQUIRED_DEPS+=("coverage" "coverage-enable-subprocess") + COVERAGE=1 ;; - v) + n) NO_VIRTUALENV=1 ;; o) @@ -57,48 +65,56 @@ while getopts "p:cvo:h" opt; do esac done +# Check that required dependencies are installed +unsatisfied_requirements=() +for dep in "${REQUIRED_DEPS[@]}"; do + pip show "$dep" --quiet || unsatisfied_requirements+=("$dep") +done +if [ ! ${#unsatisfied_requirements} -eq 0 ]; then + echo "Please install the following python packages: ${unsatisfied_requirements[*]}" + exit 1 +fi + # Check that the script is running with a virtualenv enabled -if [ -z ${NO_VIRTUALENV+x} ] && [ -z ${VIRTUAL_ENV+x} ]; then +if [ -z "$NO_VIRTUALENV" ] && [ -z "$VIRTUAL_ENV" ]; then echo "It is highly recommended to run this script with a virtualenv activated. Exiting now." - echo "If you wish to suppress this warning, please run with the -v option." + echo "If you wish to suppress this warning, please run with the -n option." exit 1 fi -if [ -z ${POSTGRES_USERNAME+x} ]; then +if [ -z "$POSTGRES_USERNAME" ]; then echo "No postgres username supplied" usage exit 1 fi +if [ -z "$OUTPUT_DIR" ]; then + echo "No output directory supplied" + usage + exit 1 +fi + read -rsp "Postgres password for '$POSTGRES_USERNAME':" POSTGRES_PASSWORD echo "" -set -xe +# Exit immediately if a command fails +set -e # cd to root of the synapse directory cd "$(dirname "$0")/.." -# Install required dependencies -echo "Installing dependencies..." -if [ -z ${COVERAGE+x} ]; then - # No coverage needed - pip install psycopg2 -else - # Coverage desired - pip install psycopg2 coverage coverage-enable-subprocess -fi +# Create temporary SQLite and Postgres homeserver db configs and key file +TMPDIR=$(mktemp -d) +KEY_FILE=$TMPDIR/test.signing.key # default Synapse signing key path +SQLITE_CONFIG=$TMPDIR/sqlite.conf +POSTGRES_CONFIG=$TMPDIR/postgres.conf -# Install Synapse itself. This won't update any libraries. -pip install -e . +# Ensure these files are delete on script exit +trap 'rm -rf $TMPDIR' EXIT -# Create temporary SQLite and Postgres homeserver db configs and key file -KEY_FILE=$(mktemp) -SQLITE_CONFIG=$(mktemp) -POSTGRES_CONFIG=$(mktemp) cat > "$SQLITE_CONFIG" < "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" From be1464878d846c4a3e30c01b838fe84eb3694005 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 2 Dec 2019 18:23:02 +0000 Subject: [PATCH 05/11] Factor out fix for synapse_port_db --- scripts/synapse_port_db | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index f24b8ffe6741..0d3321682c6e 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -782,10 +782,7 @@ class Porter(object): def _setup_state_group_id_seq(self): def r(txn): txn.execute("SELECT MAX(id) FROM state_groups") - curr_id = txn.fetchone()[0] - if not curr_id: - return - next_id = curr_id + 1 + next_id = txn.fetchone()[0] + 1 txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,)) return self.postgres_store.runInteraction("setup_state_group_id_seq", r) From 6f818ef1cc8ab144a9c647f5b5fc2de579c13a19 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 2 Dec 2019 18:42:28 +0000 Subject: [PATCH 06/11] More fixes --- scripts-dev/generate_full_schema.sh | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scripts-dev/generate_full_schema.sh b/scripts-dev/generate_full_schema.sh index bbb6247772b4..b8351d966bde 100755 --- a/scripts-dev/generate_full_schema.sh +++ b/scripts-dev/generate_full_schema.sh @@ -8,7 +8,6 @@ POSTGRES_HOST="localhost" POSTGRES_DB_NAME="synapse_full_schema.$$" -SQLITE_DB_FILE_NAME="full_schema_sqlite.db" SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" @@ -107,6 +106,7 @@ cd "$(dirname "$0")/.." TMPDIR=$(mktemp -d) KEY_FILE=$TMPDIR/test.signing.key # default Synapse signing key path SQLITE_CONFIG=$TMPDIR/sqlite.conf +SQLITE_DB=$TMPDIR/homeserver.db POSTGRES_CONFIG=$TMPDIR/postgres.conf # Ensure these files are delete on script exit @@ -115,6 +115,7 @@ trap 'rm -rf $TMPDIR' EXIT cat > "$SQLITE_CONFIG" < "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT echo "Dumping Postgres schema to '$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" -echo "Cleaning up temporary files and databases..." -rm "$SQLITE_DB_FILE_NAME" -rm "$POSTGRES_CONFIG" -rm "$SQLITE_CONFIG" -rm "$KEY_FILE" +echo "Cleaning up temporary Postgres database..." dropdb $POSTGRES_DB_NAME # Remove last pesky instance of this table from the output From 9b7d57456bae0a17ff06c9e39201e2966c5e25f2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 2 Dec 2019 18:48:01 +0000 Subject: [PATCH 07/11] Rename script to not conflict with generate_sample_config --- .../{generate_full_schema.sh => make_full_schema.sh} | 0 .../storage/data_stores/main/schema/full_schemas/README.md | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename scripts-dev/{generate_full_schema.sh => make_full_schema.sh} (100%) diff --git a/scripts-dev/generate_full_schema.sh b/scripts-dev/make_full_schema.sh similarity index 100% rename from scripts-dev/generate_full_schema.sh rename to scripts-dev/make_full_schema.sh diff --git a/synapse/storage/data_stores/main/schema/full_schemas/README.md b/synapse/storage/data_stores/main/schema/full_schemas/README.md index 8af2f700a462..bbd3f1860487 100644 --- a/synapse/storage/data_stores/main/schema/full_schemas/README.md +++ b/synapse/storage/data_stores/main/schema/full_schemas/README.md @@ -2,12 +2,12 @@ These schemas need to be made from a database that has had all background updates run. -To do so, use `scripts-dev/generate_full_schema.sh`. This will -produce `full.sql.postgres ` and `full.sql.sqlite` files. +To do so, use `scripts-dev/make_full_schema.sh`. This will produce +`full.sql.postgres ` and `full.sql.sqlite` files. Ensure postgres is installed and your user has the ability to run bash commands such as `createdb`. ``` -./scripts-dev/generate_full_schema.sh -p postgres_username -o output_dir/ +./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ ``` From cb78f367731a319ef4e499ba915505d6e883eaa1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Dec 2019 13:09:49 +0000 Subject: [PATCH 08/11] wip Address review comments --- scripts-dev/make_full_schema.sh | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index b8351d966bde..59fd248f4e43 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -23,15 +23,13 @@ usage() { echo " during script execution." echo "-c" echo " CI mode. Enables coverage tracking and prints every command that the script runs." - echo "-n" - echo " Suppress warning about requiring the use of a virtualenv." echo "-o " echo " Directory to output full schema files to." echo "-h" echo " Display this help text." } -while getopts "p:cno:h" opt; do +while getopts "p:co:h" opt; do case $opt in p) POSTGRES_USERNAME=$OPTARG @@ -45,9 +43,6 @@ while getopts "p:cno:h" opt; do COVERAGE=1 ;; - n) - NO_VIRTUALENV=1 - ;; o) command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1) OUTPUT_DIR="$(realpath "$OPTARG")" @@ -69,18 +64,11 @@ unsatisfied_requirements=() for dep in "${REQUIRED_DEPS[@]}"; do pip show "$dep" --quiet || unsatisfied_requirements+=("$dep") done -if [ ! ${#unsatisfied_requirements} -eq 0 ]; then +if [ ${#unsatisfied_requirements} -ne 0 ]; then echo "Please install the following python packages: ${unsatisfied_requirements[*]}" exit 1 fi -# Check that the script is running with a virtualenv enabled -if [ -z "$NO_VIRTUALENV" ] && [ -z "$VIRTUAL_ENV" ]; then - echo "It is highly recommended to run this script with a virtualenv activated. Exiting now." - echo "If you wish to suppress this warning, please run with the -n option." - exit 1 -fi - if [ -z "$POSTGRES_USERNAME" ]; then echo "No postgres username supplied" usage @@ -127,7 +115,6 @@ database: # Suppress the key server warning. trusted_key_servers: [] -suppress_key_server_warning: true EOF cat > "$POSTGRES_CONFIG" < Date: Tue, 10 Dec 2019 11:24:43 +0000 Subject: [PATCH 09/11] Change default output dir to ./full_schema --- scripts-dev/make_full_schema.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 59fd248f4e43..4d5d9bb2cb81 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -11,7 +11,7 @@ POSTGRES_DB_NAME="synapse_full_schema.$$" SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" -OUTPUT_DIR=$(pwd) +OUTPUT_DIR="$(pwd)/full_schema" REQUIRED_DEPS=("matrix-synapse" "psycopg2") @@ -81,6 +81,9 @@ if [ -z "$OUTPUT_DIR" ]; then exit 1 fi +# Create the output directory if it doesn't exist +mkdir -p "$OUTPUT_DIR" + read -rsp "Postgres password for '$POSTGRES_USERNAME':" POSTGRES_PASSWORD echo "" From 6a93bd16b972a196f16afd7dc94e886196e265e6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 10 Dec 2019 13:23:39 +0000 Subject: [PATCH 10/11] Small cleanup and remove applied_module_schemas sed --- scripts-dev/make_full_schema.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 4d5d9bb2cb81..1f0016ab3d3a 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -84,7 +84,7 @@ fi # Create the output directory if it doesn't exist mkdir -p "$OUTPUT_DIR" -read -rsp "Postgres password for '$POSTGRES_USERNAME':" POSTGRES_PASSWORD +read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD echo "" # Exit immediately if a command fails @@ -172,16 +172,13 @@ DROP TABLE applied_module_schemas; sqlite3 "$SQLITE_DB" <<< "$SQL" psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL" -echo "Dumping SQLite3 schema to '$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..." -sqlite3 "$SQLITE_DB_FILE_NAME" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" +echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..." +sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" -echo "Dumping Postgres schema to '$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." +echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" echo "Cleaning up temporary Postgres database..." dropdb $POSTGRES_DB_NAME -# Remove last pesky instance of this table from the output -sed -i '/applied_module_schemas/d' "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" - echo "Done! Files dumped to: $OUTPUT_DIR" From d125d18a9c286537226769d08546599f0a6d4098 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 10 Dec 2019 13:30:11 +0000 Subject: [PATCH 11/11] Make output directory mandatory --- scripts-dev/make_full_schema.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 1f0016ab3d3a..60e8970a35a0 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -11,11 +11,11 @@ POSTGRES_DB_NAME="synapse_full_schema.$$" SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" -OUTPUT_DIR="$(pwd)/full_schema" REQUIRED_DEPS=("matrix-synapse" "psycopg2") usage() { + echo echo "Usage: $0 -p -o [-c] [-n] [-h]" echo echo "-p " @@ -52,7 +52,7 @@ while getopts "p:co:h" opt; do exit ;; \?) - echo "Invalid option: -$OPTARG" >&2 + echo "ERROR: Invalid option: -$OPTARG" >&2 usage exit ;;