From eea538b319553d6862189777a7e5cb607b1d754d Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Fri, 3 Apr 2020 19:27:33 -0400 Subject: [PATCH 01/11] Copy template `properties` files --- symmetricds/Makefile | 21 ++++++++- symmetricds/check-properties.py | 35 +++++++++++++++ symmetricds/template/engine0.properties | 57 +++++++++++++++++++++++++ symmetricds/template/engine1.properties | 51 ++++++++++++++++++++++ 4 files changed, 162 insertions(+), 2 deletions(-) create mode 100755 symmetricds/check-properties.py create mode 100644 symmetricds/template/engine0.properties create mode 100644 symmetricds/template/engine1.properties diff --git a/symmetricds/Makefile b/symmetricds/Makefile index f940e273..cc90e2a7 100644 --- a/symmetricds/Makefile +++ b/symmetricds/Makefile @@ -1,5 +1,10 @@ +SHELL=/bin/bash SYMD_DOWNLOAD_URL=https://sourceforge.net/projects/symmetricds/files/symmetricds/symmetricds-3.8/symmetric-server-3.8.43.zip/download +PROPERTIES=\ + symmetricds/engines/engine0.properties \ + symmetricds/engines/engine1.properties \ + symmetricds.zip: $(if $(shell command -v wget 2> /dev/null),\ wget -O symmetricds.zip ${SYMD_DOWNLOAD_URL},\ @@ -13,8 +18,20 @@ symmetricds: symmetricds.zip touch $@ # https://stackoverflow.com/a/37197276/1556838 -install: symmetricds +install: symmetricds ${PROPERTIES} $(if $(shell command -v java 2> /dev/null),$(info Found `java`),$(error Please install `java`)) +symmetricds/engines/engine%.properties: symmetricds + cp template/$(@F) $(@D)/$(@F) + +check-properties: ${PROPERTIES} + @for prop in $+ ; do \ + ./check-properties.py $$prop || exit 1 ; \ + done + +configure: check-properties + clean: - rm -rf symmetricds.zip symmetricds + -rm -rf symmetricds.zip symmetricds + +.PHONY: clean configure install check-properties diff --git a/symmetricds/check-properties.py b/symmetricds/check-properties.py new file mode 100755 index 00000000..5655ae40 --- /dev/null +++ b/symmetricds/check-properties.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +import sys + + +def check(file): + print(f"Check {file.name}") + dic = {} + for line in file.readlines(): + line = line.strip() + if line.startswith("#"): + continue + if "=" not in line: + continue + comp = line.split("=") + dic[comp[0]] = comp[1] + + for key in ["sync.url", "registration.url", "db.user", "db.password", "db.url"]: + if not assert_key(dic, key): + print(f"{key} is not set in {file.name}", file=sys.stderr) + exit(1) + + +def assert_key(dic, key): + if key not in dic: + return False + if not dic[key].strip(): + return False + return True + + +if __name__ == "__main__": + if len(sys.argv) < 2: + exit(1) + with open(sys.argv[1]) as f: + check(f) diff --git a/symmetricds/template/engine0.properties b/symmetricds/template/engine0.properties new file mode 100644 index 00000000..02db2f7e --- /dev/null +++ b/symmetricds/template/engine0.properties @@ -0,0 +1,57 @@ +# Sync URL where other nodes can contact this node to push/pull data or register. +# http://{hostname}:{port}/sync/engine0 +sync.url= + +# Put the same value with sync.url here +registration.url= + +db.user= +db.password= + +# https://jdbc.postgresql.org/documentation/80/connect.html +db.url=jdbc:postgresql://localhost:5432/mercury + + + + + + + + + + +########################################################################### +# Do not change anything below this line +########################################################################### + +db.driver=org.postgresql.Driver + +# Friendly name to refer to this node from command line +engine.name=engine0 + +# Node group this node belongs to, which defines what it will sync with who. +# Must match the sym_node_group configuration in database. +group.id=group0 + +# External ID for this node, which is any unique identifier you want to use. +external.id=000 + +# How often to run purge job, +job.purge.period.time.ms=7200000 + +# How to run routing (in millis), which puts changes into batches. +# job.routing.period.time.ms=5000 +job.routing.period.time.ms=1000 + +# How often to run push (in millis), which sends changes to other nodes. +job.push.period.time.ms=1000 + +# How often to run pull (in millis), which receives changes from other nodes. +job.pull.period.time.ms=1000 + +# Automatically register new nodes when they request it +# If this is false, accept the registration requests using "symadmin open-registration" command. +auto.registration=true + +# When this node sends an initial load of data to another node, first send table create scripts. +initial.load.create.first=true diff --git a/symmetricds/template/engine1.properties b/symmetricds/template/engine1.properties new file mode 100644 index 00000000..95aa8849 --- /dev/null +++ b/symmetricds/template/engine1.properties @@ -0,0 +1,51 @@ +# Put engine0's sync.url here. +registration.url= + +db.user= +db.password= + +# https://jdbc.postgresql.org/documentation/80/connect.html +db.url= + + + + + + + + + + +########################################################################### +# Do not change anything below this line +########################################################################### + +db.driver=org.postgresql.Driver + +# Friendly name to refer to this node from command line +engine.name=engine1 + +# Node group this node belongs to, which defines what it will sync with who. +# Must match the sym_node_group configuration in database. +group.id=group1 + +# External ID for this node, which is any unique identifier you want to use. +external.id=001 + +# Sync URL where other nodes can contact this node to push/pull data or register. +# http://{hostname}:{port}/sync/{engine.name} +# sync.url=http://localhost:31415/sync/engine1 + + +# How to run routing (in millis), which puts changes into batches. +job.routing.period.time.ms=1000 + +# How often to run push (in millis), which sends changes to other nodes. +job.push.period.time.ms=1000 + +# How often to run pull (in millis), which receives changes from other nodes. +job.pull.period.time.ms=1000 + + +# seems it doesn't work +# auto.reload=true From fbbe01f54edad2971dda274732449c01df697280 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Fri, 3 Apr 2020 19:33:53 -0400 Subject: [PATCH 02/11] Specify default port of SymmetricDS in the template --- symmetricds/template/engine0.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetricds/template/engine0.properties b/symmetricds/template/engine0.properties index 02db2f7e..9dcca0bd 100644 --- a/symmetricds/template/engine0.properties +++ b/symmetricds/template/engine0.properties @@ -1,5 +1,5 @@ # Sync URL where other nodes can contact this node to push/pull data or register. -# http://{hostname}:{port}/sync/engine0 +# http://{hostname}:31415/sync/engine0 sync.url= # Put the same value with sync.url here From 015ae0d64de931c2509373c22867a5b5c0b43f28 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Sun, 5 Apr 2020 13:40:39 -0400 Subject: [PATCH 03/11] Update `properties` templates --- symmetricds/template/engine0.properties | 2 ++ symmetricds/template/engine1.properties | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/symmetricds/template/engine0.properties b/symmetricds/template/engine0.properties index 9dcca0bd..8e192462 100644 --- a/symmetricds/template/engine0.properties +++ b/symmetricds/template/engine0.properties @@ -1,5 +1,6 @@ # Sync URL where other nodes can contact this node to push/pull data or register. # http://{hostname}:31415/sync/engine0 +# sync.url=http://localhost:31415/sync/engine0 sync.url= # Put the same value with sync.url here @@ -9,6 +10,7 @@ db.user= db.password= # https://jdbc.postgresql.org/documentation/80/connect.html +# db.url=jdbc:postgresql://localhost:5432/mercury db.url=jdbc:postgresql://localhost:5432/mercury diff --git a/symmetricds/template/engine1.properties b/symmetricds/template/engine1.properties index 95aa8849..a92885d2 100644 --- a/symmetricds/template/engine1.properties +++ b/symmetricds/template/engine1.properties @@ -1,10 +1,24 @@ # Put engine0's sync.url here. +# registration.url=http://localhost:31415/sync/engine0 registration.url= +# How to get db.user/db.password/db.url for a postgres running on heroku +# +# 1. Run +# heroku config:get DATABASE_URL --app +# +# 2. The output would be the following format: +# postgres://:@ +# +# 3. Your configuration would be: +# db.user= +# db.password= +# db.url=jdbc:postgresql://?sslmode=require + +# Please make sure your db.url starts with "jdbc:postgresql://" and you appended "?sslmode=require" at the end. + db.user= db.password= - -# https://jdbc.postgresql.org/documentation/80/connect.html db.url= From ebbcb23231c102b85ff1fb741ae9cf2eaf6e3367 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Sun, 5 Apr 2020 13:41:23 -0400 Subject: [PATCH 04/11] Do not check sync.url as a mandatory --- symmetricds/check-properties.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/symmetricds/check-properties.py b/symmetricds/check-properties.py index 5655ae40..bcb55b7f 100755 --- a/symmetricds/check-properties.py +++ b/symmetricds/check-properties.py @@ -14,11 +14,24 @@ def check(file): comp = line.split("=") dic[comp[0]] = comp[1] - for key in ["sync.url", "registration.url", "db.user", "db.password", "db.url"]: + for key in ["registration.url", "db.user", "db.password", "db.url"]: if not assert_key(dic, key): print(f"{key} is not set in {file.name}", file=sys.stderr) exit(1) + for key in ["sync.url"]: + if not assert_key_not_empty(dic, key): + print(f"{key} is empty in {file.name}", file=sys.stderr) + exit(1) + + +def assert_key_not_empty(dic, key): + if key not in dic: + return True + if not dic[key].strip(): + return False + return True + def assert_key(dic, key): if key not in dic: From 2ebce3e3b47e4c9d335e5786dbb35ca8797ba872 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Sun, 5 Apr 2020 13:41:56 -0400 Subject: [PATCH 05/11] Implement `make configure` --- symmetricds/Makefile | 5 +++ symmetricds/configure.sql | 74 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 symmetricds/configure.sql diff --git a/symmetricds/Makefile b/symmetricds/Makefile index cc90e2a7..60708f02 100644 --- a/symmetricds/Makefile +++ b/symmetricds/Makefile @@ -29,7 +29,12 @@ check-properties: ${PROPERTIES} ./check-properties.py $$prop || exit 1 ; \ done +SYMD_BIN=symmetricds/bin configure: check-properties + ${SYMD_BIN}/dbsql -e engine0 --sql "" || (echo "Failed to connect to engine0."; exit 1) + ${SYMD_BIN}/dbsql -e engine1 --sql "" || (echo "Failed to connect to engine1."; exit 1) + ${SYMD_BIN}/dbimport -e engine0 configure.sql + clean: -rm -rf symmetricds.zip symmetricds diff --git a/symmetricds/configure.sql b/symmetricds/configure.sql new file mode 100644 index 00000000..ff25852e --- /dev/null +++ b/symmetricds/configure.sql @@ -0,0 +1,74 @@ +------------------------------------------------------------------------------ +-- Clear and load SymmetricDS Configuration +------------------------------------------------------------------------------ + +delete from sym_trigger_router; +delete from sym_trigger; +delete from sym_router; +delete from sym_channel where channel_id in ('main_channel'); +delete from sym_node_group_link; +delete from sym_node_group; +delete from sym_node_host; +delete from sym_node_identity; +delete from sym_node_security; +delete from sym_node; + +------------------------------------------------------------------------------ +-- Channels +------------------------------------------------------------------------------ + +-- Channel "sensor_data" for tables related to items for purchase +insert into sym_channel +(channel_id, processing_order, max_batch_size, enabled, description) +values('main_channel', 1, 100000, 1, 'Main channel for mercury'); + +------------------------------------------------------------------------------ +-- Node Groups +------------------------------------------------------------------------------ + +insert into sym_node_group (node_group_id) values ('group0'); +insert into sym_node_group (node_group_id) values ('group1'); + +------------------------------------------------------------------------------ +-- Node Group Links +------------------------------------------------------------------------------ + +-- Corp sends changes to Store when Store pulls from Corp +insert into sym_node_group_link (source_node_group_id, target_node_group_id, data_event_action) values ('group0', 'group1', 'P'); + +-- Store sends changes to Corp when Store pushes to Corp +insert into sym_node_group_link (source_node_group_id, target_node_group_id, data_event_action) values ('group1', 'group0', 'P'); + +------------------------------------------------------------------------------ +-- Triggers +------------------------------------------------------------------------------ + +insert into sym_trigger +(trigger_id,source_table_name,channel_id,last_update_time,create_time) +values('all_mercury_table_trigger','mercury_*','main_channel',current_timestamp,current_timestamp); + +------------------------------------------------------------------------------ +-- Routers +------------------------------------------------------------------------------ + +-- Default router sends all data from corp to store +insert into sym_router +(router_id,source_node_group_id,target_node_group_id,router_type,create_time,last_update_time) +values('group0_to_1', 'group0', 'group1', 'default',current_timestamp, current_timestamp); + +-- Default router sends all data from store to corp +insert into sym_router +(router_id,source_node_group_id,target_node_group_id,router_type,create_time,last_update_time) +values('group1_to_0', 'group1', 'group0', 'default',current_timestamp, current_timestamp); + +------------------------------------------------------------------------------ +-- Trigger Routers +------------------------------------------------------------------------------ + +insert into sym_trigger_router +(trigger_id,router_id,initial_load_order,last_update_time,create_time) +values('all_mercury_table_trigger','group0_to_1', 100, current_timestamp, current_timestamp); + +insert into sym_trigger_router +(trigger_id,router_id,initial_load_order,last_update_time,create_time) +values('all_mercury_table_trigger','group1_to_0', 100, current_timestamp, current_timestamp); From d3cf4936fba58843509e76c7e38e6b020442d791 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Sun, 5 Apr 2020 15:28:13 -0400 Subject: [PATCH 06/11] Add clean-symd-table --- symmetricds/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/symmetricds/Makefile b/symmetricds/Makefile index 60708f02..99488c86 100644 --- a/symmetricds/Makefile +++ b/symmetricds/Makefile @@ -36,7 +36,11 @@ configure: check-properties ${SYMD_BIN}/dbimport -e engine0 configure.sql -clean: +clean-symd-table: + -${SYMD_BIN}/symadmin -e engine0 uninstall + -${SYMD_BIN}/symadmin -e engine1 uninstall + +clean: clean-symd-table -rm -rf symmetricds.zip symmetricds -.PHONY: clean configure install check-properties +.PHONY: clean clean-symd-table configure install check-properties From 3ad0ffc377e62b300d58e747d7272e4203dbc37a Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Sun, 5 Apr 2020 16:52:56 -0400 Subject: [PATCH 07/11] Change tables to sync from mercury_* to ag_* --- symmetricds/configure.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetricds/configure.sql b/symmetricds/configure.sql index ff25852e..5324df4c 100644 --- a/symmetricds/configure.sql +++ b/symmetricds/configure.sql @@ -45,7 +45,7 @@ insert into sym_node_group_link (source_node_group_id, target_node_group_id, dat insert into sym_trigger (trigger_id,source_table_name,channel_id,last_update_time,create_time) -values('all_mercury_table_trigger','mercury_*','main_channel',current_timestamp,current_timestamp); +values('all_mercury_table_trigger','ag_*','main_channel',current_timestamp,current_timestamp); ------------------------------------------------------------------------------ -- Routers From ecbe94aa70f1ec5e7eb5d4159092f12adcfd32cd Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Tue, 7 Apr 2020 10:59:05 -0400 Subject: [PATCH 08/11] Add `create-sym-table` to configuration script --- symmetricds/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/symmetricds/Makefile b/symmetricds/Makefile index 99488c86..46d3325b 100644 --- a/symmetricds/Makefile +++ b/symmetricds/Makefile @@ -33,6 +33,7 @@ SYMD_BIN=symmetricds/bin configure: check-properties ${SYMD_BIN}/dbsql -e engine0 --sql "" || (echo "Failed to connect to engine0."; exit 1) ${SYMD_BIN}/dbsql -e engine1 --sql "" || (echo "Failed to connect to engine1."; exit 1) + ${SYMD_BIN}/symadmin -e engine0 create-sym-tables || exit 1 ${SYMD_BIN}/dbimport -e engine0 configure.sql From d4440d94e36ada85337a39bfcee00013eb7f6738 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Tue, 7 Apr 2020 11:03:08 -0400 Subject: [PATCH 09/11] Update properties file template --- symmetricds/template/engine0.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/symmetricds/template/engine0.properties b/symmetricds/template/engine0.properties index 8e192462..7f20093b 100644 --- a/symmetricds/template/engine0.properties +++ b/symmetricds/template/engine0.properties @@ -4,9 +4,10 @@ sync.url= # Put the same value with sync.url here +# registration.url=http://localhost:31415/sync/engine0 registration.url= -db.user= +db.user=postgres db.password= # https://jdbc.postgresql.org/documentation/80/connect.html From c73e97eefe00f05cc8116685a466d867c09074c7 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Tue, 7 Apr 2020 11:03:27 -0400 Subject: [PATCH 10/11] Add README.md --- symmetricds/README.md | 112 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 symmetricds/README.md diff --git a/symmetricds/README.md b/symmetricds/README.md new file mode 100644 index 00000000..0dd03abc --- /dev/null +++ b/symmetricds/README.md @@ -0,0 +1,112 @@ +# SymmetricDS + +## Overall +- This guide will describe how to set up two postgres instances running on your local machine/heroku to be synchronized by a SymmetricDS instance running on your local machine. + +- On the local laptop: + - postgres (+mercury django application) + - SymmetricDS + +- On the heroku: + - postgres (+mercury django application) + +## Prerequsite +- A postgres instance running on a heroku instance, which is attached to a mercury django application + +## How to setup +### Step 0 +Make sure you are in `symmetricds` directory under the project root. + +### Step 1 +``` +make install +``` +You will have a new `symmetricds` directory in the current directory. + +### Step 2 +Edit `engine*.properties` files under `symmetricds/engines` and fill the information. + +#### `engine0.properties` +- This is a configuration file for your local machine. +- In this specific setup, `engine0` is where the initial configuration starts. +- Read the comments in the file and fill the following properties: + - sync.url + - registration.url + - db.user + - db.password + - db.url + +#### `engine1.properties` +- This is a configuration file for a heroku. +- Read the comments in the file and fill the following properties: + - registration.url + - db.user + - db.password + - db.url + +#### Get db user/password/url of a postgres on the heroku + +1. Run +``` +heroku config:get DATABASE_URL --app +``` + +2. The output would be the following format: +``` +postgres://:@ +``` + +3. Your configuration would be: +``` +db.user= +db.password= +db.url=jdbc:postgresql://?sslmode=require +``` + +Please make sure your db.url starts with `jdbc:postgresql://` and you appended `?sslmode=require` at the end. + +For example, given the following output, +``` +postgres://abcdeabcdefghi:11d804e1c01111a9c111114fcc528753829a314c30cc51938f4192979102c12@ec2-1-000-00-000.compute-2.amazonaws.com:5432/948f928kjfjv827 +``` +- username: `abcdeabcdefghi` +- password: `11d804e1c01111a9c111114fcc528753829a314c30cc51938f4192979102c12` +- url: `ec2-1-000-00-000.compute-2.amazonaws.com:5432/948f928kjfjv827` + +Your `engine1.properties` should be like: +``` +db.user=abcdeabcdefghi +db.password=11d804e1c01111a9c111114fcc528753829a314c30cc51938f4192979102c12 +db.url=jdbc:postgresql://ec2-1-000-00-000.compute-2.amazonaws.com:5432/948f928kjfjv827?sslmode=require +``` + +### Step 3 +``` +make configure +``` + +### Step 4 +Run SymmetricDS: + +``` +symmetricds/bin/sym +``` + +Give it seoconds for SymmetricDS to finish its initial processes. Once it's finished, every changes will be synchronized from now on. + +### Step 5 +SymmetricDS only synchronizes the changes. That's why you should explicitely trigger an initial load. + +To trigger an initial load (do it once for the initialization): + +local -> heroku +``` +bin/symadmin -e engine0 reload-node 001 +``` + +heroku -> local +``` +bin/symadmin -e engine1 reload-node 000 +``` + +This can be done regardless of whether SymmetricDS is currently running or not. From 6cfa17faa39b207745015c07f1a27c0feb114576 Mon Sep 17 00:00:00 2001 From: Yonguk Jeong Date: Tue, 7 Apr 2020 11:04:05 -0400 Subject: [PATCH 11/11] Do not check db.password --- symmetricds/check-properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/symmetricds/check-properties.py b/symmetricds/check-properties.py index bcb55b7f..e5074707 100755 --- a/symmetricds/check-properties.py +++ b/symmetricds/check-properties.py @@ -14,7 +14,7 @@ def check(file): comp = line.split("=") dic[comp[0]] = comp[1] - for key in ["registration.url", "db.user", "db.password", "db.url"]: + for key in ["registration.url", "db.user", "db.url"]: if not assert_key(dic, key): print(f"{key} is not set in {file.name}", file=sys.stderr) exit(1)