Add update script for PR #121

Also includes some update infrastructure changes. Main difference
is that sql/updates/ scripts are now split into pre- and post- files.
This is needed because table updates need to happen before reloading
functions (some functions checked for accessing right schema at
definition time). Meanwhile trigger updates need to happen after
since triggers must refer to valid functions at definition time.

Makefile

@@ -8,9 +8,12 @@ endif
 
 EXTENSION = timescaledb
 SQL_FILES = $(shell cat sql/setup_load_order.txt sql/functions_load_order.txt sql/init_load_order.txt)
+MANUAL_UPDATE_FILES = $(shell echo sql/setup_main.sql && cat sql/setup_load_order.txt sql/init_load_order.txt)
 EXT_VERSION = $(shell cat timescaledb.control | grep 'default' | sed "s/^.*'\(.*\)'$\/\1/g")
-UPDATE_VERSIONS = $(shell ls -1 sql/updates/*.sql | xargs basename | grep $(EXT_VERSION) | sed "s/\.sql//g")
-UPDATE_FILES = $(shell echo sql/updates/${UPDATE_VERSIONS}.sql && cat sql/functions_load_order.txt)
+UPDATE_VERSIONS = $(shell ls -1 sql/updates/*.sql | xargs basename | grep $(EXT_VERSION) | sed "s/\.sql//g"| sed "s/^pre-//g;s/^post-//g"| head -1)
+UPDATE_VERSIONS_PRE = $(shell ls -1 sql/updates/pre-$(UPDATE_VERSIONS).sql)
+UPDATE_VERSIONS_POST = $(shell ls -1 sql/updates/post-$(UPDATE_VERSIONS).sql)
+UPDATE_FILES = $(shell echo ${UPDATE_VERSIONS_PRE} && cat sql/functions_load_order.txt && echo ${UPDATE_VERSIONS_POST} )
 UPDATE_FILE = sql/$(EXTENSION)--$(UPDATE_VERSIONS).sql
 EXT_GIT_COMMIT := $(shell git describe --abbrev=4 --dirty --always --tags || echo $(EXT_GIT_COMMIT))
 EXT_SQL_FILE = sql/$(EXTENSION)--$(EXT_VERSION).sql
@@ -127,4 +130,7 @@ typedef.list: clean $(OBJS)
 pgindent: typedef.list
 	pgindent --typedef=typedef.list
 
+manualupdate:
+	git diff origin/master $(MANUAL_UPDATE_FILES)
+
 .PHONY: check-sql-files all

scripts/test_updates.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e
+set -o pipefail
 
 PGTEST_TMPDIR=${PGTEST_TMPDIR:-/tmp}
 UPDATE_PG_PORT=${UPDATE_PG_PORT:-6432}
@@ -32,25 +33,34 @@ docker run -d --name timescaledb-orig -v ${PGTEST_TMPDIR}/pg_data:/var/lib/postg
 
 wait_for_pg ${UPDATE_PG_PORT}
 
+echo "Executing setup script on 0.1.0"
 psql -h localhost -U postgres -p ${UPDATE_PG_PORT} -f test/sql/updates/setup.sql
 docker rm -vf timescaledb-orig
 
 docker run -d --name timescaledb-updated -v /tmp/pg_data:/var/lib/postgresql/data -p ${UPDATE_PG_PORT}:5432 update_test:latest
 
 wait_for_pg ${UPDATE_PG_PORT}
 
+echo "Executing ALTER EXTENSION timescaledb UPDATE"
 psql -h localhost -U postgres -d single -p ${UPDATE_PG_PORT} -c "ALTER EXTENSION timescaledb UPDATE"
 
 
 wait_for_pg ${CLEAN_PG_PORT}
 
+echo "Executing setup script on new version"
 psql -h localhost -U postgres -p ${CLEAN_PG_PORT} -f test/sql/updates/setup.sql
-psql -h localhost -U postgres -d single -p ${UPDATE_PG_PORT} -f test/sql/updates/test-0.1.1.sql > /tmp/updated.out
-psql -h localhost -U postgres -d single -p ${CLEAN_PG_PORT} -f test/sql/updates/test-0.1.1.sql > /tmp/clean.out
 
-docker rm -f timescaledb-updated timescaledb-clean || rm -rf  ${PGTEST_TMPDIR}/pg_data ${PGTEST_TMPDIR}/pg_data_clean
+echo "Restarting clean container"
+#below is needed so the clean container looks like updated, which has been restarted after the setup script
+#(especially needed for sequences which might otherwise be in different states -- e.g. some backends may have reserved batches)
+docker rm -vf timescaledb-clean
+docker run -d --name timescaledb-clean -v /tmp/pg_data_clean:/var/lib/postgresql/data -p ${CLEAN_PG_PORT}:5432 update_test:latest
+wait_for_pg ${CLEAN_PG_PORT}
 
-diff ${PGTEST_TMPDIR}/clean.out ${PGTEST_TMPDIR}/updated.out > ${PGTEST_TMPDIR}/update_test.output
-cat ${PGTEST_TMPDIR}/update_test.output
+echo "Testing"
+psql -X -v ECHO=ALL -h localhost -U postgres -d single -p ${UPDATE_PG_PORT} -f test/sql/updates/test-0.1.1.sql > /tmp/updated.out
+psql -X -v ECHO=ALL -h localhost -U postgres -d single -p ${CLEAN_PG_PORT} -f test/sql/updates/test-0.1.1.sql > /tmp/clean.out
+
+docker rm -f timescaledb-updated timescaledb-clean || rm -rf  ${PGTEST_TMPDIR}/pg_data ${PGTEST_TMPDIR}/pg_data_clean
 
-cmp ${PGTEST_TMPDIR}/clean.out ${PGTEST_TMPDIR}/updated.out > ${PGTEST_TMPDIR}/update_test.output
+diff ${PGTEST_TMPDIR}/clean.out ${PGTEST_TMPDIR}/updated.out | tee ${PGTEST_TMPDIR}/update_test.output

sql/updates/README.md

@@ -1,2 +1,9 @@
-Folder for update SQL-scripts named oldversion--newversion (0.1.0--0.1.1)
-It is assumed that there is only one file matching the newversion.
+Folder for update SQL-scripts named pre-oldversion--newversion or post-oldversion--newversion
+
+pre files are run before reloading the functions. These should include new
+tables, constraints, indexes, etc.
+
+post functions are run after reloading the functions. These include
+any entities depending on functions-- e.g. triggers.
+
+It is assumed that there is only one file for pre or post matching the newversion.

sql/updates/post-0.1.0--0.1.1-dev.sql

@@ -0,0 +1,3 @@
+CREATE EVENT TRIGGER ddl_drop_trigger
+ON sql_drop
+EXECUTE PROCEDURE _timescaledb_internal.ddl_process_drop_trigger();

sql/updates/pre-0.1.0--0.1.1-dev.sql

@@ -0,0 +1,17 @@
+ALTER TABLE _timescaledb_catalog.hypertable ADD UNIQUE (id, schema_name);
+
+ALTER TABLE _timescaledb_catalog.hypertable_index
+DROP CONSTRAINT hypertable_index_hypertable_id_fkey,
+ADD CONSTRAINT hypertable_index_hypertable_id_fkey
+FOREIGN KEY (hypertable_id, main_schema_name)
+REFERENCES _timescaledb_catalog.hypertable(id, schema_name)
+ON UPDATE CASCADE
+ON DELETE CASCADE;
+
+ALTER TABLE _timescaledb_catalog.chunk_index
+DROP CONSTRAINT chunk_index_main_schema_name_fkey,
+ADD CONSTRAINT chunk_index_main_schema_name_fkey
+FOREIGN KEY (main_schema_name, main_index_name) 
+REFERENCES _timescaledb_catalog.hypertable_index(main_schema_name, main_index_name) 
+ON UPDATE CASCADE 
+ON DELETE CASCADE;

test/sql/updates/test-0.1.1.sql

@@ -1,2 +1,11 @@
-\d+ _timescaledb_catalog.hypertable;
-\d+ _timescaledb_catalog.hypertable_index;
+\d+ _timescaledb_catalog.*;
+\df+ _timescaledb_internal.*;
+\dy
+
+SELECT count(*)
+  FROM pg_depend
+ WHERE refclassid = 'pg_extension'::regclass
+     AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb');
+
+-- The list of tables configured to be dumped.
+SELECT unnest(extconfig)::regclass::text c from pg_extension where extname='timescaledb' ORDER BY c;