Add telemetry job and turn off default jobs in tests

This adds the telemetry job to the job scheduler. Telemetry is
scheduled to run every 24 hours with a 1 hour exponential backoff
retry period. Additional fixes related to the  telemetry job:

- Add separate ID space to the bgw_job table for default jobs. We do not dump this ID space inside pg_dump to prevent job insertion conflicts.
- Add check to update scripts for default jobs.
- Change shmem_callback so that it doesn't modify state since state transitions are not atomic with respect to interrupts and interrupt callbacks.
- Disable default telemetry job in regression and docker tests.

scripts/docker-build.sh

@@ -31,7 +31,7 @@ else
 fi
 
 # Build and install the extension with debug symbols and assertions
-docker exec -u root -it ${BUILD_CONTAINER_NAME} /bin/bash -c "cp -a /src/{src,sql,test,scripts,version.config,CMakeLists.txt,timescaledb.control.in} /build/ && cd build/debug && cmake -DCMAKE_BUILD_TYPE=Debug .. && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
+docker exec -u root -it ${BUILD_CONTAINER_NAME} /bin/bash -c "cp -a /src/{src,sql,test,scripts,version.config,CMakeLists.txt,timescaledb.control.in} /build/ && cd build/debug && cmake -DCMAKE_BUILD_TYPE=Debug .. && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
 
 docker commit -a $USER -m "TimescaleDB development image" ${BUILD_CONTAINER_NAME} ${IMAGE_NAME}:${TAG_NAME}
 

sql/bgw_scheduler.sql

@@ -1,16 +1,20 @@
 CREATE OR REPLACE FUNCTION _timescaledb_internal.restart_background_workers()
-RETURNS BOOL 
+RETURNS BOOL
 AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_restart'
 LANGUAGE C VOLATILE;
 
 CREATE OR REPLACE FUNCTION _timescaledb_internal.stop_background_workers()
-RETURNS BOOL 
+RETURNS BOOL
 AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_stop'
 LANGUAGE C VOLATILE;
 
 CREATE OR REPLACE FUNCTION _timescaledb_internal.start_background_workers()
-RETURNS BOOL 
+RETURNS BOOL
 AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_start'
 LANGUAGE C VOLATILE;
 
 SELECT _timescaledb_internal.start_background_workers();
+
+INSERT INTO _timescaledb_config.bgw_job (id, application_name, job_type, schedule_INTERVAL, max_runtime, max_retries, retry_period) VALUES
+(1, 'Telemetry Reporter', 'telemetry_and_version_check_if_enabled', INTERVAL '24h', INTERVAL '100s', -1, INTERVAL '1h')
+ON CONFLICT (id) DO NOTHING;

sql/pre_install/tables.sql

@@ -146,18 +146,26 @@ CREATE INDEX IF NOT EXISTS chunk_index_hypertable_id_hypertable_index_name_idx
 ON _timescaledb_catalog.chunk_index(hypertable_id, hypertable_index_name);
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_index', '');
 
+-- Default jobs are given the id space [1,1000). User-installed jobs and any jobs created inside tests
+-- are given the id space [1000, INT_MAX). That way, we do not pg_dump jobs that are always default-installed
+-- inside other .sql scripts. This avoids insertion conflicts during pg_restore.
+
+CREATE SEQUENCE IF NOT EXISTS _timescaledb_config.bgw_job_id_seq MINVALUE 1000;
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job_id_seq', '');
+
 CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_job (
-    id                  SERIAL      PRIMARY KEY,
+    id                  INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_config.bgw_job_id_seq'),
     application_name    NAME        NOT NULL,
     job_type            NAME        NOT NULL,
     schedule_interval   INTERVAL    NOT NULL,
     max_runtime         INTERVAL    NOT NULL,
     max_retries         INT         NOT NULL,
     retry_period        INTERVAL    NOT NULL,
-    CONSTRAINT  valid_job_type CHECK (job_type IN ('update_check'))
+    CONSTRAINT  valid_job_type CHECK (job_type IN ('telemetry_and_version_check_if_enabled'))
 );
-SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', '');
-SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_config.bgw_job','id'), '');
+ALTER SEQUENCE _timescaledb_config.bgw_job_id_seq OWNED BY _timescaledb_config.bgw_job.id;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE id >= 1000');
 
 CREATE TABLE IF NOT EXISTS _timescaledb_internal.bgw_job_stat (
     job_id                  INT         PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,

sql/updates/0.11.0--0.11.1-dev.sql

@@ -14,18 +14,22 @@ DROP FUNCTION _timescaledb_internal.insert_blocker_trigger_add(REGCLASS);
 CREATE SCHEMA IF NOT EXISTS _timescaledb_config;
 GRANT USAGE ON SCHEMA _timescaledb_config TO PUBLIC;
 
+CREATE SEQUENCE IF NOT EXISTS _timescaledb_config.bgw_job_id_seq MINVALUE 1000;
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job_id_seq', '');
+
 CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_job (
-    id                  SERIAL      PRIMARY KEY,
+    id                  INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_config.bgw_job_id_seq'),
     application_name    NAME        NOT NULL,
     job_type            NAME        NOT NULL,
     schedule_interval   INTERVAL    NOT NULL,
     max_runtime         INTERVAL    NOT NULL,
     max_retries         INT         NOT NULL,
     retry_period        INTERVAL    NOT NULL,
-    CONSTRAINT  valid_job_type CHECK (job_type IN ('update_check'))
+    CONSTRAINT  valid_job_type CHECK (job_type IN ('telemetry_and_version_check_if_enabled'))
 );
-SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', '');
-SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_config.bgw_job','id'), '');
+ALTER SEQUENCE _timescaledb_config.bgw_job_id_seq OWNED BY _timescaledb_config.bgw_job.id;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE id >= 1000');
 
 CREATE TABLE IF NOT EXISTS _timescaledb_internal.bgw_job_stat (
     job_id                  INT         PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,

sql/version.sql

@@ -4,5 +4,5 @@ CREATE OR REPLACE FUNCTION _timescaledb_internal.get_git_commit() RETURNS TEXT
 CREATE OR REPLACE FUNCTION _timescaledb_internal.get_os_info() RETURNS TABLE(sysname TEXT, version TEXT, release TEXT)
     AS '@MODULE_PATHNAME@', 'ts_get_os_info' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
 
-CREATE OR REPLACE FUNCTION get_report() RETURNS TEXT
+CREATE OR REPLACE FUNCTION get_telemetry_report() RETURNS TEXT
     AS '@MODULE_PATHNAME@', 'ts_get_telemetry_report' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;

src/bgw/job.c

@@ -14,7 +14,7 @@
 #include "telemetry/telemetry.h"
 
 const char *job_type_names[_MAX_JOB_TYPE] = {
-	[JOB_TYPE_VERSION_CHECK] = "check_for_update_if_enabled",
+	[JOB_TYPE_VERSION_CHECK] = "telemetry_and_version_check_if_enabled",
 	[JOB_TYPE_UNKNOWN] = "unknown"
 };
 

src/bgw/scheduler.c

@@ -129,6 +129,8 @@ worker_state_cleanup(ScheduledBgwJob *sjob)
 	{
 		BgwJobStat *job_stat = bgw_job_stat_find(sjob->job.fd.id);
 
+		Assert(job_stat != NULL);
+
 		if (!bgw_job_stat_end_was_marked(job_stat))
 		{
 			/*
@@ -385,8 +387,13 @@ terminate_all_jobs_and_release_workers()
 	{
 		ScheduledBgwJob *sjob = lfirst(lc);
 
-		if (sjob->state == JOB_STATE_STARTED)
-			scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_TERMINATING);
+		/*
+		 * Clean up the background workers. Don't worry about state of the
+		 * sjobs, because this callback might have interrupted a state
+		 * transition.
+		 */
+		if (sjob->handle != NULL)
+			TerminateBackgroundWorker(sjob->handle);
 
 		if (sjob->reserved_worker)
 		{

src/telemetry/telemetry.c

@@ -209,16 +209,18 @@ build_version_body(void)
 	JsonbValue *result;
 	Jsonb	   *jb;
 	StringInfo	jtext;
-	JsonbParseState *parseState = NULL;
 	VersionOSInfo osinfo;
+	JsonbParseState *parseState = NULL;
 
 	pushJsonbValue(&parseState, WJB_BEGIN_OBJECT, NULL);
+
 	jsonb_add_pair(parseState, REQ_DB_UUID,
 				   DatumGetCString(DirectFunctionCall1(uuid_out, metadata_get_uuid())));
 	jsonb_add_pair(parseState, REQ_EXPORTED_DB_UUID,
 				   DatumGetCString(DirectFunctionCall1(uuid_out, metadata_get_exported_uuid())));
 	jsonb_add_pair(parseState, REQ_INSTALL_TIME,
 				   DatumGetCString(DirectFunctionCall1(timestamptz_out, metadata_get_install_timestamp())));
+
 	jsonb_add_pair(parseState, REQ_INSTALL_METHOD, TIMESCALEDB_INSTALL_METHOD);
 
 	if (version_get_os_info(&osinfo))
@@ -310,10 +312,17 @@ telemetry_main()
 	Connection *conn;
 	HttpRequest *req;
 	HttpResponseState *rsp;
+	bool		started = false;
 
 	if (!telemetry_on())
 		return;
 
+	if (!IsTransactionOrTransactionBlock())
+	{
+		started = true;
+		StartTransactionCommand();
+	}
+
 	conn = telemetry_connect();
 
 	req = build_version_request(TELEMETRY_HOST, TELEMETRY_PATH);
@@ -339,6 +348,10 @@ telemetry_main()
 	process_response(http_response_state_body_start(rsp));
 
 	http_response_state_destroy(rsp);
+
+	if (started)
+		CommitTransactionCommand();
+	return;
 }
 
 TS_FUNCTION_INFO_V1(ts_get_telemetry_report);

src/telemetry/uuid.c

@@ -41,7 +41,7 @@ uuid_create(void)
 	{
 		TimestampTz ts = GetCurrentTimestamp();
 
-		memcpy(&gen_uuid[9], &ts, sizeof(TimestampTz));
+		memcpy(&gen_uuid[8], &ts, sizeof(TimestampTz));
 	}
 
 	gen_uuid[6] = (gen_uuid[6] & 0x0f) | 0x40;	/* "version" field */