1.7.5 backporting (created from master repo)

appveyor.yml

@@ -1,16 +1,16 @@
 version: 1.0.{build}
 image: Windows Server 2019
 services:
-  - postgresql101
+  - postgresql12
   - docker
 build_script:
 - ps: >-
 
     # Create directories for tablespaces
 
-    New-Item -ItemType directory -Path "C:\Users\$env:UserName\Documents\tablespace1"
+    New-Item -ItemType directory -Path "C:\Users\$env:UserName\Documents\tablespace1\_default"
 
-    New-Item -ItemType directory -Path "C:\Users\$env:UserName\Documents\tablespace2"
+    New-Item -ItemType directory -Path "C:\Users\$env:UserName\Documents\tablespace2\_default"
 
     New-Item -ItemType directory -Path "C:\Users\$env:UserName\Documents\log"
 
@@ -52,84 +52,95 @@ build_script:
 
     SET PGPASSWORD=Password12!
 
-    Write-Output "postgresql-x64-10 should start out running"
+    Write-Output "postgresql-x64-12 should start out running"
 
-    Get-Service -Name "postgresql-x64-10"
+    Get-Service -Name "postgresql-x64-12"
 
-    Write-Output "If the binary for postgres in not in 'C:\Program Files\postgresql\10' the rest of this script won't work"
+    Write-Output "If the binary for postgres in not in 'C:\Program Files\postgresql\12' the rest of this script won't work"
 
-    reg query "HKLM\System\CurrentControlSet\Services\postgresql-x64-10" /v "ImagePath"
+    reg query "HKLM\System\CurrentControlSet\Services\postgresql-x64-12" /v "ImagePath"
 
-    Stop-Service postgresql-x64-10
+    Stop-Service postgresql-x64-12
 
-    # postgresql-x64-10 should stop
+    # postgresql-x64-12 should stop
 
-    Get-Service -Name "postgresql-x64-10"
+    Get-Service -Name "postgresql-x64-12"
 
     # we preload timescale, and allow all netowrk connections to postgres
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "shared_preload_libraries = 'timescaledb'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "shared_preload_libraries = 'timescaledb'"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "log_directory = 'pg_log'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "logging_collector = on"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "log_line_prefix = '%u [%p] %d '"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "log_directory = 'pg_log'"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "max_worker_processes=16"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "log_line_prefix = '%u [%p] %d '"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "autovacuum=false"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "max_worker_processes=16"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "random_page_cost=1.0"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "autovacuum=false"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "timescaledb.last_tuned='1971-02-03 04:05:06.789012 -0300'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "fsync=false"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "timescaledb.last_tuned_version='0.0.1'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "random_page_cost=1.0"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "extra_float_digits=0"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "datestyle='postgres'"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timezone='US/Pacific'"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timescaledb.last_tuned='1971-02-03 04:05:06.789012 -0300'"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timescaledb.last_tuned_version='0.0.1'"
+
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timescaledb_telemetry.cloud='ci'"
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "timescaledb_telemetry.cloud='ci'"
 
     # TODO removing the following line causes a stack overflow on appveyor
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "timescaledb.telemetry_level='off'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timescaledb.telemetry_level='off'"
 
-    # Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "log_min_messages='debug5'"
+    # Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "log_min_messages='debug5'"
 
 
-    Set-Content "C:\Program Files\postgresql\10\data\pg_hba.conf" "host    all             all             ::1/128            trust"
+    Set-Content "C:\Program Files\postgresql\12\data\pg_hba.conf" "host    all             all             ::1/128            trust"
 
-    Add-Content "C:\Program Files\postgresql\10\data\pg_hba.conf" "host    all             all             127.0.0.1/32            trust"
+    Add-Content "C:\Program Files\postgresql\12\data\pg_hba.conf" "host    all             all             127.0.0.1/32            trust"
 
     # build timescale
 
-     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\10" -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP
+     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\12" -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP
 
     cmake --build ./build --config Debug
 
     cmake --build ./build --config Debug --target install
 
     # postgres should restart
 
-    Start-Service postgresql-x64-10
+    Start-Service postgresql-x64-12
 
-    Get-Service -Name "postgresql-x64-10"
+    Get-Service -Name "postgresql-x64-12"
 
     # create pg user root b/c docker will need it
 
-    & "C:\Program Files\PostgreSQL\10\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'CREATE USER root WITH SUPERUSER IN ROLE postgres;' -U postgres
+    & "C:\Program Files\PostgreSQL\12\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'CREATE USER root WITH SUPERUSER IN ROLE postgres;' -U postgres
 
-    & "C:\Program Files\PostgreSQL\10\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'CREATE USER super_user WITH SUPERUSER IN ROLE postgres;' -U postgres
+    & "C:\Program Files\PostgreSQL\12\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'CREATE USER super_user WITH SUPERUSER IN ROLE postgres;' -U postgres
 
-    & "C:\Program Files\PostgreSQL\10\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show log_directory' -U postgres
+    & "C:\Program Files\PostgreSQL\12\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show log_directory' -U postgres
 
-    & "C:\Program Files\PostgreSQL\10\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show log_destination;' -U postgres
+    & "C:\Program Files\PostgreSQL\12\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show log_destination;' -U postgres
 
-    & "C:\Program Files\PostgreSQL\10\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show logging_collector;' -U postgres
+    & "C:\Program Files\PostgreSQL\12\bin\psql.exe" -a -e -E -v "VERBOSITY=verbose" -c 'show logging_collector;' -U postgres
 
     docker-switch-linux
 
-    # docker run -d --name pgregress -v c:/projects/timescaledb:/timescale timescaledev/postgresdev:exec_backend-10-alpine
+    docker run -d --name pgregress alpine:3.12 sh -c "while sleep 3600; do :; done"
 
-    docker run -d --name pgregress timescaledev/postgresdev:exec_backend-10-alpine
+    docker exec -it pgregress /bin/sh -c "apk add --no-cache --virtual .build-deps bash coreutils findutils gcc libc-dev make util-linux-dev diffutils cmake curl git openssl-dev openssl postgresql-client postgresql-dev"
 
-    docker exec -it pgregress /bin/bash -c "apk add cmake git diffutils"
+    docker exec -it pgregress /bin/sh -c "ln -s /usr/lib/postgresql/pgxs/src/test/regress/pg_regress /usr/local/bin/pg_regress"
 
     # we clone the current commit in the docker instance to ensure the correct tests run
     # (Ideally we'd use the same folder, but that's difficult to set up)
@@ -145,29 +156,29 @@ test_script:
 
     Set-PSDebug -Trace 1
 
-    Get-Service -Name "postgresql-x64-10"
+    Get-Service -Name "postgresql-x64-12"
 
     docker exec -it pgregress /bin/bash -c "psql -a -e -E -U postgres --host='docker.for.win.localhost' -v VERBOSITY=verbose -c'\dx;'"
 
     #right now we only run timescale regression tests, others will be set up later
 
-    docker exec -e IGNORES="bgw_db_scheduler" -e TEST_TABLESPACE1_PATH="C:\Users\$env:UserName\Documents\tablespace1\" -e TEST_TABLESPACE2_PATH="C:\Users\$env:UserName\Documents\tablespace2\" -e TEST_SPINWAIT_ITERS=10000 -e USER=postgres -it pgregress /bin/bash -c "cd /timescaledb/build && make regresschecklocal"
+    docker exec -e IGNORES="bgw_db_scheduler chunk_utils loader cluster" -e SKIPS="cluster" -e TEST_TABLESPACE1_PREFIX="C:\Users\$env:UserName\Documents\tablespace1\" -e TEST_TABLESPACE2_PREFIX="C:\Users\$env:UserName\Documents\tablespace2\" -e TEST_SPINWAIT_ITERS=10000 -e USER=postgres -it pgregress /bin/bash -c "cd /timescaledb/build && make regresschecklocal"
 
     $TESTS1 = $?
 
     # TODO can we switch these to file swapping?
 
-    Add-Content "C:\Program Files\postgresql\10\data\postgresql.conf" "timescaledb.license_key = 'E1eyJlbmRfdGltZSI6IjIwMTgtMTAtMDEgKzAwMDAiLCAic3RhcnRfdGltZSI6IjIwMTgtMDktMDEgKzAwMDAiLCAiaWQiOiI0OTBGQjI2MC1BMjkyLTRBRDktOUFBMi0wMzYwODM1NzkxQjgiLCAia2luZCI6InRyaWFsIn0K'"
+    Add-Content "C:\Program Files\postgresql\12\data\postgresql.conf" "timescaledb.license_key = 'E1eyJlbmRfdGltZSI6IjIwMTgtMTAtMDEgKzAwMDAiLCAic3RhcnRfdGltZSI6IjIwMTgtMDktMDEgKzAwMDAiLCAiaWQiOiI0OTBGQjI2MC1BMjkyLTRBRDktOUFBMi0wMzYwODM1NzkxQjgiLCAia2luZCI6InRyaWFsIn0K'"
 
-    Restart-Service postgresql-x64-10
+    Restart-Service postgresql-x64-12
 
-    docker exec -e IGNORES="bgw_db_scheduler" -e TEST_TABLESPACE1_PATH="C:\Users\$env:UserName\Documents\tablespace1\" -e TEST_TABLESPACE2_PATH="C:\Users\$env:UserName\Documents\tablespace2\" -e TEST_SPINWAIT_ITERS=10000 -e USER=postgres -it pgregress /bin/bash -c "cd /timescaledb/build && make regresschecklocal-t"
+    docker exec -e IGNORES="bgw_db_scheduler compression_algos continuous_aggs_bgw remote_connection" -e SKIPS="bgw_db_scheduler" -e TEST_TABLESPACE1_PREFIX="C:\Users\$env:UserName\Documents\tablespace1\" -e TEST_TABLESPACE2_PREFIX="C:\Users\$env:UserName\Documents\tablespace2\" -e TEST_SPINWAIT_ITERS=10000 -e USER=postgres -it pgregress /bin/bash -c "cd /timescaledb/build && make regresschecklocal-t"
 
     if( -not $? -or -not $TESTS1 ) { exit 1 }
 
 on_failure:
   - ps: >-
       docker exec -it pgregress cat /timescaledb/build/test/regression.diffs /timescaledb/build/tsl/test/regression.diffs /timescaledb/build/test/pgtest/regressions.diffs
 
-      Get-Content -Path "C:\Program Files\postgresql\10\data\pg_log\*"
+      Get-Content -Path "C:\Program Files\postgresql\12\data\pg_log\*"
 

sql/updates/latest-dev.sql

@@ -76,3 +76,6 @@ SELECT DISTINCT
             END
        END AS range_end
   FROM unparsed_missing_slices;
+
+-- set compressed_chunk_id to NULL for dropped chunks
+UPDATE _timescaledb_catalog.chunk SET compressed_chunk_id = NULL WHERE dropped = true AND compressed_chunk_id IS NOT NULL;

src/chunk.c

@@ -2373,6 +2373,7 @@ chunk_tuple_delete(TupleInfo *ti, DropBehavior behavior, bool preserve_chunk_cat
 
 		Assert(!form.dropped);
 
+		form.compressed_chunk_id = INVALID_CHUNK_ID;
 		form.dropped = true;
 		new_tuple = chunk_formdata_make_tuple(&form, ti->desc);
 		ts_catalog_update_tid(ti->scanrel, &ti->tuple->t_self, new_tuple);
@@ -2495,12 +2496,21 @@ ts_chunk_exists_with_compression(int32 hypertable_id)
 	init_scan_by_hypertable_id(&iterator, hypertable_id);
 	ts_scanner_foreach(&iterator)
 	{
-		bool isnull;
+		bool isnull_chunk_id;
+		bool isnull_dropped;
+		bool dropped;
 		heap_getattr(ts_scan_iterator_tuple_info(&iterator)->tuple,
 					 Anum_chunk_compressed_chunk_id,
 					 ts_scan_iterator_tuple_info(&iterator)->desc,
-					 &isnull);
-		if (!isnull)
+					 &isnull_chunk_id);
+		dropped = DatumGetBool(heap_getattr(ts_scan_iterator_tuple_info(&iterator)->tuple,
+											Anum_chunk_dropped,
+											ts_scan_iterator_tuple_info(&iterator)->desc,
+											&isnull_dropped));
+		/* dropped is not NULLABLE */
+		Assert(!isnull_dropped);
+
+		if (!isnull_chunk_id && !dropped)
 		{
 			found = true;
 			break;

src/chunk_append/planner.c

@@ -104,7 +104,6 @@ ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path
 	cscan->scan.scanrelid = rel->relid;
 
 	tlist = ts_build_path_tlist(root, (Path *) path);
-	cscan->custom_scan_tlist = tlist;
 	cscan->scan.plan.targetlist = tlist;
 
 	if (path->path.pathkeys == NIL)
@@ -227,6 +226,8 @@ ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path
 		}
 	}
 
+	/* decouple input tlist from output tlist in case output tlist gets modified later */
+	cscan->custom_scan_tlist = list_copy(tlist);
 	cscan->custom_plans = custom_plans;
 
 	/*

src/plan_expand_hypertable.c

@@ -59,6 +59,7 @@ typedef struct CollectQualCtx
 	List *join_conditions;
 	List *propagate_conditions;
 	List *all_quals;
+	int join_level;
 } CollectQualCtx;
 
 static void propagate_join_quals(PlannerInfo *root, RelOptInfo *rel, CollectQualCtx *ctx);
@@ -618,7 +619,7 @@ timebucket_annotate(Node *quals, CollectQualCtx *ctx)
  * a JOIN.
  */
 static void
-collect_join_quals(Node *quals, CollectQualCtx *ctx, bool is_outer_join)
+collect_join_quals(Node *quals, CollectQualCtx *ctx, bool can_propagate)
 {
 	ListCell *lc;
 
@@ -631,7 +632,7 @@ collect_join_quals(Node *quals, CollectQualCtx *ctx, bool is_outer_join)
 		/*
 		 * collect quals to propagate to join relations
 		 */
-		if (num_rels == 1 && !is_outer_join && IsA(qual, OpExpr) &&
+		if (num_rels == 1 && can_propagate && IsA(qual, OpExpr) &&
 			list_length(castNode(OpExpr, qual)->args) == 2)
 			ctx->all_quals = lappend(ctx->all_quals, qual);
 
@@ -655,7 +656,7 @@ collect_join_quals(Node *quals, CollectQualCtx *ctx, bool is_outer_join)
 				{
 					ctx->join_conditions = lappend(ctx->join_conditions, op);
 
-					if (!is_outer_join)
+					if (can_propagate)
 						ctx->propagate_conditions = lappend(ctx->propagate_conditions, op);
 				}
 			}
@@ -674,13 +675,22 @@ collect_quals_walker(Node *node, CollectQualCtx *ctx)
 	{
 		FromExpr *f = castNode(FromExpr, node);
 		f->quals = process_quals(f->quals, ctx, false);
-		collect_join_quals(f->quals, ctx, false);
+		/* if this is a nested join we don't propagate join quals */
+		collect_join_quals(f->quals, ctx, ctx->join_level == 0);
 	}
 	else if (IsA(node, JoinExpr))
 	{
 		JoinExpr *j = castNode(JoinExpr, node);
 		j->quals = process_quals(j->quals, ctx, IS_OUTER_JOIN(j->jointype));
-		collect_join_quals(j->quals, ctx, IS_OUTER_JOIN(j->jointype));
+		collect_join_quals(j->quals, ctx, ctx->join_level == 0 && !IS_OUTER_JOIN(j->jointype));
+
+		if (IS_OUTER_JOIN(j->jointype))
+		{
+			ctx->join_level++;
+			bool result = expression_tree_walker(node, collect_quals_walker, ctx);
+			ctx->join_level--;
+			return result;
+		}
 	}
 
 	/* skip processing if we found a chunks_in call for current relation */
@@ -961,13 +971,11 @@ timebucket_annotate_walker(Node *node, CollectQualCtx *ctx)
 	{
 		FromExpr *f = castNode(FromExpr, node);
 		f->quals = timebucket_annotate(f->quals, ctx);
-		collect_join_quals(f->quals, ctx, true);
 	}
 	else if (IsA(node, JoinExpr))
 	{
 		JoinExpr *j = castNode(JoinExpr, node);
 		j->quals = timebucket_annotate(j->quals, ctx);
-		collect_join_quals(j->quals, ctx, !IS_OUTER_JOIN(j->jointype));
 	}
 
 	/* skip processing if we found a chunks_in call for current relation */
@@ -1021,6 +1029,7 @@ ts_plan_expand_hypertable_chunks(Hypertable *ht, PlannerInfo *root, RelOptInfo *
 		.all_quals = NIL,
 		.join_conditions = NIL,
 		.propagate_conditions = NIL,
+		.join_level = 0,
 	};
 	Size old_rel_array_len;
 	Index first_chunk_index = 0;
@@ -1045,6 +1054,8 @@ ts_plan_expand_hypertable_chunks(Hypertable *ht, PlannerInfo *root, RelOptInfo *
 
 	/* Walk the tree and find restrictions or chunk exclusion functions */
 	collect_quals_walker((Node *) root->parse->jointree, &ctx);
+	/* check join_level bookkeeping is balanced */
+	Assert(ctx.join_level == 0);
 
 #if PG12_GE
 	rel->baserestrictinfo = remove_exclusion_fns(rel->baserestrictinfo);

src/process_utility.c

@@ -2342,6 +2342,13 @@ process_altertable_clusteron_end(Hypertable *ht, AlterTableCmd *cmd)
 {
 	Oid index_relid =
 		get_relname_relid(cmd->name, get_namespace_oid(NameStr(ht->fd.schema_name), false));
+
+	/* If this is part of changing the type of a column that is used in a clustered index
+	 * the above lookup might fail. But in this case we don't need to mark the index clustered
+	 * as postgres takes care of that already (except PG11 < 11.8 and PG12 < 12.3) */
+	if (!OidIsValid(index_relid))
+		return;
+
 	List *chunk_indexes = ts_chunk_index_get_mappings(ht, index_relid);
 	ListCell *lc;