Refactor chunk index handling

This change refactors the chunk index handling to make better use
of standard PostgreSQL catalog information, while removing the
hypertable_index metadata table and associated triggers, including
those on the chunk_index table. The chunk_index table itself is
also simplified.

A benefit of this refactoring is that indexes are no longer
created using string mangling to construct the CREATE INDEX command
for a chunk, based on the string definition of the hypertable
index. Instead, indexes are created in C using proper index-related
internal data structures.

Chunk indexes can now also be renamed and are added in the parent
index tablespace. Changing tablespace on a hypertable index also
recurses to chunks, as expected. Default indexes that are added when
creating a hypertable use the hypertable's tablespace.

Creating Hypertable indexes with the CONCURRENTLY modifier is
currently blocked, due to unclear semantics regarding concurrent
creation over many tables, including how to deal with snapshots.

Makefile

@@ -19,7 +19,7 @@ UPDATE_VERSIONS_PRE = $(shell ls -1 sql/updates/pre-$(UPDATE_VERSIONS).sql)
 UPDATE_VERSIONS_POST = $(shell ls -1 sql/updates/post-$(UPDATE_VERSIONS).sql)
 
 # Get SQL files that are needed to build update script.
-UPDATE_SQL_FILES = $(shell echo ${UPDATE_VERSIONS_PRE} && cat sql/functions_load_order.txt && echo ${UPDATE_VERSIONS_POST} )
+UPDATE_SQL_FILES = $(shell echo ${UPDATE_VERSIONS_PRE} && cat sql/setup_load_order.txt && cat sql/functions_load_order.txt && echo ${UPDATE_VERSIONS_POST} )
 
 EXT_SQL_FILE = sql/$(EXTENSION)--$(EXT_VERSION).sql
 UPDATE_FILE = sql/$(EXTENSION)--$(UPDATE_VERSIONS).sql
@@ -45,6 +45,8 @@ SRCS = \
 	src/event_trigger.c \
 	src/trigger.c \
 	src/chunk.c \
+	src/chunk_index.c \
+	src/indexing.c \
 	src/scanner.c \
 	src/histogram.c \
 	src/hypertable_cache.c \

scripts/dump_meta_data.sql

@@ -22,9 +22,6 @@ SELECT * FROM _timescaledb_internal.get_git_commit();
 \echo 'List of hypertables'
 SELECT * FROM _timescaledb_catalog.hypertable;
 
-\echo 'List of hypertable indexes'
-SELECT * FROM _timescaledb_catalog.hypertable_index;
-
 \echo 'List of chunk indexes'
 SELECT * FROM _timescaledb_catalog.chunk_index;
 
@@ -102,14 +99,19 @@ SELECT *,
 ) sub2;
 
 \echo 'Hypertable index sizes'
-SELECT h.schema_name || '.' || h.table_name as hypertable, hi.main_schema_name || '.' || hi.main_index_name as index_name,
-       sum(pg_relation_size('"' || ci.schema_name || '"."' || ci.index_name || '"'))::bigint as index_bytes
+SELECT h.schema_name || '.' || h.table_name AS hypertable,
+       h.schema_name || '.' || ci.hypertable_index_name AS index_name,
+       sum(pg_relation_size(c.oid))::bigint AS index_bytes
 FROM
+pg_class c,
+pg_namespace n,
 _timescaledb_catalog.hypertable h,
-_timescaledb_catalog.hypertable_index hi,
+_timescaledb_catalog.chunk ch,
 _timescaledb_catalog.chunk_index ci
-WHERE h.id = hi.hypertable_id
-      AND ci.main_index_name = hi.main_index_name
-      AND ci.main_schema_name = hi.main_schema_name
-GROUP BY h.schema_name || '.' || h.table_name, hi.main_schema_name || '.' || hi.main_index_name;
-
+WHERE ch.schema_name = n.nspname
+      AND c.relnamespace = n.oid
+      AND c.relname = ci.index_name
+      AND ch.id = ci.chunk_id
+      AND h.id = ci.hypertable_id
+GROUP BY h.schema_name, h.table_name, ci.hypertable_index_name
+ORDER BY h.schema_name, h.table_name, ci.hypertable_index_name;

sql/chunk.sql

@@ -85,12 +85,6 @@ BEGIN
     FROM _timescaledb_catalog.chunk_constraint cc
     WHERE cc.chunk_id = chunk_create.chunk_id AND cc.dimension_slice_id IS NOT NULL;
 
-    PERFORM _timescaledb_internal.create_chunk_index_row(chunk_row.schema_name, chunk_row.table_name,
-                            hi.main_schema_name, hi.main_index_name, hi.definition)
-    FROM _timescaledb_catalog.hypertable_index hi
-    WHERE hi.hypertable_id = chunk_row.hypertable_id
-    ORDER BY main_schema_name, main_index_name;
-
     SELECT * INTO STRICT hypertable_row FROM _timescaledb_catalog.hypertable WHERE id = chunk_row.hypertable_id;
     main_table_oid := format('%I.%I', hypertable_row.schema_name, hypertable_row.table_name)::regclass;
 

sql/chunk_constraint.sql

@@ -17,13 +17,13 @@ BEGIN
     IF chunk_constraint_row.dimension_slice_id IS NOT NULL THEN
         def := format('CHECK (%s)',  _timescaledb_internal.dimension_slice_get_constraint_sql(chunk_constraint_row.dimension_slice_id));
     ELSIF chunk_constraint_row.hypertable_constraint_name IS NOT NULL THEN
-        SELECT oid INTO STRICT constraint_oid FROM pg_constraint 
-        WHERE conname=chunk_constraint_row.hypertable_constraint_name AND 
+        SELECT oid INTO STRICT constraint_oid FROM pg_constraint
+        WHERE conname=chunk_constraint_row.hypertable_constraint_name AND
               conrelid = format('%I.%I', hypertable_row.schema_name, hypertable_row.table_name)::regclass::oid;
         def := pg_get_constraintdef(constraint_oid);
     ELSE
         RAISE 'Unknown constraint type';
-    END IF; 
+    END IF;
 
     sql_code := format(
         $$ ALTER TABLE %I.%I ADD CONSTRAINT %I %s $$,
@@ -67,7 +67,7 @@ DECLARE
 BEGIN
     SELECT * INTO STRICT constraint_row FROM pg_constraint WHERE OID = constraint_oid;
     hypertable_constraint_name := constraint_row.conname;
-    constraint_name := format('%s_%s_%s', chunk_id,  nextval('_timescaledb_catalog.chunk_constraint_name'), hypertable_constraint_name); 
+    constraint_name := format('%s_%s_%s', chunk_id,  nextval('_timescaledb_catalog.chunk_constraint_name'), hypertable_constraint_name);
 
     INSERT INTO _timescaledb_catalog.chunk_constraint (chunk_id, constraint_name, dimension_slice_id, hypertable_constraint_name)
     VALUES (chunk_id, constraint_name, NULL, hypertable_constraint_name) RETURNING * INTO STRICT chunk_constraint_row;
@@ -91,12 +91,12 @@ DECLARE
 BEGIN
     SELECT * INTO STRICT chunk_row FROM _timescaledb_catalog.chunk c WHERE c.id = chunk_id;
 
-    DELETE FROM _timescaledb_catalog.chunk_constraint cc 
-    WHERE  cc.constraint_name = drop_chunk_constraint.constraint_name 
+    DELETE FROM _timescaledb_catalog.chunk_constraint cc
+    WHERE  cc.constraint_name = drop_chunk_constraint.constraint_name
     AND cc.chunk_id = drop_chunk_constraint.chunk_id
     RETURNING * INTO STRICT chunk_constraint_row;
 
-    IF alter_table THEN 
+    IF alter_table THEN
         EXECUTE format(
             $$  ALTER TABLE %I.%I DROP CONSTRAINT %I $$,
                 chunk_row.schema_name, chunk_row.table_name, chunk_constraint_row.constraint_name
@@ -148,13 +148,6 @@ BEGIN
     IF _timescaledb_internal.need_chunk_constraint(constraint_oid) THEN
         SELECT * INTO STRICT constraint_row FROM pg_constraint WHERE OID = constraint_oid;
 
-        --check the validity of an index if a constraint uses an index
-        --note: foreign-key constraints are excluded because they point to indexes on the foreign table /not/ the hypertable
-        IF constraint_row.conindid <> 0 AND constraint_row.contype != 'f' THEN
-            SELECT * INTO STRICT hypertable_row FROM _timescaledb_catalog.hypertable WHERE id = hypertable_id;
-            PERFORM _timescaledb_internal.check_index(constraint_row.conindid, hypertable_row);
-        END IF;
-
         PERFORM _timescaledb_internal.create_chunk_constraint(c.id, constraint_oid)
         FROM _timescaledb_catalog.chunk c
         WHERE c.hypertable_id = add_constraint.hypertable_id;
@@ -171,7 +164,7 @@ $BODY$
 DECLARE
     constraint_oid OID;
 BEGIN
-    SELECT oid INTO STRICT constraint_oid FROM pg_constraint WHERE conname = constraint_name 
+    SELECT oid INTO STRICT constraint_oid FROM pg_constraint WHERE conname = constraint_name
     AND conrelid = _timescaledb_internal.main_table_from_hypertable(hypertable_id);
 
     PERFORM _timescaledb_internal.add_constraint(hypertable_id, constraint_oid);
@@ -193,4 +186,3 @@ BEGIN
     WHERE c.hypertable_id = drop_constraint.hypertable_id AND cc.hypertable_constraint_name = drop_constraint.hypertable_constraint_name;
 END
 $BODY$;
-

sql/ddl_api.sql

@@ -122,18 +122,7 @@ BEGIN
     FROM pg_constraint
     WHERE conrelid = main_table;
 
-    PERFORM 1
-    FROM pg_index,
-    LATERAL _timescaledb_internal.add_index(
-        hypertable_row.id,
-        hypertable_row.schema_name,
-        (SELECT relname FROM pg_class WHERE oid = indexrelid::regclass),
-        _timescaledb_internal.get_general_index_definition(indexrelid, indrelid, hypertable_row)
-    )
-    WHERE indrelid = main_table AND _timescaledb_internal.need_chunk_index(hypertable_row.id, pg_index.indexrelid)
-    ORDER BY pg_index.indexrelid;
-
-    IF create_default_indexes THEN
+   IF create_default_indexes THEN
         PERFORM _timescaledb_internal.create_default_indexes(hypertable_row, main_table, partitioning_column);
     END IF;
 END