dist_hypertable-15.out

-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- Need to be super user to create extension and add data nodes
\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER;
\unset ECHO
psql:include/filter_exec.sql:5: NOTICE:  schema "test" already exists, skipping
psql:include/remote_exec.sql:5: NOTICE:  schema "test" already exists, skipping
\set DATA_NODE_1 :TEST_DBNAME _1
\set DATA_NODE_2 :TEST_DBNAME _2
\set DATA_NODE_3 :TEST_DBNAME _3
\set DATA_NODE_4 :TEST_DBNAME _4
\set TABLESPACE_1 :TEST_DBNAME _1
\set TABLESPACE_2 :TEST_DBNAME _2
SELECT
    test.make_tablespace_path(:'TEST_TABLESPACE1_PREFIX', :'TEST_DBNAME') AS spc1path,
    test.make_tablespace_path(:'TEST_TABLESPACE2_PREFIX', :'TEST_DBNAME') AS spc2path
\gset
SELECT node_name, database, node_created, database_created, extension_created
FROM (
  SELECT (add_data_node(name, host => 'localhost', DATABASE => name)).*
  FROM (VALUES (:'DATA_NODE_1'), (:'DATA_NODE_2'), (:'DATA_NODE_3')) v(name)
) a;
      node_name       |       database       | node_created | database_created | extension_created 
----------------------+----------------------+--------------+------------------+-------------------
 db_dist_hypertable_1 | db_dist_hypertable_1 | t            | t                | t
 db_dist_hypertable_2 | db_dist_hypertable_2 | t            | t                | t
 db_dist_hypertable_3 | db_dist_hypertable_3 | t            | t                | t
(3 rows)

GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC;
-- View to see dimension partitions. Note RIGHT JOIN to see that
-- dimension partitions are cleaned up (deleted) properly.
CREATE VIEW hypertable_partitions AS
SELECT table_name, dimension_id, range_start, data_nodes
FROM _timescaledb_catalog.hypertable h
INNER JOIN _timescaledb_catalog.dimension d ON (d.hypertable_id = h.id)
RIGHT JOIN _timescaledb_catalog.dimension_partition dp ON (dp.dimension_id = d.id)
ORDER BY dimension_id, range_start;
GRANT SELECT ON hypertable_partitions TO :ROLE_1;
-- Import testsupport.sql file to data nodes
\unset ECHO
GRANT CREATE ON SCHEMA public TO :ROLE_1;
SET ROLE :ROLE_1;
--Ensure INSERTs use DataNodeDispatch. DataNodeCopy is tested later
SET timescaledb.enable_distributed_insert_with_copy=false;
-- Verify lack of tables
SELECT node_name FROM timescaledb_information.data_nodes ORDER BY node_name;
      node_name       
----------------------
 db_dist_hypertable_1
 db_dist_hypertable_2
 db_dist_hypertable_3
(3 rows)

\set ON_ERROR_STOP 0
-- Test that one cannot directly create TimescaleDB foreign tables
CREATE FOREIGN TABLE foreign_table (time timestamptz, device int, temp float) SERVER :DATA_NODE_1;
ERROR:  operation not supported
\set ON_ERROR_STOP 1
-- Create distributed hypertables. Add a trigger and primary key
-- constraint to test how those work
CREATE TABLE disttable(time timestamptz, device int CHECK (device > 0), color int, temp float, PRIMARY KEY (time,device));
SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 1);
WARNING:  insufficient number of partitions for dimension "device"
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
             1 | public      | disttable  | t
(1 row)

-- Increase the number of partitions. Expect warning since still too
-- low. Dimension partitions should be updated to reflect new
-- partitioning.
SELECT * FROM hypertable_partitions WHERE table_name = 'disttable';
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 disttable  |            2 | -9223372036854775808 | {db_dist_hypertable_1}
(1 row)

SELECT * FROM set_number_partitions('disttable', 2);
WARNING:  insufficient number of partitions for dimension "device"
 set_number_partitions 
-----------------------
 
(1 row)

SELECT * FROM hypertable_partitions WHERE table_name = 'disttable';
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 disttable  |            2 | -9223372036854775808 | {db_dist_hypertable_1}
 disttable  |            2 |           1073741823 | {db_dist_hypertable_2}
(2 rows)

-- Set number of partitions equal to the number of servers should not
-- raise a warning.
SELECT * FROM hypertable_partitions WHERE table_name = 'disttable';
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 disttable  |            2 | -9223372036854775808 | {db_dist_hypertable_1}
 disttable  |            2 |           1073741823 | {db_dist_hypertable_2}
(2 rows)

SELECT * FROM set_number_partitions('disttable', 3, 'device');
 set_number_partitions 
-----------------------
 
(1 row)

SELECT * FROM hypertable_partitions WHERE table_name = 'disttable';
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 disttable  |            2 | -9223372036854775808 | {db_dist_hypertable_1}
 disttable  |            2 |            715827882 | {db_dist_hypertable_2}
 disttable  |            2 |           1431655764 | {db_dist_hypertable_3}
(3 rows)

-- Show the number of slices
SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'disttable';
 table_name | column_name | num_slices 
------------+-------------+------------
 disttable  | device      |          3
 disttable  | time        |           
(2 rows)

-- This table tests both 1-dimensional tables and under-replication
-- (replication_factor > num_data_nodes).
CREATE TABLE underreplicated(time timestamptz, device int, temp float);
\set ON_ERROR_STOP 0
-- can't create an under-replicated hypertable
SELECT * FROM create_hypertable('underreplicated', 'time', replication_factor => 4);
ERROR:  replication factor too large for hypertable "underreplicated"
\set ON_ERROR_STOP 1
-- can create under-replicated hypertable byf force detaching a data node
RESET ROLE;
SELECT node_name, database, node_created, database_created, extension_created
FROM add_data_node(:'DATA_NODE_4', host => 'localhost', database => :'DATA_NODE_4');
      node_name       |       database       | node_created | database_created | extension_created 
----------------------+----------------------+--------------+------------------+-------------------
 db_dist_hypertable_4 | db_dist_hypertable_4 | t            | t                | t
(1 row)

GRANT USAGE ON FOREIGN SERVER :DATA_NODE_4 TO PUBLIC;
GRANT CREATE ON SCHEMA public TO :ROLE_1;
SET ROLE :ROLE_1;
SELECT * FROM create_hypertable('underreplicated', 'time', replication_factor => 4);
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name |   table_name    | created 
---------------+-------------+-----------------+---------
             2 | public      | underreplicated | t
(1 row)

--create new session to clear out connections
\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER;
SELECT * FROM delete_data_node(:'DATA_NODE_4', force => true, drop_database => true);
WARNING:  insufficient number of data nodes for distributed hypertable "underreplicated"
 delete_data_node 
------------------
 t
(1 row)

SET ROLE :ROLE_1;
CREATE OR REPLACE FUNCTION test_trigger()
    RETURNS TRIGGER LANGUAGE PLPGSQL AS
$BODY$
DECLARE
    cnt INTEGER;
BEGIN
    SELECT count(*) INTO cnt FROM public.disttable;
    RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %',
        tg_when, tg_level, tg_op, cnt, tg_name;

    IF TG_OP = 'DELETE' THEN
        RETURN OLD;
    END IF;
    RETURN NEW;
END
$BODY$;
-- Create the trigger function on the data nodes:
CALL distributed_exec($$
CREATE OR REPLACE FUNCTION test_trigger()
    RETURNS TRIGGER LANGUAGE PLPGSQL AS
$BODY$
DECLARE
    cnt INTEGER;
BEGIN
    SELECT count(*) INTO cnt FROM public.disttable;
    RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %',
        tg_when, tg_level, tg_op, cnt, tg_name;

    IF TG_OP = 'DELETE' THEN
        RETURN OLD;
    END IF;
    RETURN NEW;
END
$BODY$;
$$);
CREATE TRIGGER _0_test_trigger_insert
    BEFORE INSERT ON disttable
    FOR EACH ROW EXECUTE FUNCTION test_trigger();
SELECT * FROM _timescaledb_catalog.hypertable_data_node ORDER BY 1,2,3;
 hypertable_id | node_hypertable_id |      node_name       | block_chunks 
---------------+--------------------+----------------------+--------------
             1 |                  1 | db_dist_hypertable_1 | f
             1 |                  1 | db_dist_hypertable_2 | f
             1 |                  1 | db_dist_hypertable_3 | f
             2 |                  2 | db_dist_hypertable_1 | f
             2 |                  2 | db_dist_hypertable_2 | f
             2 |                  2 | db_dist_hypertable_3 | f
(6 rows)

SELECT * FROM _timescaledb_catalog.chunk_data_node ORDER BY 1,2,3;
 chunk_id | node_chunk_id | node_name 
----------+---------------+-----------
(0 rows)

-- The constraints, indexes, and triggers on the hypertable
SELECT * FROM test.show_constraints('disttable');
       Constraint       | Type |    Columns    |     Index      |     Expr     | Deferrable | Deferred | Validated 
------------------------+------+---------------+----------------+--------------+------------+----------+-----------
 disttable_device_check | c    | {device}      | -              | (device > 0) | f          | f        | t
 disttable_pkey         | p    | {time,device} | disttable_pkey |              | f          | f        | t
(2 rows)

SELECT * FROM test.show_indexes('disttable');
           Index           |    Columns    | Expr | Unique | Primary | Exclusion | Tablespace 
---------------------------+---------------+------+--------+---------+-----------+------------
 disttable_device_time_idx | {device,time} |      | f      | f       | f         | 
 disttable_pkey            | {time,device} |      | t      | t       | f         | 
 disttable_time_idx        | {time}        |      | f      | f       | f         | 
(3 rows)

SELECT * FROM test.show_triggers('disttable');
        Trigger         | Type |               Function               
------------------------+------+--------------------------------------
 _0_test_trigger_insert |    7 | test_trigger
 ts_insert_blocker      |    7 | _timescaledb_internal.insert_blocker
(2 rows)

-- Drop a column. This will make the attribute numbers of the
-- hypertable's root relation differ from newly created chunks. It is
-- a way to test that we properly handle attributed conversion between
-- the root table and chunks
ALTER TABLE disttable DROP COLUMN color;
-- EXPLAIN some inserts to see what plans and explain output for
-- remote inserts look like
EXPLAIN (COSTS FALSE)
INSERT INTO disttable VALUES
       ('2017-01-01 06:01', 1, 1.1);
                  QUERY PLAN                   
-----------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable disttable
   ->  Insert on disttable
         ->  Custom Scan (DataNodeCopy)
               ->  Custom Scan (ChunkDispatch)
                     ->  Result
(6 rows)

EXPLAIN (VERBOSE, COSTS FALSE)
INSERT INTO disttable VALUES
       ('2017-01-01 06:01', 1, 1.1);
                                                              QUERY PLAN                                                               
---------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable public.disttable
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.disttable
         ->  Custom Scan (DataNodeCopy)
               Output: 'Sun Jan 01 06:01:00 2017 PST'::timestamp with time zone, 1, NULL::integer, '1.1'::double precision
               Remote SQL: COPY public.disttable ("time", device, temp) FROM STDIN WITH (FORMAT binary)
               ->  Custom Scan (ChunkDispatch)
                     Output: 'Sun Jan 01 06:01:00 2017 PST'::timestamp with time zone, 1, NULL::integer, '1.1'::double precision
                     ->  Result
                           Output: 'Sun Jan 01 06:01:00 2017 PST'::timestamp with time zone, 1, NULL::integer, '1.1'::double precision
(11 rows)

-- Create some chunks through insertion
INSERT INTO disttable VALUES
       ('2017-01-01 06:01', 1, 1.1),
       ('2017-01-01 09:11', 3, 2.1),
       ('2017-01-01 09:21', 3, 2.2),
       ('2017-01-01 08:11', 3, 2.3),
       ('2017-01-01 08:01', 1, 1.2),
       ('2017-01-02 08:01', 2, 1.3),
       ('2017-01-02 09:01', 2, 1.4),
       ('2017-01-02 08:21', 2, 1.5),
       ('2018-07-02 08:01', 87, 1.6),
       ('2018-07-02 09:01', 87, 1.4),
       ('2018-07-02 09:21', 87, 1.8),
       ('2018-07-01 06:01', 13, 1.4),
       ('2018-07-01 06:21', 13, 1.5),
       ('2018-07-01 07:01', 13, 1.4),
       ('2018-07-01 09:11', 90, 2.7),
       ('2018-07-01 08:01', 29, 1.5),
       ('2018-07-01 09:21', 90, 2.8),
       ('2018-07-01 08:21', 29, 1.2);
-- EXPLAIN some updates/deletes to see what plans and explain output for
-- remote operations look like
EXPLAIN (VERBOSE, COSTS FALSE)
UPDATE disttable SET temp = 3.7 WHERE device = 1;
                                                               QUERY PLAN                                                                
-----------------------------------------------------------------------------------------------------------------------------------------
 Update on public.disttable
   Update on public.disttable disttable_1
   Foreign Update on _timescaledb_internal._dist_hyper_1_1_chunk disttable_2
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_1_1_chunk SET temp = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_1_4_chunk disttable_3
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_1_4_chunk SET temp = $2 WHERE ctid = $1
   ->  Result
         Output: '3.7'::double precision, disttable.tableoid, disttable.ctid, (NULL::record)
         ->  Append
               ->  Seq Scan on public.disttable disttable_1
                     Output: disttable_1.tableoid, disttable_1.ctid, NULL::record
                     Filter: (disttable_1.device = 1)
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_1_1_chunk disttable_2
                     Output: disttable_2.tableoid, disttable_2.ctid, disttable_2.*
                     Data node: db_dist_hypertable_1
                     Remote SQL: SELECT "time", device, temp, ctid FROM _timescaledb_internal._dist_hyper_1_1_chunk WHERE ((device = 1))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_1_4_chunk disttable_3
                     Output: disttable_3.tableoid, disttable_3.ctid, disttable_3.*
                     Data node: db_dist_hypertable_1
                     Remote SQL: SELECT "time", device, temp, ctid FROM _timescaledb_internal._dist_hyper_1_4_chunk WHERE ((device = 1))
(20 rows)

EXPLAIN (VERBOSE, COSTS FALSE)
DELETE FROM disttable WHERE device = 1;
                                                 QUERY PLAN                                                  
-------------------------------------------------------------------------------------------------------------
 Delete on public.disttable
   Delete on public.disttable disttable_1
   Foreign Delete on _timescaledb_internal._dist_hyper_1_1_chunk disttable_2
     Remote SQL: DELETE FROM _timescaledb_internal._dist_hyper_1_1_chunk WHERE ctid = $1
   Foreign Delete on _timescaledb_internal._dist_hyper_1_4_chunk disttable_3
     Remote SQL: DELETE FROM _timescaledb_internal._dist_hyper_1_4_chunk WHERE ctid = $1
   ->  Append
         ->  Seq Scan on public.disttable disttable_1
               Output: disttable_1.tableoid, disttable_1.ctid
               Filter: (disttable_1.device = 1)
         ->  Foreign Scan on _timescaledb_internal._dist_hyper_1_1_chunk disttable_2
               Output: disttable_2.tableoid, disttable_2.ctid
               Data node: db_dist_hypertable_1
               Remote SQL: SELECT ctid FROM _timescaledb_internal._dist_hyper_1_1_chunk WHERE ((device = 1))
         ->  Foreign Scan on _timescaledb_internal._dist_hyper_1_4_chunk disttable_3
               Output: disttable_3.tableoid, disttable_3.ctid
               Data node: db_dist_hypertable_1
               Remote SQL: SELECT ctid FROM _timescaledb_internal._dist_hyper_1_4_chunk WHERE ((device = 1))
(18 rows)

-- Test distributed ANALYZE.
--
-- First show no statistics
-- reltuples is initially -1 before any VACUUM/ANALYZE has been run on PG14
SELECT relname, relkind, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages
FROM pg_class
WHERE oid = 'disttable'::regclass;
  relname  | relkind | reltuples | relpages 
-----------+---------+-----------+----------
 disttable | r       |         0 |        0
(1 row)

SELECT relname, relkind, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages
FROM pg_class cl, (SELECT show_chunks AS chunk FROM show_chunks('disttable')) ch
WHERE cl.oid = ch.chunk::regclass;
        relname        | relkind | reltuples | relpages 
-----------------------+---------+-----------+----------
 _dist_hyper_1_1_chunk | f       |         0 |        0
 _dist_hyper_1_2_chunk | f       |         0 |        0
 _dist_hyper_1_3_chunk | f       |         0 |        0
 _dist_hyper_1_4_chunk | f       |         0 |        0
 _dist_hyper_1_5_chunk | f       |         0 |        0
 _dist_hyper_1_6_chunk | f       |         0 |        0
(6 rows)

ANALYZE disttable;
-- Show updated statistics
SELECT relname, relkind, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages
FROM pg_class
WHERE oid = 'disttable'::regclass;
  relname  | relkind | reltuples | relpages 
-----------+---------+-----------+----------
 disttable | r       |         0 |        0
(1 row)

SELECT relname, relkind, reltuples, relpages
FROM pg_class cl, (SELECT show_chunks AS chunk FROM show_chunks('disttable')) ch
WHERE cl.oid = ch.chunk::regclass;
        relname        | relkind | reltuples | relpages 
-----------------------+---------+-----------+----------
 _dist_hyper_1_1_chunk | f       |         2 |        1
 _dist_hyper_1_2_chunk | f       |         3 |        1
 _dist_hyper_1_3_chunk | f       |         3 |        1
 _dist_hyper_1_4_chunk | f       |         3 |        1
 _dist_hyper_1_5_chunk | f       |         3 |        1
 _dist_hyper_1_6_chunk | f       |         4 |        1
(6 rows)

-- Test distributed VACUUM.
--
VACUUM (FULL, ANALYZE) disttable;
VACUUM FULL disttable;
VACUUM disttable;
\set ON_ERROR_STOP 0
-- VACUUM VERBOSE is not supported at the moment
VACUUM VERBOSE disttable;
ERROR:  operation not supported on distributed hypertable
\set ON_ERROR_STOP 1
-- Test prepared statement
PREPARE dist_insert (timestamptz, int, float) AS
INSERT INTO disttable VALUES ($1, $2, $3);
EXECUTE dist_insert ('2017-01-01 06:05', 1, 1.4);
-- Show chunks created
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable');
 chunk_id | hypertable_id |      schema_name      |      table_name       | relkind |                                           slices                                            
----------+---------------+-----------------------+-----------------------+---------+---------------------------------------------------------------------------------------------
        1 |             1 | _timescaledb_internal | _dist_hyper_1_1_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
        2 |             1 | _timescaledb_internal | _dist_hyper_1_2_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
        3 |             1 | _timescaledb_internal | _dist_hyper_1_3_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}
        4 |             1 | _timescaledb_internal | _dist_hyper_1_4_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
        5 |             1 | _timescaledb_internal | _dist_hyper_1_5_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}
        6 |             1 | _timescaledb_internal | _dist_hyper_1_6_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(6 rows)

-- Show that there are assigned node_chunk_id:s in chunk data node mappings
SELECT * FROM _timescaledb_catalog.chunk_data_node ORDER BY 1,2,3;
 chunk_id | node_chunk_id |      node_name       
----------+---------------+----------------------
        1 |             1 | db_dist_hypertable_1
        2 |             1 | db_dist_hypertable_3
        3 |             1 | db_dist_hypertable_2
        4 |             2 | db_dist_hypertable_1
        5 |             2 | db_dist_hypertable_2
        6 |             2 | db_dist_hypertable_3
(6 rows)

SELECT * FROM hypertable_partitions;
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 disttable  |            2 | -9223372036854775808 | {db_dist_hypertable_1}
 disttable  |            2 |            715827882 | {db_dist_hypertable_2}
 disttable  |            2 |           1431655764 | {db_dist_hypertable_3}
(3 rows)

-- Show that chunks are created on data nodes and that each data node
-- has their own unique slice in the space (device) dimension.
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable');
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                                     
--------+-------------+---------------------+---------------------+-------+-------------------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_1_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
       2|            1|_timescaledb_internal|_dist_hyper_1_4_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                           
--------+-------------+---------------------+---------------------+-------+---------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_3_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}
       2|            1|_timescaledb_internal|_dist_hyper_1_5_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                                     
--------+-------------+---------------------+---------------------+-------+-------------------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_2_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
       2|            1|_timescaledb_internal|_dist_hyper_1_6_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(2 rows)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM disttable;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Sun Jan 01 08:01:00 2017 PST|     1| 1.2
Sun Jan 01 06:05:00 2017 PST|     1| 1.4
Mon Jul 02 08:01:00 2018 PDT|    87| 1.6
Mon Jul 02 09:01:00 2018 PDT|    87| 1.4
Mon Jul 02 09:21:00 2018 PDT|    87| 1.8
(6 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp
----------------------------+------+----
Mon Jan 02 08:01:00 2017 PST|     2| 1.3
Mon Jan 02 09:01:00 2017 PST|     2| 1.4
Mon Jan 02 08:21:00 2017 PST|     2| 1.5
Sun Jul 01 06:01:00 2018 PDT|    13| 1.4
Sun Jul 01 06:21:00 2018 PDT|    13| 1.5
Sun Jul 01 07:01:00 2018 PDT|    13| 1.4
(6 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 09:11:00 2017 PST|     3| 2.1
Sun Jan 01 09:21:00 2017 PST|     3| 2.2
Sun Jan 01 08:11:00 2017 PST|     3| 2.3
Sun Jul 01 09:11:00 2018 PDT|    90| 2.7
Sun Jul 01 08:01:00 2018 PDT|    29| 1.5
Sun Jul 01 09:21:00 2018 PDT|    90| 2.8
Sun Jul 01 08:21:00 2018 PDT|    29| 1.2
(7 rows)


 remote_exec 
-------------
 
(1 row)

SELECT node_name FROM timescaledb_information.data_nodes ORDER BY node_name;
      node_name       
----------------------
 db_dist_hypertable_1
 db_dist_hypertable_2
 db_dist_hypertable_3
(3 rows)

SELECT * FROM hypertable_detailed_size('disttable') ORDER BY node_name;
 table_bytes | index_bytes | toast_bytes | total_bytes |      node_name       
-------------+-------------+-------------+-------------+----------------------
       81920 |      122880 |           0 |      204800 | db_dist_hypertable_1
       81920 |      122880 |           0 |      204800 | db_dist_hypertable_2
       81920 |      122880 |           0 |      204800 | db_dist_hypertable_3
           0 |       24576 |           0 |       24576 | 
(4 rows)

-- Show what some queries would look like on the frontend
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT * FROM disttable;
                                                                     QUERY PLAN                                                                     
----------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: disttable."time", disttable.device, disttable.temp
   ->  Append
         ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
               Output: disttable_1."time", disttable_1.device, disttable_1.temp
               Data node: db_dist_hypertable_1
               Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
               Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
         ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
               Output: disttable_2."time", disttable_2.device, disttable_2.temp
               Data node: db_dist_hypertable_2
               Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
               Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
         ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
               Output: disttable_3."time", disttable_3.device, disttable_3.temp
               Data node: db_dist_hypertable_3
               Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
               Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
(18 rows)

SELECT * FROM disttable;
             time             | device | temp 
------------------------------+--------+------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2
(19 rows)

EXPLAIN (VERBOSE, COSTS FALSE)
SELECT time_bucket('3 hours', time) AS time, device, avg(temp) AS avg_temp
FROM disttable GROUP BY 1, 2
ORDER BY 1;
                                                                                                                           QUERY PLAN                                                                                                                           
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 GroupAggregate
   Output: (time_bucket('@ 3 hours'::interval, disttable."time")), disttable.device, avg(disttable.temp)
   Group Key: (time_bucket('@ 3 hours'::interval, disttable."time")), disttable.device
   ->  Custom Scan (AsyncAppend)
         Output: (time_bucket('@ 3 hours'::interval, disttable."time")), disttable.device, disttable.temp
         ->  Merge Append
               Sort Key: (time_bucket('@ 3 hours'::interval, disttable_1."time")), disttable_1.device
               ->  Result
                     Output: time_bucket('@ 3 hours'::interval, disttable_1."time"), disttable_1.device, disttable_1.temp
                     ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                           Output: disttable_1."time", disttable_1.device, disttable_1.temp
                           Data node: db_dist_hypertable_1
                           Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                           Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY public.time_bucket('03:00:00'::interval, "time") ASC NULLS LAST, device ASC NULLS LAST
               ->  Result
                     Output: time_bucket('@ 3 hours'::interval, disttable_2."time"), disttable_2.device, disttable_2.temp
                     ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                           Output: disttable_2."time", disttable_2.device, disttable_2.temp
                           Data node: db_dist_hypertable_2
                           Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                           Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY public.time_bucket('03:00:00'::interval, "time") ASC NULLS LAST, device ASC NULLS LAST
               ->  Result
                     Output: time_bucket('@ 3 hours'::interval, disttable_3."time"), disttable_3.device, disttable_3.temp
                     ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                           Output: disttable_3."time", disttable_3.device, disttable_3.temp
                           Data node: db_dist_hypertable_3
                           Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                           Remote SQL: SELECT "time", device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY public.time_bucket('03:00:00'::interval, "time") ASC NULLS LAST, device ASC NULLS LAST
(28 rows)

-- Execute some queries on the frontend and return the results
SELECT * FROM disttable;
             time             | device | temp 
------------------------------+--------+------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2
(19 rows)

SELECT time_bucket('3 hours', time) AS time, device, avg(temp) AS avg_temp
FROM disttable
GROUP BY 1, 2
ORDER BY 1;
             time             | device |     avg_temp     
------------------------------+--------+------------------
 Sun Jan 01 04:00:00 2017 PST |      1 |             1.25
 Sun Jan 01 07:00:00 2017 PST |      1 |              1.2
 Sun Jan 01 07:00:00 2017 PST |      3 |              2.2
 Mon Jan 02 07:00:00 2017 PST |      2 |              1.4
 Sun Jul 01 05:00:00 2018 PDT |     13 | 1.43333333333333
 Sun Jul 01 08:00:00 2018 PDT |     29 |             1.35
 Sun Jul 01 08:00:00 2018 PDT |     90 |             2.75
 Mon Jul 02 08:00:00 2018 PDT |     87 |              1.6
(8 rows)

SELECT time_bucket('3 hours', time) AS time, device, avg(temp) AS avg_temp
FROM disttable GROUP BY 1, 2
HAVING avg(temp) > 1.2
ORDER BY 1;
             time             | device |     avg_temp     
------------------------------+--------+------------------
 Sun Jan 01 04:00:00 2017 PST |      1 |             1.25
 Sun Jan 01 07:00:00 2017 PST |      3 |              2.2
 Mon Jan 02 07:00:00 2017 PST |      2 |              1.4
 Sun Jul 01 05:00:00 2018 PDT |     13 | 1.43333333333333
 Sun Jul 01 08:00:00 2018 PDT |     29 |             1.35
 Sun Jul 01 08:00:00 2018 PDT |     90 |             2.75
 Mon Jul 02 08:00:00 2018 PDT |     87 |              1.6
(7 rows)

SELECT time_bucket('3 hours', time) AS time, device, avg(temp) AS avg_temp
FROM disttable
WHERE temp > 2
GROUP BY 1, 2
HAVING avg(temp) > 1.2
ORDER BY 1;
             time             | device | avg_temp 
------------------------------+--------+----------
 Sun Jan 01 07:00:00 2017 PST |      3 |      2.2
 Sun Jul 01 08:00:00 2018 PDT |     90 |     2.75
(2 rows)

-- Test AsyncAppend when using min/max aggregates
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT max(temp)
FROM disttable;
                                                                                                    QUERY PLAN                                                                                                    
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Result
   Output: $0
   InitPlan 1 (returns $0)
     ->  Limit
           Output: disttable.temp
           ->  Custom Scan (AsyncAppend)
                 Output: disttable.temp
                 ->  Merge Append
                       Sort Key: disttable_1.temp DESC
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                             Output: disttable_1.temp
                             Data node: db_dist_hypertable_1
                             Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                             Output: disttable_2.temp
                             Data node: db_dist_hypertable_2
                             Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                             Output: disttable_3.temp
                             Data node: db_dist_hypertable_3
                             Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
(24 rows)

SELECT max(temp)
FROM disttable;
 max 
-----
 2.8
(1 row)

-- Test turning off async append
SET timescaledb.enable_async_append = OFF;
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT max(temp)
FROM disttable;
                                                                                                 QUERY PLAN                                                                                                 
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Result
   Output: $0
   InitPlan 1 (returns $0)
     ->  Limit
           Output: disttable_1.temp
           ->  Merge Append
                 Sort Key: disttable_1.temp DESC
                 ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                       Output: disttable_1.temp
                       Data node: db_dist_hypertable_1
                       Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                       Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                 ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                       Output: disttable_2.temp
                       Data node: db_dist_hypertable_2
                       Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                       Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                 ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                       Output: disttable_3.temp
                       Data node: db_dist_hypertable_3
                       Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                       Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
(22 rows)

SET timescaledb.enable_async_append = ON;
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT min(temp), max(temp)
FROM disttable;
                                                                QUERY PLAN                                                                
------------------------------------------------------------------------------------------------------------------------------------------
 Aggregate
   Output: min(disttable.temp), max(disttable.temp)
   ->  Custom Scan (AsyncAppend)
         Output: disttable.temp
         ->  Append
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                     Output: disttable_1.temp
                     Data node: db_dist_hypertable_1
                     Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                     Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                     Output: disttable_2.temp
                     Data node: db_dist_hypertable_2
                     Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                     Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                     Output: disttable_3.temp
                     Data node: db_dist_hypertable_3
                     Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                     Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2])
(20 rows)

SELECT min(temp), max(temp)
FROM disttable;
 min | max 
-----+-----
 1.1 | 2.8
(1 row)

-- Test AsyncAppend when using window functions
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT device, temp, avg(temp) OVER (PARTITION BY device)
FROM disttable
ORDER BY device, temp;
                                                                                   QUERY PLAN                                                                                    
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Incremental Sort
   Output: disttable_1.device, disttable_1.temp, (avg(disttable_1.temp) OVER (?))
   Sort Key: disttable_1.device, disttable_1.temp
   Presorted Key: disttable_1.device
   ->  WindowAgg
         Output: disttable_1.device, disttable_1.temp, avg(disttable_1.temp) OVER (?)
         ->  Merge Append
               Sort Key: disttable_1.device
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                     Output: disttable_1.device, disttable_1.temp
                     Data node: db_dist_hypertable_1
                     Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                     Remote SQL: SELECT device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY device ASC NULLS LAST
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                     Output: disttable_2.device, disttable_2.temp
                     Data node: db_dist_hypertable_2
                     Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                     Remote SQL: SELECT device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY device ASC NULLS LAST
               ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                     Output: disttable_3.device, disttable_3.temp
                     Data node: db_dist_hypertable_3
                     Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                     Remote SQL: SELECT device, temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) ORDER BY device ASC NULLS LAST
(23 rows)

SELECT device, temp, avg(temp) OVER (PARTITION BY device)
FROM disttable
ORDER BY device, temp;
 device | temp |       avg        
--------+------+------------------
      1 |  1.1 | 1.23333333333333
      1 |  1.2 | 1.23333333333333
      1 |  1.4 | 1.23333333333333
      2 |  1.3 |              1.4
      2 |  1.4 |              1.4
      2 |  1.5 |              1.4
      3 |  2.1 |              2.2
      3 |  2.2 |              2.2
      3 |  2.3 |              2.2
     13 |  1.4 | 1.43333333333333
     13 |  1.4 | 1.43333333333333
     13 |  1.5 | 1.43333333333333
     29 |  1.2 |             1.35
     29 |  1.5 |             1.35
     87 |  1.4 |              1.6
     87 |  1.6 |              1.6
     87 |  1.8 |              1.6
     90 |  2.7 |             2.75
     90 |  2.8 |             2.75
(19 rows)

-- Test remote explain
-- Make sure that chunks_in function only expects one-dimensional integer arrays
\set ON_ERROR_STOP 0
SELECT "time" FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[[2], [1]])
ORDER BY "time" DESC NULLS FIRST LIMIT 1;
ERROR:  invalid number of array dimensions for chunks_in
\set ON_ERROR_STOP 1
SET timescaledb.enable_remote_explain = ON;
-- Check that datanodes use ChunkAppend plans with chunks_in function in the
-- "Remote SQL" when using max(time).
EXPLAIN (VERBOSE, COSTS FALSE)
SELECT max(time)
FROM disttable;
                                                                                                       QUERY PLAN                                                                                                       
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Result
   Output: $0
   InitPlan 1 (returns $0)
     ->  Limit
           Output: disttable."time"
           ->  Custom Scan (AsyncAppend)
                 Output: disttable."time"
                 ->  Merge Append
                       Sort Key: disttable_1."time" DESC
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                             Output: disttable_1."time"
                             Data node: db_dist_hypertable_1
                             Chunks: _dist_hyper_1_4_chunk, _dist_hyper_1_1_chunk
                             Remote SQL: SELECT "time" FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[2, 1]) AND (("time" IS NOT NULL)) ORDER BY "time" DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: disttable."time"
                                 ->  Custom Scan (ChunkAppend) on public.disttable
                                       Output: disttable."time"
                                       Order: disttable."time" DESC
                                       Startup Exclusion: false
                                       Runtime Exclusion: false
                                       ->  Index Only Scan using _dist_hyper_1_4_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_4_chunk
                                             Output: _dist_hyper_1_4_chunk."time"
                                             Index Cond: (_dist_hyper_1_4_chunk."time" IS NOT NULL)
                                       ->  Index Only Scan using _dist_hyper_1_1_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_1_chunk
                                             Output: _dist_hyper_1_1_chunk."time"
                                             Index Cond: (_dist_hyper_1_1_chunk."time" IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                             Output: disttable_2."time"
                             Data node: db_dist_hypertable_2
                             Chunks: _dist_hyper_1_5_chunk, _dist_hyper_1_3_chunk
                             Remote SQL: SELECT "time" FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[2, 1]) AND (("time" IS NOT NULL)) ORDER BY "time" DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: disttable."time"
                                 ->  Custom Scan (ChunkAppend) on public.disttable
                                       Output: disttable."time"
                                       Order: disttable."time" DESC
                                       Startup Exclusion: false
                                       Runtime Exclusion: false
                                       ->  Index Only Scan using _dist_hyper_1_5_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_5_chunk
                                             Output: _dist_hyper_1_5_chunk."time"
                                             Index Cond: (_dist_hyper_1_5_chunk."time" IS NOT NULL)
                                       ->  Index Only Scan using _dist_hyper_1_3_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_3_chunk
                                             Output: _dist_hyper_1_3_chunk."time"
                                             Index Cond: (_dist_hyper_1_3_chunk."time" IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                             Output: disttable_3."time"
                             Data node: db_dist_hypertable_3
                             Chunks: _dist_hyper_1_6_chunk, _dist_hyper_1_2_chunk
                             Remote SQL: SELECT "time" FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[2, 1]) AND (("time" IS NOT NULL)) ORDER BY "time" DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: disttable."time"
                                 ->  Custom Scan (ChunkAppend) on public.disttable
                                       Output: disttable."time"
                                       Order: disttable."time" DESC
                                       Startup Exclusion: false
                                       Runtime Exclusion: false
                                       ->  Index Only Scan using _dist_hyper_1_6_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_6_chunk
                                             Output: _dist_hyper_1_6_chunk."time"
                                             Index Cond: (_dist_hyper_1_6_chunk."time" IS NOT NULL)
                                       ->  Index Only Scan using _dist_hyper_1_2_chunk_disttable_time_idx on _timescaledb_internal._dist_hyper_1_2_chunk
                                             Output: _dist_hyper_1_2_chunk."time"
                                             Index Cond: (_dist_hyper_1_2_chunk."time" IS NOT NULL)
 
(69 rows)

EXPLAIN (VERBOSE, COSTS FALSE)
SELECT max(temp)
FROM disttable;
                                                                                                    QUERY PLAN                                                                                                    
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Result
   Output: $0
   InitPlan 1 (returns $0)
     ->  Limit
           Output: disttable.temp
           ->  Custom Scan (AsyncAppend)
                 Output: disttable.temp
                 ->  Merge Append
                       Sort Key: disttable_1.temp DESC
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_1
                             Output: disttable_1.temp
                             Data node: db_dist_hypertable_1
                             Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: _dist_hyper_1_1_chunk.temp
                                 ->  Sort
                                       Output: _dist_hyper_1_1_chunk.temp
                                       Sort Key: _dist_hyper_1_1_chunk.temp DESC
                                       ->  Append
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_1_chunk
                                                   Output: _dist_hyper_1_1_chunk.temp
                                                   Filter: (_dist_hyper_1_1_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_4_chunk
                                                   Output: _dist_hyper_1_4_chunk.temp
                                                   Filter: (_dist_hyper_1_4_chunk.temp IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_2
                             Output: disttable_2.temp
                             Data node: db_dist_hypertable_2
                             Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: _dist_hyper_1_3_chunk.temp
                                 ->  Sort
                                       Output: _dist_hyper_1_3_chunk.temp
                                       Sort Key: _dist_hyper_1_3_chunk.temp DESC
                                       ->  Append
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_3_chunk
                                                   Output: _dist_hyper_1_3_chunk.temp
                                                   Filter: (_dist_hyper_1_3_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_5_chunk
                                                   Output: _dist_hyper_1_5_chunk.temp
                                                   Filter: (_dist_hyper_1_5_chunk.temp IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_3
                             Output: disttable_3.temp
                             Data node: db_dist_hypertable_3
                             Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit
                                 Output: _dist_hyper_1_2_chunk.temp
                                 ->  Sort
                                       Output: _dist_hyper_1_2_chunk.temp
                                       Sort Key: _dist_hyper_1_2_chunk.temp DESC
                                       ->  Append
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_2_chunk
                                                   Output: _dist_hyper_1_2_chunk.temp
                                                   Filter: (_dist_hyper_1_2_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_6_chunk
                                                   Output: _dist_hyper_1_6_chunk.temp
                                                   Filter: (_dist_hyper_1_6_chunk.temp IS NOT NULL)
 
(66 rows)

-- Don't remote explain if there is no VERBOSE flag
EXPLAIN (COSTS FALSE)
SELECT max(temp)
FROM disttable;
                                  QUERY PLAN                                   
-------------------------------------------------------------------------------
 Result
   InitPlan 1 (returns $0)
     ->  Limit
           ->  Custom Scan (AsyncAppend)
                 ->  Merge Append
                       Sort Key: disttable_1.temp DESC
                       ->  Custom Scan (DataNodeScan) on disttable disttable_1
                       ->  Custom Scan (DataNodeScan) on disttable disttable_2
                       ->  Custom Scan (DataNodeScan) on disttable disttable_3
(9 rows)

-- Test additional EXPLAIN flags
EXPLAIN (ANALYZE, VERBOSE, COSTS FALSE, BUFFERS OFF, TIMING OFF, SUMMARY OFF)
SELECT max(temp)
FROM disttable;
                                                                                                    QUERY PLAN                                                                                                    
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Result (actual rows=1 loops=1)
   Output: $0
   InitPlan 1 (returns $0)
     ->  Limit (actual rows=1 loops=1)
           Output: disttable.temp
           ->  Custom Scan (AsyncAppend) (actual rows=1 loops=1)
                 Output: disttable.temp
                 ->  Merge Append (actual rows=1 loops=1)
                       Sort Key: disttable_1.temp DESC
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_1 (actual rows=1 loops=1)
                             Output: disttable_1.temp
                             Data node: db_dist_hypertable_1
                             Fetcher Type: COPY
                             Chunks: _dist_hyper_1_1_chunk, _dist_hyper_1_4_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit (actual rows=1 loops=1)
                                 Output: _dist_hyper_1_1_chunk.temp
                                 ->  Sort (actual rows=1 loops=1)
                                       Output: _dist_hyper_1_1_chunk.temp
                                       Sort Key: _dist_hyper_1_1_chunk.temp DESC
                                       Sort Method: top-N heapsort 
                                       ->  Append (actual rows=6 loops=1)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_1_chunk (actual rows=3 loops=1)
                                                   Output: _dist_hyper_1_1_chunk.temp
                                                   Filter: (_dist_hyper_1_1_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_4_chunk (actual rows=3 loops=1)
                                                   Output: _dist_hyper_1_4_chunk.temp
                                                   Filter: (_dist_hyper_1_4_chunk.temp IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_2 (actual rows=1 loops=1)
                             Output: disttable_2.temp
                             Data node: db_dist_hypertable_2
                             Fetcher Type: COPY
                             Chunks: _dist_hyper_1_3_chunk, _dist_hyper_1_5_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit (actual rows=1 loops=1)
                                 Output: _dist_hyper_1_3_chunk.temp
                                 ->  Sort (actual rows=1 loops=1)
                                       Output: _dist_hyper_1_3_chunk.temp
                                       Sort Key: _dist_hyper_1_3_chunk.temp DESC
                                       Sort Method: top-N heapsort 
                                       ->  Append (actual rows=6 loops=1)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_3_chunk (actual rows=3 loops=1)
                                                   Output: _dist_hyper_1_3_chunk.temp
                                                   Filter: (_dist_hyper_1_3_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_5_chunk (actual rows=3 loops=1)
                                                   Output: _dist_hyper_1_5_chunk.temp
                                                   Filter: (_dist_hyper_1_5_chunk.temp IS NOT NULL)
 
                       ->  Custom Scan (DataNodeScan) on public.disttable disttable_3 (actual rows=1 loops=1)
                             Output: disttable_3.temp
                             Data node: db_dist_hypertable_3
                             Fetcher Type: COPY
                             Chunks: _dist_hyper_1_2_chunk, _dist_hyper_1_6_chunk
                             Remote SQL: SELECT temp FROM public.disttable WHERE _timescaledb_internal.chunks_in(public.disttable.*, ARRAY[1, 2]) AND ((temp IS NOT NULL)) ORDER BY temp DESC NULLS FIRST LIMIT 1
                             Remote EXPLAIN: 
                               Limit (actual rows=1 loops=1)
                                 Output: _dist_hyper_1_2_chunk.temp
                                 ->  Sort (actual rows=1 loops=1)
                                       Output: _dist_hyper_1_2_chunk.temp
                                       Sort Key: _dist_hyper_1_2_chunk.temp DESC
                                       Sort Method: top-N heapsort 
                                       ->  Append (actual rows=7 loops=1)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_2_chunk (actual rows=3 loops=1)
                                                   Output: _dist_hyper_1_2_chunk.temp
                                                   Filter: (_dist_hyper_1_2_chunk.temp IS NOT NULL)
                                             ->  Seq Scan on _timescaledb_internal._dist_hyper_1_6_chunk (actual rows=4 loops=1)
                                                   Output: _dist_hyper_1_6_chunk.temp
                                                   Filter: (_dist_hyper_1_6_chunk.temp IS NOT NULL)
 
(72 rows)

-- The constraints, indexes, and triggers on foreign chunks. Only
-- check constraints should recurse to foreign chunks (although they
-- aren't enforced on a foreign table)
SELECT st."Child" as chunk_relid, test.show_constraints((st)."Child")
FROM test.show_subtables('disttable') st;
                 chunk_relid                 |                                                                                   show_constraints                                                                                   
---------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 _timescaledb_internal._dist_hyper_1_1_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (constraint_2,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) < 715827882)",f,f,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (constraint_3,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) >= 1431655764)",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (constraint_4,c,{device},-,"((_timescaledb_internal.get_partition_hash(device) >= 715827882) AND (_timescaledb_internal.get_partition_hash(device) < 1431655764))",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (constraint_2,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) < 715827882)",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (constraint_4,c,{device},-,"((_timescaledb_internal.get_partition_hash(device) >= 715827882) AND (_timescaledb_internal.get_partition_hash(device) < 1431655764))",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (constraint_3,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) >= 1431655764)",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
(18 rows)

SELECT st."Child" as chunk_relid, test.show_indexes((st)."Child")
FROM test.show_subtables('disttable') st;
 chunk_relid | show_indexes 
-------------+--------------
(0 rows)

SELECT st."Child" as chunk_relid, test.show_triggers((st)."Child")
FROM test.show_subtables('disttable') st;
 chunk_relid | show_triggers 
-------------+---------------
(0 rows)

-- Check that the chunks are assigned data nodes
SELECT * FROM _timescaledb_catalog.chunk_data_node ORDER BY 1,2,3;
 chunk_id | node_chunk_id |      node_name       
----------+---------------+----------------------
        1 |             1 | db_dist_hypertable_1
        2 |             1 | db_dist_hypertable_3
        3 |             1 | db_dist_hypertable_2
        4 |             2 | db_dist_hypertable_1
        5 |             2 | db_dist_hypertable_2
        6 |             2 | db_dist_hypertable_3
(6 rows)

-- Adding a new trigger should not recurse to foreign chunks
CREATE TRIGGER _1_test_trigger_insert
    AFTER INSERT ON disttable
    FOR EACH ROW EXECUTE FUNCTION test_trigger();
SELECT st."Child" as chunk_relid, test.show_triggers((st)."Child")
FROM test.show_subtables('disttable') st;
 chunk_relid | show_triggers 
-------------+---------------
(0 rows)

-- Check that we can create indexes on distributed hypertables and
-- that they don't recurse to foreign chunks
CREATE INDEX ON disttable (time, device);
SELECT * FROM test.show_indexes('disttable');
           Index           |    Columns    | Expr | Unique | Primary | Exclusion | Tablespace 
---------------------------+---------------+------+--------+---------+-----------+------------
 disttable_device_time_idx | {device,time} |      | f      | f       | f         | 
 disttable_pkey            | {time,device} |      | t      | t       | f         | 
 disttable_time_device_idx | {time,device} |      | f      | f       | f         | 
 disttable_time_idx        | {time}        |      | f      | f       | f         | 
(4 rows)

SELECT st."Child" as chunk_relid, test.show_indexes((st)."Child")
FROM test.show_subtables('disttable') st;
 chunk_relid | show_indexes 
-------------+--------------
(0 rows)

-- No index mappings should exist either
SELECT * FROM _timescaledb_catalog.chunk_index;
 chunk_id | index_name | hypertable_id | hypertable_index_name 
----------+------------+---------------+-----------------------
(0 rows)

-- Check that creating columns work
ALTER TABLE disttable ADD COLUMN "Color" int;
SELECT * FROM test.show_columns('disttable');
 Column |           Type           | NotNull 
--------+--------------------------+---------
 time   | timestamp with time zone | t
 device | integer                  | t
 temp   | double precision         | f
 Color  | integer                  | f
(4 rows)

SELECT st."Child" as chunk_relid, test.show_columns((st)."Child")
FROM test.show_subtables('disttable') st;
                 chunk_relid                 |            show_columns             
---------------------------------------------+-------------------------------------
 _timescaledb_internal._dist_hyper_1_1_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_1_chunk | (Color,integer,f)
 _timescaledb_internal._dist_hyper_1_2_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_2_chunk | (Color,integer,f)
 _timescaledb_internal._dist_hyper_1_3_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_3_chunk | (Color,integer,f)
 _timescaledb_internal._dist_hyper_1_4_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_4_chunk | (Color,integer,f)
 _timescaledb_internal._dist_hyper_1_5_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_5_chunk | (Color,integer,f)
 _timescaledb_internal._dist_hyper_1_6_chunk | (time,"timestamp with time zone",t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (device,integer,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (temp,"double precision",f)
 _timescaledb_internal._dist_hyper_1_6_chunk | (Color,integer,f)
(24 rows)

-- Adding a new unique constraint should not recurse to foreign
-- chunks, but a check constraint should
ALTER TABLE disttable ADD CONSTRAINT disttable_color_unique UNIQUE (time, device, "Color");
ALTER TABLE disttable ADD CONSTRAINT disttable_temp_non_negative CHECK (temp > 0.0);
SELECT st."Child" as chunk_relid, test.show_constraints((st)."Child")
FROM test.show_subtables('disttable') st;
                 chunk_relid                 |                                                                                   show_constraints                                                                                   
---------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 _timescaledb_internal._dist_hyper_1_1_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (constraint_2,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) < 715827882)",f,f,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_1_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (constraint_3,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) >= 1431655764)",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_2_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (constraint_1,c,{time},-,"((""time"" >= 'Wed Dec 28 16:00:00 2016 PST'::timestamp with time zone) AND (""time"" < 'Wed Jan 04 16:00:00 2017 PST'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (constraint_4,c,{device},-,"((_timescaledb_internal.get_partition_hash(device) >= 715827882) AND (_timescaledb_internal.get_partition_hash(device) < 1431655764))",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_3_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (constraint_2,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) < 715827882)",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_4_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (constraint_4,c,{device},-,"((_timescaledb_internal.get_partition_hash(device) >= 715827882) AND (_timescaledb_internal.get_partition_hash(device) < 1431655764))",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_5_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (constraint_3,c,{device},-,"(_timescaledb_internal.get_partition_hash(device) >= 1431655764)",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (constraint_5,c,{time},-,"((""time"" >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (""time"" < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone))",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (disttable_device_check,c,{device},-,"(device > 0)",f,f,t)
 _timescaledb_internal._dist_hyper_1_6_chunk | (disttable_temp_non_negative,c,{temp},-,"(temp > (0.0)::double precision)",f,f,t)
(24 rows)

SELECT cc.*
FROM (SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
      FROM show_chunks('disttable')) c,
      _timescaledb_catalog.chunk_constraint cc
WHERE c.chunk_id = cc.chunk_id;
 chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name 
----------+--------------------+-----------------+----------------------------
        1 |                  2 | constraint_2    | 
        1 |                  1 | constraint_1    | 
        2 |                  3 | constraint_3    | 
        2 |                  1 | constraint_1    | 
        3 |                  4 | constraint_4    | 
        3 |                  1 | constraint_1    | 
        4 |                  2 | constraint_2    | 
        4 |                  5 | constraint_5    | 
        5 |                  4 | constraint_4    | 
        5 |                  5 | constraint_5    | 
        6 |                  3 | constraint_3    | 
        6 |                  5 | constraint_5    | 
(12 rows)

-- Show contents after re-adding column
SELECT * FROM disttable;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1 |      
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2 |      
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4 |      
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6 |      
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4 |      
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8 |      
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3 |      
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4 |      
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5 |      
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5 |      
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1 |      
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2 |      
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3 |      
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7 |      
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5 |      
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8 |      
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2 |      
(19 rows)

-- Test INSERTS with RETURNING. Since we previously dropped a column
-- on the hypertable, this also tests that we handle conversion of the
-- attribute numbers in the RETURNING clause, since they now differ
-- between the hypertable root relation and the chunk currently
-- RETURNING from.
INSERT INTO disttable (time, device, "Color", temp)
VALUES ('2017-09-02 06:09', 4, 1, 9.8)
RETURNING time, "Color", temp;
             time             | Color | temp 
------------------------------+-------+------
 Sat Sep 02 06:09:00 2017 PDT |     1 |  9.8
(1 row)

INSERT INTO disttable (time, device, "Color", temp)
VALUES ('2017-09-03 06:18', 9, 3, 8.7)
RETURNING 1;
 ?column? 
----------
        1
(1 row)

-- On conflict
INSERT INTO disttable (time, device, "Color", temp)
VALUES ('2017-09-02 06:09', 6, 2, 10.5)
ON CONFLICT DO NOTHING;
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable');
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                                     
--------+-------------+---------------------+---------------------+-------+-------------------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_1_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
       2|            1|_timescaledb_internal|_dist_hyper_1_4_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
       3|            1|_timescaledb_internal|_dist_hyper_1_9_chunk|r      |{"time": [1504137600000000, 1504742400000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                           
--------+-------------+---------------------+---------------------+-------+---------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_3_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}
       2|            1|_timescaledb_internal|_dist_hyper_1_5_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}
       3|            1|_timescaledb_internal|_dist_hyper_1_7_chunk|r      |{"time": [1504137600000000, 1504742400000000], "device": [715827882, 1431655764]}
(3 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name           |relkind|slices                                                                                     
--------+-------------+---------------------+---------------------+-------+-------------------------------------------------------------------------------------------
       1|            1|_timescaledb_internal|_dist_hyper_1_2_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
       2|            1|_timescaledb_internal|_dist_hyper_1_6_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
       3|            1|_timescaledb_internal|_dist_hyper_1_8_chunk|r      |{"time": [1504137600000000, 1504742400000000], "device": [1431655764, 9223372036854775807]}
(3 rows)


 remote_exec 
-------------
 
(1 row)

-- Show new row and that conflicting row is not inserted
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM disttable;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp|Color
----------------------------+------+----+-----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1|     
Sun Jan 01 08:01:00 2017 PST|     1| 1.2|     
Sun Jan 01 06:05:00 2017 PST|     1| 1.4|     
Mon Jul 02 08:01:00 2018 PDT|    87| 1.6|     
Mon Jul 02 09:01:00 2018 PDT|    87| 1.4|     
Mon Jul 02 09:21:00 2018 PDT|    87| 1.8|     
Sat Sep 02 06:09:00 2017 PDT|     6|10.5|    2
(7 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp|Color
----------------------------+------+----+-----
Mon Jan 02 08:01:00 2017 PST|     2| 1.3|     
Mon Jan 02 09:01:00 2017 PST|     2| 1.4|     
Mon Jan 02 08:21:00 2017 PST|     2| 1.5|     
Sun Jul 01 06:01:00 2018 PDT|    13| 1.4|     
Sun Jul 01 06:21:00 2018 PDT|    13| 1.5|     
Sun Jul 01 07:01:00 2018 PDT|    13| 1.4|     
Sat Sep 02 06:09:00 2017 PDT|     4| 9.8|    1
(7 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp|Color
----------------------------+------+----+-----
Sun Jan 01 09:11:00 2017 PST|     3| 2.1|     
Sun Jan 01 09:21:00 2017 PST|     3| 2.2|     
Sun Jan 01 08:11:00 2017 PST|     3| 2.3|     
Sun Jul 01 09:11:00 2018 PDT|    90| 2.7|     
Sun Jul 01 08:01:00 2018 PDT|    29| 1.5|     
Sun Jul 01 09:21:00 2018 PDT|    90| 2.8|     
Sun Jul 01 08:21:00 2018 PDT|    29| 1.2|     
Sun Sep 03 06:18:00 2017 PDT|     9| 8.7|    3
(8 rows)


 remote_exec 
-------------
 
(1 row)

\set ON_ERROR_STOP 0
-- ON CONFLICT DO NOTHING only works when index inference is omitted
\set VERBOSITY default
INSERT INTO disttable
VALUES ('2017-09-02 06:09', 6)
ON CONFLICT(time,device) DO NOTHING;
ERROR:  could not find arbiter index for hypertable index "disttable_pkey" on chunk "_dist_hyper_1_9_chunk"
HINT:  Omit the index inference specification for the distributed hypertable in the ON CONFLICT clause.
INSERT INTO disttable
VALUES ('2017-09-02 06:09', 6)
ON CONFLICT(time,device,"Color") DO NOTHING;
ERROR:  could not find arbiter index for hypertable index "disttable_color_unique" on chunk "_dist_hyper_1_9_chunk"
HINT:  Omit the index inference specification for the distributed hypertable in the ON CONFLICT clause.
INSERT INTO disttable
VALUES ('2017-09-02 06:09', 6)
ON CONFLICT ON CONSTRAINT disttable_color_unique DO NOTHING;
ERROR:  could not find arbiter index for hypertable index "disttable_color_unique" on chunk "_dist_hyper_1_9_chunk"
HINT:  Omit the index inference specification for the distributed hypertable in the ON CONFLICT clause.
\set VERBOSITY terse
SELECT * FROM disttable ORDER BY disttable;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1 |      
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4 |      
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2 |      
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3 |      
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1 |      
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2 |      
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3 |      
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5 |      
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4 |      
 Sat Sep 02 06:09:00 2017 PDT |      4 |  9.8 |     1
 Sat Sep 02 06:09:00 2017 PDT |      6 | 10.5 |     2
 Sun Sep 03 06:18:00 2017 PDT |      9 |  8.7 |     3
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5 |      
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5 |      
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2 |      
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7 |      
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8 |      
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6 |      
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4 |      
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8 |      
(22 rows)

-- ON CONFLICT only works with DO NOTHING for now
INSERT INTO disttable (time, device, "Color", temp)
VALUES ('2017-09-09 08:13', 7, 3, 27.5)
ON CONFLICT (time) DO UPDATE SET temp = 3.2;
ERROR:  ON CONFLICT DO UPDATE not supported on distributed hypertables
-- Test that an INSERT that would create a chunk does not work on a
-- data node
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1'], $$
    INSERT INTO disttable VALUES ('2019-01-02 12:34', 1, 2, 9.3)
$$);
NOTICE:  [db_dist_hypertable_1]: 
    INSERT INTO disttable VALUES ('2019-01-02 12:34', 1, 2, 9.3)

ERROR:  [db_dist_hypertable_1]: distributed hypertable member cannot create chunk on its own
\set ON_ERROR_STOP 1
-- However, INSERTs on a data node that does not create a chunk works.
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1'], $$
    INSERT INTO disttable VALUES ('2017-09-03 06:09', 1, 2, 9.3)
$$);
NOTICE:  [db_dist_hypertable_1]: 
    INSERT INTO disttable VALUES ('2017-09-03 06:09', 1, 2, 9.3)

 remote_exec 
-------------
 
(1 row)

-- Test updates
UPDATE disttable SET "Color" = 4 WHERE "Color" = 3;
SELECT * FROM disttable;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1 |      
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2 |      
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4 |      
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6 |      
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4 |      
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8 |      
 Sat Sep 02 06:09:00 2017 PDT |      6 | 10.5 |     2
 Sun Sep 03 06:09:00 2017 PDT |      1 |    2 |     9
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3 |      
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4 |      
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5 |      
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5 |      
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4 |      
 Sat Sep 02 06:09:00 2017 PDT |      4 |  9.8 |     1
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1 |      
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2 |      
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3 |      
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7 |      
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5 |      
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8 |      
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2 |      
 Sun Sep 03 06:18:00 2017 PDT |      9 |  8.7 |     4
(23 rows)

WITH devices AS (
     SELECT DISTINCT device FROM disttable ORDER BY device
)
UPDATE disttable SET "Color" = 2 WHERE device = (SELECT device FROM devices LIMIT 1);
\set ON_ERROR_STOP 0
-- Updates referencing non-existing column
UPDATE disttable SET device = 4 WHERE no_such_column = 2;
ERROR:  column "no_such_column" does not exist at character 39
UPDATE disttable SET no_such_column = 4 WHERE device = 2;
ERROR:  column "no_such_column" of relation "disttable" does not exist at character 22
-- Update to system column
UPDATE disttable SET tableoid = 4 WHERE device = 2;
ERROR:  cannot assign to system column "tableoid" at character 22
\set ON_ERROR_STOP 1
-- Test deletes (no rows deleted)
DELETE FROM disttable WHERE device = 3
RETURNING *;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 09:11:00 2017 PST |      3 |  2.1 |      
 Sun Jan 01 09:21:00 2017 PST |      3 |  2.2 |      
 Sun Jan 01 08:11:00 2017 PST |      3 |  2.3 |      
(3 rows)

DELETE FROM disttable WHERE time IS NULL;
-- Test deletes (rows deleted)
DELETE FROM disttable WHERE device = 4
RETURNING *;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sat Sep 02 06:09:00 2017 PDT |      4 |  9.8 |     1
(1 row)

-- Query to show that rows are deleted
SELECT * FROM disttable;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1 |     2
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2 |     2
 Sun Jan 01 06:05:00 2017 PST |      1 |  1.4 |     2
 Mon Jul 02 08:01:00 2018 PDT |     87 |  1.6 |      
 Mon Jul 02 09:01:00 2018 PDT |     87 |  1.4 |      
 Mon Jul 02 09:21:00 2018 PDT |     87 |  1.8 |      
 Sat Sep 02 06:09:00 2017 PDT |      6 | 10.5 |     2
 Sun Sep 03 06:09:00 2017 PDT |      1 |    2 |     2
 Mon Jan 02 08:01:00 2017 PST |      2 |  1.3 |      
 Mon Jan 02 09:01:00 2017 PST |      2 |  1.4 |      
 Mon Jan 02 08:21:00 2017 PST |      2 |  1.5 |      
 Sun Jul 01 06:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 06:21:00 2018 PDT |     13 |  1.5 |      
 Sun Jul 01 07:01:00 2018 PDT |     13 |  1.4 |      
 Sun Jul 01 09:11:00 2018 PDT |     90 |  2.7 |      
 Sun Jul 01 08:01:00 2018 PDT |     29 |  1.5 |      
 Sun Jul 01 09:21:00 2018 PDT |     90 |  2.8 |      
 Sun Jul 01 08:21:00 2018 PDT |     29 |  1.2 |      
 Sun Sep 03 06:18:00 2017 PDT |      9 |  8.7 |     4
(19 rows)

-- Ensure rows are deleted on the data nodes
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM disttable;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp|Color
----------------------------+------+----+-----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1|    2
Sun Jan 01 08:01:00 2017 PST|     1| 1.2|    2
Sun Jan 01 06:05:00 2017 PST|     1| 1.4|    2
Mon Jul 02 08:01:00 2018 PDT|    87| 1.6|     
Mon Jul 02 09:01:00 2018 PDT|    87| 1.4|     
Mon Jul 02 09:21:00 2018 PDT|    87| 1.8|     
Sat Sep 02 06:09:00 2017 PDT|     6|10.5|    2
Sun Sep 03 06:09:00 2017 PDT|     1|   2|    2
(8 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp|Color
----------------------------+------+----+-----
Mon Jan 02 08:01:00 2017 PST|     2| 1.3|     
Mon Jan 02 09:01:00 2017 PST|     2| 1.4|     
Mon Jan 02 08:21:00 2017 PST|     2| 1.5|     
Sun Jul 01 06:01:00 2018 PDT|    13| 1.4|     
Sun Jul 01 06:21:00 2018 PDT|    13| 1.5|     
Sun Jul 01 07:01:00 2018 PDT|    13| 1.4|     
(6 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp|Color
----------------------------+------+----+-----
Sun Jul 01 09:11:00 2018 PDT|    90| 2.7|     
Sun Jul 01 08:01:00 2018 PDT|    29| 1.5|     
Sun Jul 01 09:21:00 2018 PDT|    90| 2.8|     
Sun Jul 01 08:21:00 2018 PDT|    29| 1.2|     
Sun Sep 03 06:18:00 2017 PDT|     9| 8.7|    4
(5 rows)


 remote_exec 
-------------
 
(1 row)

-- Test TRUNCATE
TRUNCATE disttable;
-- No data should remain
SELECT * FROM disttable;
 time | device | temp | Color 
------+--------+------+-------
(0 rows)

-- Metadata and tables cleaned up
SELECT * FROM _timescaledb_catalog.chunk;
 id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk 
----+---------------+-------------+------------+---------------------+---------+--------+-----------
(0 rows)

SELECT * FROM show_chunks('disttable');
 show_chunks 
-------------
(0 rows)

-- Also cleaned up remotely
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM _timescaledb_catalog.chunk;
SELECT * FROM show_chunks('disttable');
SELECT * FROM disttable;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM _timescaledb_catalog.chunk
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|schema_name|table_name|compressed_chunk_id|dropped|status|osm_chunk
--+-------------+-----------+----------+-------------------+-------+------+---------
(0 rows)


NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_1]:
show_chunks
-----------
(0 rows)


NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_1]:
time|device|temp|Color
----+------+----+-----
(0 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM _timescaledb_catalog.chunk
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|schema_name|table_name|compressed_chunk_id|dropped|status|osm_chunk
--+-------------+-----------+----------+-------------------+-------+------+---------
(0 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_2]:
show_chunks
-----------
(0 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_2]:
time|device|temp|Color
----+------+----+-----
(0 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM _timescaledb_catalog.chunk
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|schema_name|table_name|compressed_chunk_id|dropped|status|osm_chunk
--+-------------+-----------+----------+-------------------+-------+------+---------
(0 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM show_chunks('disttable')
NOTICE:  [db_dist_hypertable_3]:
show_chunks
-----------
(0 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM disttable
NOTICE:  [db_dist_hypertable_3]:
time|device|temp|Color
----+------+----+-----
(0 rows)


 remote_exec 
-------------
 
(1 row)

-- The hypertable view also shows no chunks and no data
SELECT * FROM timescaledb_information.hypertables
ORDER BY hypertable_schema, hypertable_name;
 hypertable_schema | hypertable_name |    owner    | num_dimensions | num_chunks | compression_enabled | is_distributed | replication_factor |                            data_nodes                            | tablespaces 
-------------------+-----------------+-------------+----------------+------------+---------------------+----------------+--------------------+------------------------------------------------------------------+-------------
 public            | disttable       | test_role_1 |              2 |          0 | f                   | t              |                  1 | {db_dist_hypertable_1,db_dist_hypertable_2,db_dist_hypertable_3} | 
 public            | underreplicated | test_role_1 |              1 |          0 | f                   | t              |                  4 | {db_dist_hypertable_1,db_dist_hypertable_2,db_dist_hypertable_3} | 
(2 rows)

-- Test underreplicated chunk warning
INSERT INTO underreplicated VALUES ('2017-01-01 06:01', 1, 1.1),
                                   ('2017-01-02 07:01', 2, 3.5);
WARNING:  insufficient number of data nodes
SELECT * FROM _timescaledb_catalog.chunk_data_node ORDER BY 1,2,3;
 chunk_id | node_chunk_id |      node_name       
----------+---------------+----------------------
       10 |             4 | db_dist_hypertable_1
       10 |             4 | db_dist_hypertable_2
       10 |             4 | db_dist_hypertable_3
(3 rows)

SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('underreplicated');
 chunk_id | hypertable_id |      schema_name      |       table_name       | relkind |                     slices                     
----------+---------------+-----------------------+------------------------+---------+------------------------------------------------
       10 |             2 | _timescaledb_internal | _dist_hyper_2_10_chunk | f       | {"time": [1482969600000000, 1483574400000000]}
(1 row)

-- Show chunk data node mappings
SELECT * FROM _timescaledb_catalog.chunk_data_node ORDER BY 1,2,3;
 chunk_id | node_chunk_id |      node_name       
----------+---------------+----------------------
       10 |             4 | db_dist_hypertable_1
       10 |             4 | db_dist_hypertable_2
       10 |             4 | db_dist_hypertable_3
(3 rows)

-- Show that chunks are created on remote data nodes and that all
-- data nodes/chunks have the same data due to replication
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('underreplicated');
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('underreplicated')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name            |relkind|slices                                        
--------+-------------+---------------------+----------------------+-------+----------------------------------------------
       4|            2|_timescaledb_internal|_dist_hyper_2_10_chunk|r      |{"time": [1482969600000000, 1483574400000000]}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('underreplicated')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name            |relkind|slices                                        
--------+-------------+---------------------+----------------------+-------+----------------------------------------------
       4|            2|_timescaledb_internal|_dist_hyper_2_10_chunk|r      |{"time": [1482969600000000, 1483574400000000]}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('underreplicated')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name            |relkind|slices                                        
--------+-------------+---------------------+----------------------+-------+----------------------------------------------
       4|            2|_timescaledb_internal|_dist_hyper_2_10_chunk|r      |{"time": [1482969600000000, 1483574400000000]}
(1 row)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM underreplicated;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2| 3.5
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2| 3.5
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2| 3.5
(2 rows)


 remote_exec 
-------------
 
(1 row)

-- Test updates
UPDATE underreplicated SET temp = 2.0 WHERE device = 2
RETURNING time, temp, device;
             time             | temp | device 
------------------------------+------+--------
 Mon Jan 02 07:01:00 2017 PST |    2 |      2
(1 row)

SELECT * FROM underreplicated;
             time             | device | temp 
------------------------------+--------+------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1
 Mon Jan 02 07:01:00 2017 PST |      2 |    2
(2 rows)

-- Show that all replica chunks are updated
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM underreplicated;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2|   2
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2|   2
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
Mon Jan 02 07:01:00 2017 PST|     2|   2
(2 rows)


 remote_exec 
-------------
 
(1 row)

DELETE FROM underreplicated WHERE device = 2
RETURNING *;
             time             | device | temp 
------------------------------+--------+------
 Mon Jan 02 07:01:00 2017 PST |      2 |    2
(1 row)

-- Ensure deletes across all data nodes
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM underreplicated;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_1]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_2]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Test hypertable creation fails on distributed error
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_3'], $$
CREATE TABLE remotetable(time timestamptz PRIMARY KEY, id int, cost float);
SELECT * FROM underreplicated;
$$);
NOTICE:  [db_dist_hypertable_3]: 
CREATE TABLE remotetable(time timestamptz PRIMARY KEY, id int, cost float)
NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM underreplicated
NOTICE:  [db_dist_hypertable_3]:
time                        |device|temp
----------------------------+------+----
Sun Jan 01 06:01:00 2017 PST|     1| 1.1
(1 row)


 remote_exec 
-------------
 
(1 row)

\set ON_ERROR_STOP 0
CREATE TABLE remotetable(time timestamptz PRIMARY KEY, device int CHECK (device > 0), color int, temp float);
SELECT * FROM create_hypertable('remotetable', 'time', replication_factor => 1);
ERROR:  [db_dist_hypertable_3]: relation "remotetable" already exists
-- Test distributed_hypertable creation fails with replication factor 0
CREATE TABLE remotetable2(time timestamptz PRIMARY KEY, device int CHECK (device > 0), color int, temp float);
SELECT * FROM create_distributed_hypertable('remotetable2', 'time', replication_factor => 0);
ERROR:  invalid replication factor
\set ON_ERROR_STOP 1
SELECT * FROM timescaledb_information.hypertables
ORDER BY hypertable_schema, hypertable_name;
 hypertable_schema | hypertable_name |    owner    | num_dimensions | num_chunks | compression_enabled | is_distributed | replication_factor |                            data_nodes                            | tablespaces 
-------------------+-----------------+-------------+----------------+------------+---------------------+----------------+--------------------+------------------------------------------------------------------+-------------
 public            | disttable       | test_role_1 |              2 |          0 | f                   | t              |                  1 | {db_dist_hypertable_1,db_dist_hypertable_2,db_dist_hypertable_3} | 
 public            | underreplicated | test_role_1 |              1 |          1 | f                   | t              |                  4 | {db_dist_hypertable_1,db_dist_hypertable_2,db_dist_hypertable_3} | 
(2 rows)

-- Test distributed hypertable creation with many parameters
\c :TEST_DBNAME :ROLE_SUPERUSER
--Ensure INSERTs use DataNodeDispatch.
SET timescaledb.enable_distributed_insert_with_copy=false;
CREATE SCHEMA "T3sTSch";
CREATE SCHEMA "Table\\Schema";
CREATE SCHEMA "single'schema";
GRANT ALL ON SCHEMA "T3sTSch" TO :ROLE_1;
GRANT ALL ON SCHEMA "Table\\Schema" TO :ROLE_1;
GRANT ALL ON SCHEMA "single'schema" TO :ROLE_1;
SET ROLE :ROLE_1;
CREATE TABLE "Table\\Schema"."Param_Table"("time Col %#^#@$#" timestamptz, __region text, reading float);
SELECT * FROM create_distributed_hypertable('"Table\\Schema"."Param_Table"', 'time Col %#^#@$#', partitioning_column => '__region',
associated_schema_name => 'T3sTSch', associated_table_prefix => 'test*pre_', chunk_time_interval => interval '1 week',
create_default_indexes => FALSE, if_not_exists => TRUE, replication_factor => 2,
data_nodes => ARRAY[:'DATA_NODE_2', :'DATA_NODE_3']);
NOTICE:  adding not-null constraint to column "time Col %#^#@$#"
 hypertable_id |  schema_name  | table_name  | created 
---------------+---------------+-------------+---------
             4 | Table\\Schema | Param_Table | t
(1 row)

-- Test detach and attach data node
SELECT * FROM detach_data_node(:'DATA_NODE_2', '"Table\\Schema"."Param_Table"', force => true, drop_remote_data => true);
WARNING:  insufficient number of data nodes for distributed hypertable "Param_Table"
NOTICE:  the number of partitions in dimension "__region" of hypertable "Param_Table" was decreased to 1
 detach_data_node 
------------------
                1
(1 row)

-- Test attach_data_node. First show dimensions and currently attached
-- servers.  The number of slices in the space dimension should equal
-- the number of servers since we didn't explicitly specify
-- number_partitions
SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'Param_Table'
ORDER BY 1, 2, 3;
 table_name  |   column_name    | num_slices 
-------------+------------------+------------
 Param_Table | __region         |          1
 Param_Table | time Col %#^#@$# |           
(2 rows)

SELECT h.table_name, hdn.node_name
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.hypertable_data_node hdn
WHERE h.id = hdn.hypertable_id
AND h.table_name = 'Param_Table'
ORDER BY 1, 2;
 table_name  |      node_name       
-------------+----------------------
 Param_Table | db_dist_hypertable_3
(1 row)

SELECT * FROM attach_data_node(:'DATA_NODE_1', '"Table\\Schema"."Param_Table"');
NOTICE:  the number of partitions in dimension "__region" was increased to 2
 hypertable_id | node_hypertable_id |      node_name       
---------------+--------------------+----------------------
             4 |                  3 | db_dist_hypertable_1
(1 row)

-- Show updated metadata after attach
SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'Param_Table'
ORDER BY 1, 2, 3;
 table_name  |   column_name    | num_slices 
-------------+------------------+------------
 Param_Table | __region         |          2
 Param_Table | time Col %#^#@$# |           
(2 rows)

SELECT h.table_name, hdn.node_name
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.hypertable_data_node hdn
WHERE h.id = hdn.hypertable_id
AND h.table_name = 'Param_Table'
ORDER BY 1, 2;
 table_name  |      node_name       
-------------+----------------------
 Param_Table | db_dist_hypertable_1
 Param_Table | db_dist_hypertable_3
(2 rows)

-- Attach another data node but do not auto-repartition, i.e.,
-- increase the number of slices.
SELECT * FROM attach_data_node(:'DATA_NODE_2', '"Table\\Schema"."Param_Table"', repartition => false);
WARNING:  insufficient number of partitions for dimension "__region"
 hypertable_id | node_hypertable_id |      node_name       
---------------+--------------------+----------------------
             4 |                  4 | db_dist_hypertable_2
(1 row)

-- Number of slices should not be increased
SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'Param_Table'
ORDER BY 1, 2, 3;
 table_name  |   column_name    | num_slices 
-------------+------------------+------------
 Param_Table | __region         |          2
 Param_Table | time Col %#^#@$# |           
(2 rows)

-- Manually increase the number of partitions
SELECT * FROM set_number_partitions('"Table\\Schema"."Param_Table"', 4);
 set_number_partitions 
-----------------------
 
(1 row)

-- Verify hypertables on all data nodes
SELECT * FROM _timescaledb_catalog.hypertable;
 id |  schema_name  |   table_name    | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema |  chunk_sizing_func_name  | chunk_target_size | compression_state | compressed_hypertable_id | replication_factor 
----+---------------+-----------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+--------------------
  1 | public        | disttable       | _timescaledb_internal  | _dist_hyper_1           |              2 | _timescaledb_internal    | calculate_chunk_interval |                 0 |                 0 |                          |                  1
  2 | public        | underreplicated | _timescaledb_internal  | _dist_hyper_2           |              1 | _timescaledb_internal    | calculate_chunk_interval |                 0 |                 0 |                          |                  4
  4 | Table\\Schema | Param_Table     | T3sTSch                | test*pre_               |              2 | _timescaledb_internal    | calculate_chunk_interval |                 0 |                 0 |                          |                  2
(3 rows)

SELECT * FROM _timescaledb_catalog.dimension;
 id | hypertable_id |   column_name    |       column_type        | aligned | num_slices | partitioning_func_schema | partitioning_func  | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func 
----+---------------+------------------+--------------------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------
  1 |             1 | time             | timestamp with time zone | t       |            |                          |                    |    604800000000 |                          |                         | 
  2 |             1 | device           | integer                  | f       |          3 | _timescaledb_internal    | get_partition_hash |                 |                          |                         | 
  3 |             2 | time             | timestamp with time zone | t       |            |                          |                    |    604800000000 |                          |                         | 
  5 |             4 | time Col %#^#@$# | timestamp with time zone | t       |            |                          |                    |    604800000000 |                          |                         | 
  6 |             4 | __region         | text                     | f       |          4 | _timescaledb_internal    | get_partition_hash |                 |                          |                         | 
(5 rows)

SELECT * FROM test.show_triggers('"Table\\Schema"."Param_Table"');
      Trigger      | Type |               Function               
-------------------+------+--------------------------------------
 ts_insert_blocker |    7 | _timescaledb_internal.insert_blocker
(1 row)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM _timescaledb_catalog.hypertable;
SELECT * FROM _timescaledb_catalog.dimension;
SELECT t.tgname, t.tgtype, t.tgfoid::regproc
FROM pg_trigger t, pg_class c WHERE c.relname = 'Param_Table' AND t.tgrelid = c.oid;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM _timescaledb_catalog.hypertable
NOTICE:  [db_dist_hypertable_1]:
id|schema_name  |table_name     |associated_schema_name|associated_table_prefix|num_dimensions|chunk_sizing_func_schema|chunk_sizing_func_name  |chunk_target_size|compression_state|compressed_hypertable_id|replication_factor
--+-------------+---------------+----------------------+-----------------------+--------------+------------------------+------------------------+-----------------+-----------------+------------------------+------------------
 1|public       |disttable      |_timescaledb_internal |_dist_hyper_1          |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 2|public       |underreplicated|_timescaledb_internal |_dist_hyper_2          |             1|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 3|Table\\Schema|Param_Table    |T3sTSch               |test*pre_              |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
(3 rows)


NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 4|            3|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 5|            3|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
(5 rows)


NOTICE:  [db_dist_hypertable_1]: 
SELECT t.tgname, t.tgtype, t.tgfoid::regproc
FROM pg_trigger t, pg_class c WHERE c.relname = 'Param_Table' AND t.tgrelid = c.oid
NOTICE:  [db_dist_hypertable_1]:
tgname           |tgtype|tgfoid                              
-----------------+------+------------------------------------
ts_insert_blocker|     7|_timescaledb_internal.insert_blocker
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM _timescaledb_catalog.hypertable
NOTICE:  [db_dist_hypertable_2]:
id|schema_name  |table_name     |associated_schema_name|associated_table_prefix|num_dimensions|chunk_sizing_func_schema|chunk_sizing_func_name  |chunk_target_size|compression_state|compressed_hypertable_id|replication_factor
--+-------------+---------------+----------------------+-----------------------+--------------+------------------------+------------------------+-----------------+-----------------+------------------------+------------------
 1|public       |disttable      |_timescaledb_internal |_dist_hyper_1          |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 2|public       |underreplicated|_timescaledb_internal |_dist_hyper_2          |             1|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 4|Table\\Schema|Param_Table    |T3sTSch               |test*pre_              |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
(3 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 6|            4|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 7|            4|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
(5 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT t.tgname, t.tgtype, t.tgfoid::regproc
FROM pg_trigger t, pg_class c WHERE c.relname = 'Param_Table' AND t.tgrelid = c.oid
NOTICE:  [db_dist_hypertable_2]:
tgname           |tgtype|tgfoid                              
-----------------+------+------------------------------------
ts_insert_blocker|     7|_timescaledb_internal.insert_blocker
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM _timescaledb_catalog.hypertable
NOTICE:  [db_dist_hypertable_3]:
id|schema_name  |table_name     |associated_schema_name|associated_table_prefix|num_dimensions|chunk_sizing_func_schema|chunk_sizing_func_name  |chunk_target_size|compression_state|compressed_hypertable_id|replication_factor
--+-------------+---------------+----------------------+-----------------------+--------------+------------------------+------------------------+-----------------+-----------------+------------------------+------------------
 1|public       |disttable      |_timescaledb_internal |_dist_hyper_1          |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 2|public       |underreplicated|_timescaledb_internal |_dist_hyper_2          |             1|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
 3|Table\\Schema|Param_Table    |T3sTSch               |test*pre_              |             2|_timescaledb_internal   |calculate_chunk_interval|                0|                0|                        |                -1
(3 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 4|            3|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 5|            3|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
(5 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT t.tgname, t.tgtype, t.tgfoid::regproc
FROM pg_trigger t, pg_class c WHERE c.relname = 'Param_Table' AND t.tgrelid = c.oid
NOTICE:  [db_dist_hypertable_3]:
tgname           |tgtype|tgfoid                              
-----------------+------+------------------------------------
ts_insert_blocker|     7|_timescaledb_internal.insert_blocker
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Verify that repartitioning works as expected on detach_data_node
SELECT * FROM detach_data_node(:'DATA_NODE_1', '"Table\\Schema"."Param_Table"', repartition => true);
NOTICE:  the number of partitions in dimension "__region" of hypertable "Param_Table" was decreased to 2
 detach_data_node 
------------------
                1
(1 row)

SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'Param_Table';
 table_name  |   column_name    | num_slices 
-------------+------------------+------------
 Param_Table | __region         |          2
 Param_Table | time Col %#^#@$# |           
(2 rows)

SELECT * FROM detach_data_node(:'DATA_NODE_2', '"Table\\Schema"."Param_Table"', force => true, repartition => false);
WARNING:  insufficient number of data nodes for distributed hypertable "Param_Table"
 detach_data_node 
------------------
                1
(1 row)

SELECT h.table_name, d.column_name, d.num_slices
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id
AND h.table_name = 'Param_Table';
 table_name  |   column_name    | num_slices 
-------------+------------------+------------
 Param_Table | __region         |          2
 Param_Table | time Col %#^#@$# |           
(2 rows)

-- Test multi-dimensional hypertable. The add_dimension() command
-- should be propagated to backends.
CREATE TABLE dimented_table (time timestamptz, column1 int, column2 timestamptz, column3 int);
SELECT * FROM create_distributed_hypertable('dimented_table', 'time', partitioning_column => 'column1', number_partitions  => 4, replication_factor => 1, data_nodes => ARRAY[:'DATA_NODE_1']);
WARNING:  only one data node was assigned to the hypertable
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name |   table_name   | created 
---------------+-------------+----------------+---------
             5 | public      | dimented_table | t
(1 row)

-- Create one chunk to block add_dimension
INSERT INTO dimented_table VALUES('2017-01-01 06:01', 1, '2017-01-01 08:01', 1);
CREATE VIEW dimented_table_slices AS
SELECT c.id AS chunk_id, c.hypertable_id, ds.dimension_id, cc.dimension_slice_id, c.schema_name AS
       chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id)
INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id)
INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id)
INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id)
WHERE h.table_name = 'dimented_table'
ORDER BY c.id, ds.dimension_id;
SELECT * FROM dimented_table_slices;
 chunk_id | hypertable_id | dimension_id | dimension_slice_id |     chunk_schema      |      chunk_table       |     range_start      |    range_end     
----------+---------------+--------------+--------------------+-----------------------+------------------------+----------------------+------------------
       11 |             5 |            7 |                  8 | _timescaledb_internal | _dist_hyper_5_11_chunk |     1482969600000000 | 1483574400000000
       11 |             5 |            8 |                  9 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 |        536870911
(2 rows)

-- add_dimension() with existing data
SELECT * FROM add_dimension('dimented_table', 'column2', chunk_time_interval => interval '1 week');
NOTICE:  adding not-null constraint to column "column2"
 dimension_id | schema_name |   table_name   | column_name | created 
--------------+-------------+----------------+-------------+---------
            9 | public      | dimented_table | column2     | t
(1 row)

SELECT * FROM dimented_table_slices;
 chunk_id | hypertable_id | dimension_id | dimension_slice_id |     chunk_schema      |      chunk_table       |     range_start      |      range_end      
----------+---------------+--------------+--------------------+-----------------------+------------------------+----------------------+---------------------
       11 |             5 |            7 |                  8 | _timescaledb_internal | _dist_hyper_5_11_chunk |     1482969600000000 |    1483574400000000
       11 |             5 |            8 |                  9 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 |           536870911
       11 |             5 |            9 |                 10 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 | 9223372036854775807
(3 rows)

SELECT * FROM add_dimension('dimented_table', 'column3', 4, partitioning_func => '_timescaledb_internal.get_partition_for_key');
 dimension_id | schema_name |   table_name   | column_name | created 
--------------+-------------+----------------+-------------+---------
           10 | public      | dimented_table | column3     | t
(1 row)

SELECT * FROM dimented_table_slices;
 chunk_id | hypertable_id | dimension_id | dimension_slice_id |     chunk_schema      |      chunk_table       |     range_start      |      range_end      
----------+---------------+--------------+--------------------+-----------------------+------------------------+----------------------+---------------------
       11 |             5 |            7 |                  8 | _timescaledb_internal | _dist_hyper_5_11_chunk |     1482969600000000 |    1483574400000000
       11 |             5 |            8 |                  9 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 |           536870911
       11 |             5 |            9 |                 10 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 | 9223372036854775807
       11 |             5 |           10 |                 11 | _timescaledb_internal | _dist_hyper_5_11_chunk | -9223372036854775808 | 9223372036854775807
(4 rows)

SELECT * FROM dimented_table ORDER BY time;
             time             | column1 |           column2            | column3 
------------------------------+---------+------------------------------+---------
 Sun Jan 01 06:01:00 2017 PST |       1 | Sun Jan 01 08:01:00 2017 PST |       1
(1 row)

SELECT * FROM _timescaledb_catalog.dimension;
 id | hypertable_id |   column_name    |       column_type        | aligned | num_slices | partitioning_func_schema |   partitioning_func   | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func 
----+---------------+------------------+--------------------------+---------+------------+--------------------------+-----------------------+-----------------+--------------------------+-------------------------+------------------
  1 |             1 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  2 |             1 | device           | integer                  | f       |          3 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  3 |             2 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  5 |             4 | time Col %#^#@$# | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  6 |             4 | __region         | text                     | f       |          2 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  7 |             5 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  8 |             5 | column1          | integer                  | f       |          4 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  9 |             5 | column2          | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
 10 |             5 | column3          | integer                  | f       |          4 | _timescaledb_internal    | get_partition_for_key |                 |                          |                         | 
(9 rows)

SELECT * FROM attach_data_node(:'DATA_NODE_2', 'dimented_table');
 hypertable_id | node_hypertable_id |      node_name       
---------------+--------------------+----------------------
             5 |                  5 | db_dist_hypertable_2
(1 row)

SELECT * FROM _timescaledb_catalog.dimension;
 id | hypertable_id |   column_name    |       column_type        | aligned | num_slices | partitioning_func_schema |   partitioning_func   | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func 
----+---------------+------------------+--------------------------+---------+------------+--------------------------+-----------------------+-----------------+--------------------------+-------------------------+------------------
  1 |             1 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  2 |             1 | device           | integer                  | f       |          3 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  3 |             2 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  5 |             4 | time Col %#^#@$# | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  6 |             4 | __region         | text                     | f       |          2 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  7 |             5 | time             | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
  8 |             5 | column1          | integer                  | f       |          4 | _timescaledb_internal    | get_partition_hash    |                 |                          |                         | 
  9 |             5 | column2          | timestamp with time zone | t       |            |                          |                       |    604800000000 |                          |                         | 
 10 |             5 | column3          | integer                  | f       |          4 | _timescaledb_internal    | get_partition_for_key |                 |                          |                         | 
(9 rows)

-- ensure data node has new dimensions
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM _timescaledb_catalog.dimension;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func    |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+---------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 4|            3|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 5|            3|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
 6|            4|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 7|            4|column1         |integer                 |f      |         4|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
 8|            4|column2         |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 9|            4|column3         |integer                 |f      |         4|_timescaledb_internal   |get_partition_for_key|               |                        |                       |                
(9 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func    |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+---------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 6|            4|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 7|            4|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
 8|            5|time            |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
 9|            5|column1         |integer                 |f      |         4|_timescaledb_internal   |get_partition_hash   |               |                        |                       |                
10|            5|column2         |timestamp with time zone|t      |          |                        |                     |   604800000000|                        |                       |                
11|            5|column3         |integer                 |f      |         4|_timescaledb_internal   |get_partition_for_key|               |                        |                       |                
(9 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM _timescaledb_catalog.dimension
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|column_name     |column_type             |aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+----------------+------------------------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
 1|            1|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 2|            1|device          |integer                 |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
 3|            2|time            |timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 4|            3|time Col %#^#@$#|timestamp with time zone|t      |          |                        |                  |   604800000000|                        |                       |                
 5|            3|__region        |text                    |f      |         4|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
(5 rows)


 remote_exec 
-------------
 
(1 row)

--test per-data node queries
-- Create some chunks through insertion
CREATE TABLE disttable_replicated(time timestamptz PRIMARY KEY, device int CHECK (device > 0), temp float, "Color" int);
SELECT * FROM create_hypertable('disttable_replicated', 'time', replication_factor => 2);
 hypertable_id | schema_name |      table_name      | created 
---------------+-------------+----------------------+---------
             6 | public      | disttable_replicated | t
(1 row)

INSERT INTO disttable_replicated VALUES
       ('2017-01-01 06:01', 1, 1.1, 1),
       ('2017-01-01 08:01', 1, 1.2, 2),
       ('2018-01-02 08:01', 2, 1.3, 3),
       ('2019-01-01 09:11', 3, 2.1, 4),
       ('2020-01-01 06:01', 5, 1.1, 10),
       ('2020-01-01 08:01', 6, 1.2, 11),
       ('2021-01-02 08:01', 7, 1.3, 12),
       ('2022-01-01 09:11', 8, 2.1, 13);
SELECT * FROM disttable_replicated;
             time             | device | temp | Color 
------------------------------+--------+------+-------
 Sun Jan 01 06:01:00 2017 PST |      1 |  1.1 |     1
 Sun Jan 01 08:01:00 2017 PST |      1 |  1.2 |     2
 Wed Jan 01 06:01:00 2020 PST |      5 |  1.1 |    10
 Wed Jan 01 08:01:00 2020 PST |      6 |  1.2 |    11
 Tue Jan 02 08:01:00 2018 PST |      2 |  1.3 |     3
 Sat Jan 02 08:01:00 2021 PST |      7 |  1.3 |    12
 Tue Jan 01 09:11:00 2019 PST |      3 |  2.1 |     4
 Sat Jan 01 09:11:00 2022 PST |      8 |  2.1 |    13
(8 rows)

EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated;
                                                                                    QUERY PLAN                                                                                     
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend) (actual rows=8 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   ->  Append (actual rows=8 loops=1)
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_1 (actual rows=4 loops=1)
               Output: disttable_replicated_1."time", disttable_replicated_1.device, disttable_replicated_1.temp, disttable_replicated_1."Color"
               Data node: db_dist_hypertable_1
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_12_chunk, _dist_hyper_6_15_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_2 (actual rows=2 loops=1)
               Output: disttable_replicated_2."time", disttable_replicated_2.device, disttable_replicated_2.temp, disttable_replicated_2."Color"
               Data node: db_dist_hypertable_2
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_13_chunk, _dist_hyper_6_16_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_3 (actual rows=2 loops=1)
               Output: disttable_replicated_3."time", disttable_replicated_3.device, disttable_replicated_3.temp, disttable_replicated_3."Color"
               Data node: db_dist_hypertable_3
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_14_chunk, _dist_hyper_6_17_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
(21 rows)

--guc disables the optimization
SET timescaledb.enable_per_data_node_queries = FALSE;
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated;
                                                                QUERY PLAN                                                                 
-------------------------------------------------------------------------------------------------------------------------------------------
 Append (actual rows=8 loops=1)
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_12_chunk (actual rows=2 loops=1)
         Output: _dist_hyper_6_12_chunk."time", _dist_hyper_6_12_chunk.device, _dist_hyper_6_12_chunk.temp, _dist_hyper_6_12_chunk."Color"
         Data node: db_dist_hypertable_1
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_12_chunk
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_13_chunk (actual rows=1 loops=1)
         Output: _dist_hyper_6_13_chunk."time", _dist_hyper_6_13_chunk.device, _dist_hyper_6_13_chunk.temp, _dist_hyper_6_13_chunk."Color"
         Data node: db_dist_hypertable_2
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_13_chunk
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_14_chunk (actual rows=1 loops=1)
         Output: _dist_hyper_6_14_chunk."time", _dist_hyper_6_14_chunk.device, _dist_hyper_6_14_chunk.temp, _dist_hyper_6_14_chunk."Color"
         Data node: db_dist_hypertable_3
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_14_chunk
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_15_chunk (actual rows=2 loops=1)
         Output: _dist_hyper_6_15_chunk."time", _dist_hyper_6_15_chunk.device, _dist_hyper_6_15_chunk.temp, _dist_hyper_6_15_chunk."Color"
         Data node: db_dist_hypertable_1
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_15_chunk
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_16_chunk (actual rows=1 loops=1)
         Output: _dist_hyper_6_16_chunk."time", _dist_hyper_6_16_chunk.device, _dist_hyper_6_16_chunk.temp, _dist_hyper_6_16_chunk."Color"
         Data node: db_dist_hypertable_2
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_16_chunk
   ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_17_chunk (actual rows=1 loops=1)
         Output: _dist_hyper_6_17_chunk."time", _dist_hyper_6_17_chunk.device, _dist_hyper_6_17_chunk.temp, _dist_hyper_6_17_chunk."Color"
         Data node: db_dist_hypertable_3
         Fetcher Type: Cursor
         Remote SQL: SELECT "time", device, temp, "Color" FROM _timescaledb_internal._dist_hyper_6_17_chunk
(31 rows)

SET timescaledb.enable_per_data_node_queries = TRUE;
--test WHERE clause
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated WHERE temp > 2.0;
                                                                                                      QUERY PLAN                                                                                                      
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend) (actual rows=2 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   ->  Append (actual rows=2 loops=1)
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_1 (actual rows=0 loops=1)
               Output: disttable_replicated_1."time", disttable_replicated_1.device, disttable_replicated_1.temp, disttable_replicated_1."Color"
               Data node: db_dist_hypertable_1
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_12_chunk, _dist_hyper_6_15_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND ((temp > 2::double precision))
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_2 (actual rows=0 loops=1)
               Output: disttable_replicated_2."time", disttable_replicated_2.device, disttable_replicated_2.temp, disttable_replicated_2."Color"
               Data node: db_dist_hypertable_2
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_13_chunk, _dist_hyper_6_16_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND ((temp > 2::double precision))
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_3 (actual rows=2 loops=1)
               Output: disttable_replicated_3."time", disttable_replicated_3.device, disttable_replicated_3.temp, disttable_replicated_3."Color"
               Data node: db_dist_hypertable_3
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_14_chunk, _dist_hyper_6_17_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND ((temp > 2::double precision))
(21 rows)

--test OR
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated WHERE temp > 2.0 or "Color" = 11;
                                                                                                                QUERY PLAN                                                                                                                
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend) (actual rows=3 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   ->  Append (actual rows=3 loops=1)
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_1 (actual rows=1 loops=1)
               Output: disttable_replicated_1."time", disttable_replicated_1.device, disttable_replicated_1.temp, disttable_replicated_1."Color"
               Data node: db_dist_hypertable_1
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_12_chunk, _dist_hyper_6_15_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND (((temp > 2::double precision) OR ("Color" = 11)))
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_2 (actual rows=0 loops=1)
               Output: disttable_replicated_2."time", disttable_replicated_2.device, disttable_replicated_2.temp, disttable_replicated_2."Color"
               Data node: db_dist_hypertable_2
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_13_chunk, _dist_hyper_6_16_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND (((temp > 2::double precision) OR ("Color" = 11)))
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_3 (actual rows=2 loops=1)
               Output: disttable_replicated_3."time", disttable_replicated_3.device, disttable_replicated_3.temp, disttable_replicated_3."Color"
               Data node: db_dist_hypertable_3
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_14_chunk, _dist_hyper_6_17_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) AND (((temp > 2::double precision) OR ("Color" = 11)))
(21 rows)

--test some chunks excluded
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE,  TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated WHERE time < '2018-01-01 09:11';
                                                                                                                     QUERY PLAN                                                                                                                     
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend) (actual rows=2 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   ->  Append (actual rows=2 loops=1)
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_1 (actual rows=2 loops=1)
               Output: disttable_replicated_1."time", disttable_replicated_1.device, disttable_replicated_1.temp, disttable_replicated_1."Color"
               Data node: db_dist_hypertable_1
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_12_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6]) AND (("time" < '2018-01-01 09:11:00-08'::timestamp with time zone))
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_2 (actual rows=0 loops=1)
               Output: disttable_replicated_2."time", disttable_replicated_2.device, disttable_replicated_2.temp, disttable_replicated_2."Color"
               Data node: db_dist_hypertable_2
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_13_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6]) AND (("time" < '2018-01-01 09:11:00-08'::timestamp with time zone))
(15 rows)

--test all chunks excluded
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE,  TIMING FALSE, SUMMARY FALSE)
SELECT * FROM disttable_replicated WHERE time < '2002-01-01 09:11';
                                                         QUERY PLAN                                                          
-----------------------------------------------------------------------------------------------------------------------------
 Result (actual rows=0 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   One-Time Filter: false
(3 rows)

--test cte
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE,  TIMING FALSE, SUMMARY FALSE)
WITH cte AS (
    SELECT * FROM disttable_replicated
)
SELECT * FROM cte;
                                                                                    QUERY PLAN                                                                                     
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend) (actual rows=8 loops=1)
   Output: disttable_replicated."time", disttable_replicated.device, disttable_replicated.temp, disttable_replicated."Color"
   ->  Append (actual rows=8 loops=1)
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_1 (actual rows=4 loops=1)
               Output: disttable_replicated_1."time", disttable_replicated_1.device, disttable_replicated_1.temp, disttable_replicated_1."Color"
               Data node: db_dist_hypertable_1
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_12_chunk, _dist_hyper_6_15_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_2 (actual rows=2 loops=1)
               Output: disttable_replicated_2."time", disttable_replicated_2.device, disttable_replicated_2.temp, disttable_replicated_2."Color"
               Data node: db_dist_hypertable_2
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_13_chunk, _dist_hyper_6_16_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
         ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_3 (actual rows=2 loops=1)
               Output: disttable_replicated_3."time", disttable_replicated_3.device, disttable_replicated_3.temp, disttable_replicated_3."Color"
               Data node: db_dist_hypertable_3
               Fetcher Type: COPY
               Chunks: _dist_hyper_6_14_chunk, _dist_hyper_6_17_chunk
               Remote SQL: SELECT "time", device, temp, "Color" FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8])
(21 rows)

--queries that involve updates/inserts are not optimized
EXPLAIN (VERBOSE, ANALYZE, COSTS FALSE,  TIMING FALSE, SUMMARY FALSE)
WITH devices AS (
     SELECT DISTINCT device FROM disttable_replicated ORDER BY device
)
UPDATE disttable_replicated SET device = 2 WHERE device = (SELECT device FROM devices LIMIT 1);
                                                                                                       QUERY PLAN                                                                                                       
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Update on public.disttable_replicated (actual rows=0 loops=1)
   Update on public.disttable_replicated disttable_replicated_1
   Foreign Update on _timescaledb_internal._dist_hyper_6_12_chunk disttable_replicated_2
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_12_chunk SET device = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_6_13_chunk disttable_replicated_3
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_13_chunk SET device = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_6_14_chunk disttable_replicated_4
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_14_chunk SET device = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_6_15_chunk disttable_replicated_5
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_15_chunk SET device = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_6_16_chunk disttable_replicated_6
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_16_chunk SET device = $2 WHERE ctid = $1
   Foreign Update on _timescaledb_internal._dist_hyper_6_17_chunk disttable_replicated_7
     Remote SQL: UPDATE _timescaledb_internal._dist_hyper_6_17_chunk SET device = $2 WHERE ctid = $1
   InitPlan 1 (returns $0)
     ->  Limit (actual rows=1 loops=1)
           Output: disttable_replicated_8.device
           ->  Unique (actual rows=1 loops=1)
                 Output: disttable_replicated_8.device
                 ->  Custom Scan (AsyncAppend) (actual rows=1 loops=1)
                       Output: disttable_replicated_8.device
                       ->  Merge Append (actual rows=1 loops=1)
                             Sort Key: disttable_replicated_9.device
                             ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_9 (actual rows=1 loops=1)
                                   Output: disttable_replicated_9.device
                                   Data node: db_dist_hypertable_1
                                   Fetcher Type: Cursor
                                   Chunks: _dist_hyper_6_12_chunk, _dist_hyper_6_15_chunk
                                   Remote SQL: SELECT DISTINCT device FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) ORDER BY device ASC NULLS LAST
                             ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_10 (actual rows=1 loops=1)
                                   Output: disttable_replicated_10.device
                                   Data node: db_dist_hypertable_2
                                   Fetcher Type: Cursor
                                   Chunks: _dist_hyper_6_13_chunk, _dist_hyper_6_16_chunk
                                   Remote SQL: SELECT DISTINCT device FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) ORDER BY device ASC NULLS LAST
                             ->  Custom Scan (DataNodeScan) on public.disttable_replicated disttable_replicated_11 (actual rows=1 loops=1)
                                   Output: disttable_replicated_11.device
                                   Data node: db_dist_hypertable_3
                                   Fetcher Type: Cursor
                                   Chunks: _dist_hyper_6_14_chunk, _dist_hyper_6_17_chunk
                                   Remote SQL: SELECT DISTINCT device FROM public.disttable_replicated WHERE _timescaledb_internal.chunks_in(public.disttable_replicated.*, ARRAY[6, 8]) ORDER BY device ASC NULLS LAST
   ->  Result (actual rows=2 loops=1)
         Output: 2, disttable_replicated.tableoid, disttable_replicated.ctid, (NULL::record)
         ->  Append (actual rows=2 loops=1)
               ->  Seq Scan on public.disttable_replicated disttable_replicated_1 (actual rows=0 loops=1)
                     Output: disttable_replicated_1.tableoid, disttable_replicated_1.ctid, NULL::record
                     Filter: (disttable_replicated_1.device = $0)
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_12_chunk disttable_replicated_2 (actual rows=2 loops=1)
                     Output: disttable_replicated_2.tableoid, disttable_replicated_2.ctid, disttable_replicated_2.*
                     Data node: db_dist_hypertable_1
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_12_chunk WHERE ((device = $1::integer))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_13_chunk disttable_replicated_3 (actual rows=0 loops=1)
                     Output: disttable_replicated_3.tableoid, disttable_replicated_3.ctid, disttable_replicated_3.*
                     Data node: db_dist_hypertable_2
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_13_chunk WHERE ((device = $1::integer))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_14_chunk disttable_replicated_4 (actual rows=0 loops=1)
                     Output: disttable_replicated_4.tableoid, disttable_replicated_4.ctid, disttable_replicated_4.*
                     Data node: db_dist_hypertable_3
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_14_chunk WHERE ((device = $1::integer))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_15_chunk disttable_replicated_5 (actual rows=0 loops=1)
                     Output: disttable_replicated_5.tableoid, disttable_replicated_5.ctid, disttable_replicated_5.*
                     Data node: db_dist_hypertable_1
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_15_chunk WHERE ((device = $1::integer))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_16_chunk disttable_replicated_6 (actual rows=0 loops=1)
                     Output: disttable_replicated_6.tableoid, disttable_replicated_6.ctid, disttable_replicated_6.*
                     Data node: db_dist_hypertable_2
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_16_chunk WHERE ((device = $1::integer))
               ->  Foreign Scan on _timescaledb_internal._dist_hyper_6_17_chunk disttable_replicated_7 (actual rows=0 loops=1)
                     Output: disttable_replicated_7.tableoid, disttable_replicated_7.ctid, disttable_replicated_7.*
                     Data node: db_dist_hypertable_3
                     Fetcher Type: Cursor
                     Remote SQL: SELECT "time", device, temp, "Color", ctid FROM _timescaledb_internal._dist_hyper_6_17_chunk WHERE ((device = $1::integer))
(77 rows)

-- Test inserts with smaller batch size and more tuples to reach full
-- batch
SET timescaledb.max_insert_batch_size=4;
CREATE TABLE twodim (time timestamptz DEFAULT '2019-02-10 10:11', "Color" int DEFAULT 11 CHECK ("Color" > 0), temp float DEFAULT 22.1);
-- Create a replicated table to ensure we handle that case correctly
-- with batching
SELECT * FROM create_hypertable('twodim', 'time', 'Color', 3, replication_factor => 2, data_nodes => ARRAY[:'DATA_NODE_1',:'DATA_NODE_2',:'DATA_NODE_3']);
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
             7 | public      | twodim     | t
(1 row)

SELECT * FROM twodim
ORDER BY time;
 time | Color | temp 
------+-------+------
(0 rows)

-- INSERT enough data to stretch across multiple batches per
-- data node. Also return a system column. Although we write tuples to
-- multiple data nodes, the returned tuple should only be the ones in the
-- original insert statement (without the replica tuples).
WITH result AS (
     INSERT INTO twodim VALUES
       ('2017-02-01 06:01', 1, 1.1),
       ('2017-02-01 08:01', 1, 1.2),
       ('2018-02-02 08:01', 2, 1.3),
       ('2019-02-01 09:11', 3, 2.1),
       ('2019-02-02 09:11', 3, 2.1),
       ('2019-02-02 10:01', 5, 1.2),
       ('2019-02-03 11:11', 6, 3.5),
       ('2019-02-04 08:21', 4, 6.6),
       ('2019-02-04 10:11', 7, 7.4),
       ('2019-02-04 12:11', 8, 2.1),
       ('2019-02-05 13:31', 8, 6.3),
       ('2019-02-06 02:11', 5, 1.8),
       ('2019-02-06 01:13', 7, 7.9),
       ('2019-02-06 19:24', 9, 5.9),
       ('2019-02-07 18:44', 5, 9.7),
       ('2019-02-07 20:24', 6, NULL),
       ('2019-02-07 09:33', 7, 9.5),
       ('2019-02-08 08:54', 1, 7.3),
       ('2019-02-08 18:14', 4, 8.2),
       ('2019-02-09 19:23', 8, 9.1)
     RETURNING tableoid = 'twodim'::regclass AS is_tableoid, time, temp, "Color"
) SELECT * FROM result ORDER BY time;
 is_tableoid |             time             | temp | Color 
-------------+------------------------------+------+-------
 t           | Wed Feb 01 06:01:00 2017 PST |  1.1 |     1
 t           | Wed Feb 01 08:01:00 2017 PST |  1.2 |     1
 t           | Fri Feb 02 08:01:00 2018 PST |  1.3 |     2
 t           | Fri Feb 01 09:11:00 2019 PST |  2.1 |     3
 t           | Sat Feb 02 09:11:00 2019 PST |  2.1 |     3
 t           | Sat Feb 02 10:01:00 2019 PST |  1.2 |     5
 t           | Sun Feb 03 11:11:00 2019 PST |  3.5 |     6
 t           | Mon Feb 04 08:21:00 2019 PST |  6.6 |     4
 t           | Mon Feb 04 10:11:00 2019 PST |  7.4 |     7
 t           | Mon Feb 04 12:11:00 2019 PST |  2.1 |     8
 t           | Tue Feb 05 13:31:00 2019 PST |  6.3 |     8
 t           | Wed Feb 06 01:13:00 2019 PST |  7.9 |     7
 t           | Wed Feb 06 02:11:00 2019 PST |  1.8 |     5
 t           | Wed Feb 06 19:24:00 2019 PST |  5.9 |     9
 t           | Thu Feb 07 09:33:00 2019 PST |  9.5 |     7
 t           | Thu Feb 07 18:44:00 2019 PST |  9.7 |     5
 t           | Thu Feb 07 20:24:00 2019 PST |      |     6
 t           | Fri Feb 08 08:54:00 2019 PST |  7.3 |     1
 t           | Fri Feb 08 18:14:00 2019 PST |  8.2 |     4
 t           | Sat Feb 09 19:23:00 2019 PST |  9.1 |     8
(20 rows)

-- Test insert with default values and a batch size of 1.
SET timescaledb.max_insert_batch_size=1;
EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
INSERT INTO twodim DEFAULT VALUES;
                                                        QUERY PLAN                                                        
--------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable public.twodim
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.twodim
         ->  Custom Scan (DataNodeDispatch)
               Output: 'Sun Feb 10 10:11:00 2019 PST'::timestamp with time zone, 11, '22.1'::double precision
               Batch size: 1
               Remote SQL: INSERT INTO public.twodim("time", "Color", temp) VALUES ($1, $2, $3)
               ->  Custom Scan (ChunkDispatch)
                     Output: 'Sun Feb 10 10:11:00 2019 PST'::timestamp with time zone, 11, '22.1'::double precision
                     ->  Result
                           Output: 'Sun Feb 10 10:11:00 2019 PST'::timestamp with time zone, 11, '22.1'::double precision
(12 rows)

INSERT INTO twodim DEFAULT VALUES;
-- Reset the batch size
SET timescaledb.max_insert_batch_size=4;
-- Constraint violation error check
--
-- Execute and filter mentioned data node name in the error message.
\set ON_ERROR_STOP 0
SELECT test.execute_sql_and_filter_data_node_name_on_error($$ INSERT INTO twodim VALUES ('2019-02-10 17:54', 0, 10.2) $$, :'TEST_DBNAME');
ERROR:  [db_dist_hypertable_x]: new row for relation "_dist_hyper_7_23_chunk" violates check constraint "twodim_Color_check"
\set ON_ERROR_STOP 1
-- Disable batching, reverting to FDW tuple-by-tuple inserts.
-- First EXPLAIN with batching turned on.
EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
INSERT INTO twodim VALUES
       ('2019-02-10 16:23', 5, 7.1),
       ('2019-02-10 17:11', 7, 3.2);
                                                      QUERY PLAN                                                      
----------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable public.twodim
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.twodim
         ->  Custom Scan (DataNodeDispatch)
               Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3
               Batch size: 4
               Remote SQL: INSERT INTO public.twodim("time", "Color", temp) VALUES ($1, $2, $3), ..., ($10, $11, $12)
               ->  Custom Scan (ChunkDispatch)
                     Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3
                     ->  Values Scan on "*VALUES*"
                           Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3
(12 rows)

SET timescaledb.max_insert_batch_size=0;
-- Compare without batching
EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
INSERT INTO twodim VALUES
       ('2019-02-10 16:23', 5, 7.1),
       ('2019-02-10 17:11', 7, 3.2);
                                       QUERY PLAN                                       
----------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable public.twodim
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   Remote SQL: INSERT INTO public.twodim("time", "Color", temp) VALUES ($1, $2, $3)
   ->  Insert on public.twodim
         ->  Custom Scan (ChunkDispatch)
               Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3
               ->  Values Scan on "*VALUES*"
                     Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3
(9 rows)

-- Insert without batching
INSERT INTO twodim VALUES
       ('2019-02-10 16:23', 5, 7.1),
       ('2019-02-10 17:11', 7, 3.2);
-- Check that datanodes use ChunkAppend plans with chunks_in function in the
-- "Remote SQL" when multiple dimensions are involved.
SET timescaledb.enable_remote_explain = ON;
EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
SELECT * FROM twodim
ORDER BY time;
                                                                                     QUERY PLAN                                                                                     
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: twodim."time", twodim."Color", twodim.temp
   ->  Merge Append
         Sort Key: twodim_1."time"
         ->  Custom Scan (DataNodeScan) on public.twodim twodim_1
               Output: twodim_1."time", twodim_1."Color", twodim_1.temp
               Data node: db_dist_hypertable_1
               Chunks: _dist_hyper_7_18_chunk, _dist_hyper_7_22_chunk, _dist_hyper_7_25_chunk
               Remote SQL: SELECT "time", "Color", temp FROM public.twodim WHERE _timescaledb_internal.chunks_in(public.twodim.*, ARRAY[10, 12, 14]) ORDER BY "time" ASC NULLS LAST
               Remote EXPLAIN: 
                 Custom Scan (ChunkAppend) on public.twodim
                   Output: twodim."time", twodim."Color", twodim.temp
                   Order: twodim."time"
                   Startup Exclusion: false
                   Runtime Exclusion: false
                   ->  Index Scan Backward using _dist_hyper_7_18_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_18_chunk
                         Output: _dist_hyper_7_18_chunk."time", _dist_hyper_7_18_chunk."Color", _dist_hyper_7_18_chunk.temp
                   ->  Index Scan Backward using _dist_hyper_7_22_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_22_chunk
                         Output: _dist_hyper_7_22_chunk."time", _dist_hyper_7_22_chunk."Color", _dist_hyper_7_22_chunk.temp
                   ->  Index Scan Backward using _dist_hyper_7_25_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_25_chunk
                         Output: _dist_hyper_7_25_chunk."time", _dist_hyper_7_25_chunk."Color", _dist_hyper_7_25_chunk.temp
 
         ->  Custom Scan (DataNodeScan) on public.twodim twodim_2
               Output: twodim_2."time", twodim_2."Color", twodim_2.temp
               Data node: db_dist_hypertable_2
               Chunks: _dist_hyper_7_19_chunk, _dist_hyper_7_21_chunk, _dist_hyper_7_24_chunk
               Remote SQL: SELECT "time", "Color", temp FROM public.twodim WHERE _timescaledb_internal.chunks_in(public.twodim.*, ARRAY[10, 11, 13]) ORDER BY "time" ASC NULLS LAST
               Remote EXPLAIN: 
                 Custom Scan (ChunkAppend) on public.twodim
                   Output: twodim."time", twodim."Color", twodim.temp
                   Order: twodim."time"
                   Startup Exclusion: false
                   Runtime Exclusion: false
                   ->  Index Scan Backward using _dist_hyper_7_19_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_19_chunk
                         Output: _dist_hyper_7_19_chunk."time", _dist_hyper_7_19_chunk."Color", _dist_hyper_7_19_chunk.temp
                   ->  Index Scan Backward using _dist_hyper_7_21_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_21_chunk
                         Output: _dist_hyper_7_21_chunk."time", _dist_hyper_7_21_chunk."Color", _dist_hyper_7_21_chunk.temp
                   ->  Index Scan Backward using _dist_hyper_7_24_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_24_chunk
                         Output: _dist_hyper_7_24_chunk."time", _dist_hyper_7_24_chunk."Color", _dist_hyper_7_24_chunk.temp
 
         ->  Custom Scan (DataNodeScan) on public.twodim twodim_3
               Output: twodim_3."time", twodim_3."Color", twodim_3.temp
               Data node: db_dist_hypertable_3
               Chunks: _dist_hyper_7_20_chunk, _dist_hyper_7_23_chunk
               Remote SQL: SELECT "time", "Color", temp FROM public.twodim WHERE _timescaledb_internal.chunks_in(public.twodim.*, ARRAY[10, 12]) ORDER BY "time" ASC NULLS LAST
               Remote EXPLAIN: 
                 Custom Scan (ChunkAppend) on public.twodim
                   Output: twodim."time", twodim."Color", twodim.temp
                   Order: twodim."time"
                   Startup Exclusion: false
                   Runtime Exclusion: false
                   ->  Index Scan Backward using _dist_hyper_7_20_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_20_chunk
                         Output: _dist_hyper_7_20_chunk."time", _dist_hyper_7_20_chunk."Color", _dist_hyper_7_20_chunk.temp
                   ->  Index Scan Backward using _dist_hyper_7_23_chunk_twodim_time_idx on _timescaledb_internal._dist_hyper_7_23_chunk
                         Output: _dist_hyper_7_23_chunk."time", _dist_hyper_7_23_chunk."Color", _dist_hyper_7_23_chunk.temp
 
(56 rows)

SET timescaledb.enable_remote_explain = OFF;
-- Check results
SELECT * FROM twodim
ORDER BY time;
             time             | Color | temp 
------------------------------+-------+------
 Wed Feb 01 06:01:00 2017 PST |     1 |  1.1
 Wed Feb 01 08:01:00 2017 PST |     1 |  1.2
 Fri Feb 02 08:01:00 2018 PST |     2 |  1.3
 Fri Feb 01 09:11:00 2019 PST |     3 |  2.1
 Sat Feb 02 09:11:00 2019 PST |     3 |  2.1
 Sat Feb 02 10:01:00 2019 PST |     5 |  1.2
 Sun Feb 03 11:11:00 2019 PST |     6 |  3.5
 Mon Feb 04 08:21:00 2019 PST |     4 |  6.6
 Mon Feb 04 10:11:00 2019 PST |     7 |  7.4
 Mon Feb 04 12:11:00 2019 PST |     8 |  2.1
 Tue Feb 05 13:31:00 2019 PST |     8 |  6.3
 Wed Feb 06 01:13:00 2019 PST |     7 |  7.9
 Wed Feb 06 02:11:00 2019 PST |     5 |  1.8
 Wed Feb 06 19:24:00 2019 PST |     9 |  5.9
 Thu Feb 07 09:33:00 2019 PST |     7 |  9.5
 Thu Feb 07 18:44:00 2019 PST |     5 |  9.7
 Thu Feb 07 20:24:00 2019 PST |     6 |     
 Fri Feb 08 08:54:00 2019 PST |     1 |  7.3
 Fri Feb 08 18:14:00 2019 PST |     4 |  8.2
 Sat Feb 09 19:23:00 2019 PST |     8 |  9.1
 Sun Feb 10 10:11:00 2019 PST |    11 | 22.1
 Sun Feb 10 16:23:00 2019 PST |     5 |  7.1
 Sun Feb 10 17:11:00 2019 PST |     7 |  3.2
(23 rows)

SELECT count(*) FROM twodim;
 count 
-------
    23
(1 row)

-- Show distribution across data nodes
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT * FROM twodim
ORDER BY time;
SELECT count(*) FROM twodim;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT * FROM twodim
ORDER BY time
NOTICE:  [db_dist_hypertable_1]:
time                        |Color|temp
----------------------------+-----+----
Wed Feb 01 06:01:00 2017 PST|    1| 1.1
Wed Feb 01 08:01:00 2017 PST|    1| 1.2
Fri Feb 01 09:11:00 2019 PST|    3| 2.1
Sat Feb 02 09:11:00 2019 PST|    3| 2.1
Sun Feb 03 11:11:00 2019 PST|    6| 3.5
Mon Feb 04 12:11:00 2019 PST|    8| 2.1
Tue Feb 05 13:31:00 2019 PST|    8| 6.3
Wed Feb 06 19:24:00 2019 PST|    9| 5.9
Thu Feb 07 20:24:00 2019 PST|    6|    
Fri Feb 08 08:54:00 2019 PST|    1| 7.3
Sat Feb 09 19:23:00 2019 PST|    8| 9.1
Sun Feb 10 10:11:00 2019 PST|   11|22.1
(12 rows)


NOTICE:  [db_dist_hypertable_1]: 
SELECT count(*) FROM twodim
NOTICE:  [db_dist_hypertable_1]:
count
-----
   12
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT * FROM twodim
ORDER BY time
NOTICE:  [db_dist_hypertable_2]:
time                        |Color|temp
----------------------------+-----+----
Wed Feb 01 06:01:00 2017 PST|    1| 1.1
Wed Feb 01 08:01:00 2017 PST|    1| 1.2
Fri Feb 02 08:01:00 2018 PST|    2| 1.3
Sat Feb 02 10:01:00 2019 PST|    5| 1.2
Sun Feb 03 11:11:00 2019 PST|    6| 3.5
Mon Feb 04 08:21:00 2019 PST|    4| 6.6
Mon Feb 04 10:11:00 2019 PST|    7| 7.4
Mon Feb 04 12:11:00 2019 PST|    8| 2.1
Tue Feb 05 13:31:00 2019 PST|    8| 6.3
Wed Feb 06 01:13:00 2019 PST|    7| 7.9
Wed Feb 06 02:11:00 2019 PST|    5| 1.8
Thu Feb 07 09:33:00 2019 PST|    7| 9.5
Thu Feb 07 18:44:00 2019 PST|    5| 9.7
Thu Feb 07 20:24:00 2019 PST|    6|    
Fri Feb 08 08:54:00 2019 PST|    1| 7.3
Fri Feb 08 18:14:00 2019 PST|    4| 8.2
Sat Feb 09 19:23:00 2019 PST|    8| 9.1
Sun Feb 10 16:23:00 2019 PST|    5| 7.1
Sun Feb 10 17:11:00 2019 PST|    7| 3.2
(19 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT count(*) FROM twodim
NOTICE:  [db_dist_hypertable_2]:
count
-----
   19
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
SELECT * FROM twodim
ORDER BY time
NOTICE:  [db_dist_hypertable_3]:
time                        |Color|temp
----------------------------+-----+----
Fri Feb 02 08:01:00 2018 PST|    2| 1.3
Fri Feb 01 09:11:00 2019 PST|    3| 2.1
Sat Feb 02 09:11:00 2019 PST|    3| 2.1
Sat Feb 02 10:01:00 2019 PST|    5| 1.2
Mon Feb 04 08:21:00 2019 PST|    4| 6.6
Mon Feb 04 10:11:00 2019 PST|    7| 7.4
Wed Feb 06 01:13:00 2019 PST|    7| 7.9
Wed Feb 06 02:11:00 2019 PST|    5| 1.8
Wed Feb 06 19:24:00 2019 PST|    9| 5.9
Thu Feb 07 09:33:00 2019 PST|    7| 9.5
Thu Feb 07 18:44:00 2019 PST|    5| 9.7
Fri Feb 08 18:14:00 2019 PST|    4| 8.2
Sun Feb 10 10:11:00 2019 PST|   11|22.1
Sun Feb 10 16:23:00 2019 PST|    5| 7.1
Sun Feb 10 17:11:00 2019 PST|    7| 3.2
(15 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT count(*) FROM twodim
NOTICE:  [db_dist_hypertable_3]:
count
-----
   15
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Distributed table with custom type that has no binary output
CREATE TABLE disttable_with_ct(time timestamptz, txn_id rxid, val float, info text);
SELECT * FROM create_hypertable('disttable_with_ct', 'time', replication_factor => 2);
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name |    table_name     | created 
---------------+-------------+-------------------+---------
             8 | public      | disttable_with_ct | t
(1 row)

-- Insert data with custom type
INSERT INTO disttable_with_ct VALUES
    ('2019-01-01 01:01', 'ts-1-10-20-30', 1.1, 'a'),
    ('2019-01-01 01:02', 'ts-1-11-20-30', 2.0, repeat('abc', 1000000)); -- TOAST
-- Test queries on distributed table with custom type
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct;
             time             |    txn_id     | val |      substring       
------------------------------+---------------+-----+----------------------
 Tue Jan 01 01:01:00 2019 PST | ts-1-10-20-30 | 1.1 | a
 Tue Jan 01 01:02:00 2019 PST | ts-1-11-20-30 |   2 | abcabcabcabcabcabcab
(2 rows)

SET timescaledb.enable_connection_binary_data=false;
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct;
             time             |    txn_id     | val |      substring       
------------------------------+---------------+-----+----------------------
 Tue Jan 01 01:01:00 2019 PST | ts-1-10-20-30 | 1.1 | a
 Tue Jan 01 01:02:00 2019 PST | ts-1-11-20-30 |   2 | abcabcabcabcabcabcab
(2 rows)

-- Test DELETE with replication
DELETE FROM disttable_with_ct WHERE info = 'a';
-- Check if row is gone
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct;
             time             |    txn_id     | val |      substring       
------------------------------+---------------+-----+----------------------
 Tue Jan 01 01:02:00 2019 PST | ts-1-11-20-30 |   2 | abcabcabcabcabcabcab
(1 row)

-- Connect to data nodes to see if data is gone
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct;
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct
NOTICE:  [db_dist_hypertable_1]:
time                        |txn_id       |val|substring           
----------------------------+-------------+---+--------------------
Tue Jan 01 01:02:00 2019 PST|ts-1-11-20-30|  2|abcabcabcabcabcabcab
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct
NOTICE:  [db_dist_hypertable_2]:
time|txn_id|val|substring
----+------+---+---------
(0 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT time, txn_id, val, substring(info for 20) FROM disttable_with_ct
NOTICE:  [db_dist_hypertable_3]:
time                        |txn_id       |val|substring           
----------------------------+-------------+---+--------------------
Tue Jan 01 01:02:00 2019 PST|ts-1-11-20-30|  2|abcabcabcabcabcabcab
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Test single quote in names
SET SCHEMA 'single''schema';
CREATE TABLE "disttable'quote"(time timestamptz, "device'quote" int, val float, info text);
SELECT public.create_distributed_hypertable(
    'disttable''quote', 'time', 'device''quote', data_nodes => ARRAY[:'DATA_NODE_1']
);
WARNING:  only one data node was assigned to the hypertable
NOTICE:  adding not-null constraint to column "time"
    create_distributed_hypertable    
-------------------------------------
 (9,single'schema,disttable'quote,t)
(1 row)

SET SCHEMA 'public';
CREATE TABLE disttable_drop_chunks(time timestamptz, device int CHECK (device > 0), color int, PRIMARY KEY (time,device));
SELECT * FROM create_distributed_hypertable('disttable_drop_chunks', 'time', 'device', number_partitions => 3, replication_factor => 2);
 hypertable_id | schema_name |      table_name       | created 
---------------+-------------+-----------------------+---------
            10 | public      | disttable_drop_chunks | t
(1 row)

INSERT INTO disttable_drop_chunks VALUES
       ('2017-01-01 06:01', 1, 1.1),
       ('2017-01-01 09:11', 3, 2.1),
       ('2017-01-01 08:01', 1, 1.2),
       ('2017-01-02 08:01', 2, 1.3),
       ('2018-07-02 08:01', 87, 1.6),
       ('2018-07-01 06:01', 13, 1.4),
       ('2018-07-01 09:11', 90, 2.7),
       ('2018-07-01 08:01', 29, 1.5);
-- Show chunks on access node
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks');
 chunk_id | hypertable_id |      schema_name      |       table_name        | relkind |                                           slices                                            
----------+---------------+-----------------------+-------------------------+---------+---------------------------------------------------------------------------------------------
       27 |            10 | _timescaledb_internal | _dist_hyper_10_27_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
       28 |            10 | _timescaledb_internal | _dist_hyper_10_28_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
       29 |            10 | _timescaledb_internal | _dist_hyper_10_29_chunk | f       | {"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}
       30 |            10 | _timescaledb_internal | _dist_hyper_10_30_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
       31 |            10 | _timescaledb_internal | _dist_hyper_10_31_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}
       32 |            10 | _timescaledb_internal | _dist_hyper_10_32_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(6 rows)

-- Show chunks on data nodes
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks');
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      16|            9|_timescaledb_internal|_dist_hyper_10_27_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
      17|            9|_timescaledb_internal|_dist_hyper_10_28_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
      18|            9|_timescaledb_internal|_dist_hyper_10_30_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
      19|            9|_timescaledb_internal|_dist_hyper_10_32_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(4 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      15|            9|_timescaledb_internal|_dist_hyper_10_27_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [-9223372036854775808, 715827882]}
      16|            9|_timescaledb_internal|_dist_hyper_10_29_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}          
      17|            9|_timescaledb_internal|_dist_hyper_10_30_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
      18|            9|_timescaledb_internal|_dist_hyper_10_31_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}          
(4 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      15|            7|_timescaledb_internal|_dist_hyper_10_28_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [1431655764, 9223372036854775807]}
      16|            7|_timescaledb_internal|_dist_hyper_10_29_chunk|r      |{"time": [1482969600000000, 1483574400000000], "device": [715827882, 1431655764]}          
      17|            7|_timescaledb_internal|_dist_hyper_10_31_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}          
      18|            7|_timescaledb_internal|_dist_hyper_10_32_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(4 rows)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM drop_chunks('disttable_drop_chunks', older_than => '2018-01-01'::timestamptz);
                  drop_chunks                  
-----------------------------------------------
 _timescaledb_internal._dist_hyper_10_27_chunk
 _timescaledb_internal._dist_hyper_10_28_chunk
 _timescaledb_internal._dist_hyper_10_29_chunk
(3 rows)

SELECT * FROM disttable_drop_chunks;
             time             | device | color 
------------------------------+--------+-------
 Mon Jul 02 08:01:00 2018 PDT |     87 |     2
 Sun Jul 01 06:01:00 2018 PDT |     13 |     1
 Sun Jul 01 09:11:00 2018 PDT |     90 |     3
 Sun Jul 01 08:01:00 2018 PDT |     29 |     2
(4 rows)

SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks');
 chunk_id | hypertable_id |      schema_name      |       table_name        | relkind |                                           slices                                            
----------+---------------+-----------------------+-------------------------+---------+---------------------------------------------------------------------------------------------
       30 |            10 | _timescaledb_internal | _dist_hyper_10_30_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
       31 |            10 | _timescaledb_internal | _dist_hyper_10_31_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}
       32 |            10 | _timescaledb_internal | _dist_hyper_10_32_chunk | f       | {"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(3 rows)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks');
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      18|            9|_timescaledb_internal|_dist_hyper_10_30_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
      19|            9|_timescaledb_internal|_dist_hyper_10_32_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      17|            9|_timescaledb_internal|_dist_hyper_10_30_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [-9223372036854775808, 715827882]}
      18|            9|_timescaledb_internal|_dist_hyper_10_31_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}          
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
FROM show_chunks('disttable_drop_chunks')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      17|            7|_timescaledb_internal|_dist_hyper_10_31_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [715827882, 1431655764]}          
      18|            7|_timescaledb_internal|_dist_hyper_10_32_chunk|r      |{"time": [1530144000000000, 1530748800000000], "device": [1431655764, 9223372036854775807]}
(2 rows)


 remote_exec 
-------------
 
(1 row)

-- test passing newer_than as interval
SELECT * FROM drop_chunks('disttable_drop_chunks', newer_than => interval '10 years');
                  drop_chunks                  
-----------------------------------------------
 _timescaledb_internal._dist_hyper_10_30_chunk
 _timescaledb_internal._dist_hyper_10_31_chunk
 _timescaledb_internal._dist_hyper_10_32_chunk
(3 rows)

SELECT * FROM disttable_drop_chunks;
 time | device | color 
------+--------+-------
(0 rows)

CREATE TABLE "weird nAme\\#^."(time bigint, device int CHECK (device > 0), color int, PRIMARY KEY (time,device));
SELECT * FROM create_distributed_hypertable('"weird nAme\\#^."', 'time', 'device', 3, chunk_time_interval => 100, replication_factor => 2);
 hypertable_id | schema_name |   table_name    | created 
---------------+-------------+-----------------+---------
            11 | public      | weird nAme\\#^. | t
(1 row)

INSERT INTO "weird nAme\\#^." VALUES
       (300, 1, 1.1),
       (400, 3, 2.1),
       (350, 1, 1.2);
SELECT * FROM "weird nAme\\#^.";
 time | device | color 
------+--------+-------
  300 |      1 |     1
  350 |      1 |     1
  400 |      3 |     2
(3 rows)

-- drop chunks using bigint as time
SELECT * FROM drop_chunks('"weird nAme\\#^."', older_than => 1000);
                  drop_chunks                  
-----------------------------------------------
 _timescaledb_internal._dist_hyper_11_33_chunk
 _timescaledb_internal._dist_hyper_11_34_chunk
(2 rows)

SELECT * FROM "weird nAme\\#^.";
 time | device | color 
------+--------+-------
(0 rows)

-----------------------------------------------------------------------------------------
-- Test that settings on hypertables are distributed to data nodes
-----------------------------------------------------------------------------------------
DROP TABLE disttable CASCADE;
CREATE TABLE disttable (time bigint, device int, temp float);
SELECT create_distributed_hypertable('disttable', 'time', chunk_time_interval => 1000000::bigint);
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (12,public,disttable,t)
(1 row)

-- Show the dimension configuration on data nodes
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable';
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func|interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+-----------------+---------------+------------------------+-----------------------+----------------
20|           11|time       |bigint     |t      |          |                        |                 |        1000000|                        |                       |                
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func|interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+-----------------+---------------+------------------------+-----------------------+----------------
20|           11|time       |bigint     |t      |          |                        |                 |        1000000|                        |                       |                
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func|interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+-----------------+---------------+------------------------+-----------------------+----------------
14|            9|time       |bigint     |t      |          |                        |                 |        1000000|                        |                       |                
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Test adding a space dimension. Should apply to data nodes as
-- well. We're setting num_partitions lower than the number of servers
-- and expect a warning.
SELECT * FROM add_dimension('disttable', 'device', 1, partitioning_func => '_timescaledb_internal.get_partition_hash');
WARNING:  insufficient number of partitions for dimension "device"
 dimension_id | schema_name | table_name | column_name | created 
--------------+-------------+------------+-------------+---------
           22 | public      | disttable  | device      | t
(1 row)

CREATE INDEX disttable_device_time_idx ON disttable (device, time);
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable';
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
21|           11|device     |integer    |f      |         1|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
20|           11|time       |bigint     |t      |          |                        |                  |        1000000|                        |                       |                
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
21|           11|device     |integer    |f      |         1|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
20|           11|time       |bigint     |t      |          |                        |                  |        1000000|                        |                       |                
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
15|            9|device     |integer    |f      |         1|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
14|            9|time       |bigint     |t      |          |                        |                  |        1000000|                        |                       |                
(2 rows)


 remote_exec 
-------------
 
(1 row)

-- Show that changing dimension settings apply to data nodes
SELECT * FROM set_chunk_time_interval('disttable', 2000000000::bigint);
 set_chunk_time_interval 
-------------------------
 
(1 row)

SELECT * FROM set_number_partitions('disttable', 3);
 set_number_partitions 
-----------------------
 
(1 row)

CREATE OR REPLACE FUNCTION dummy_now() RETURNS BIGINT LANGUAGE SQL IMMUTABLE as  'SELECT 2::BIGINT';
CALL distributed_exec($$
CREATE OR REPLACE FUNCTION dummy_now() RETURNS BIGINT LANGUAGE SQL IMMUTABLE as  'SELECT 2::BIGINT'
$$);
SELECT * FROM set_integer_now_func('disttable', 'dummy_now');
 set_integer_now_func 
----------------------
 
(1 row)

-- Show changes to dimensions
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable';
$$);
NOTICE:  [db_dist_hypertable_1]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_1]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
21|           11|device     |integer    |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
20|           11|time       |bigint     |t      |          |                        |                  |     2000000000|                        |public                 |dummy_now       
(2 rows)


NOTICE:  [db_dist_hypertable_2]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_2]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
21|           11|device     |integer    |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
20|           11|time       |bigint     |t      |          |                        |                  |     2000000000|                        |public                 |dummy_now       
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
SELECT d.* FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.dimension d
WHERE h.id = d.hypertable_id AND h.table_name = 'disttable'
NOTICE:  [db_dist_hypertable_3]:
id|hypertable_id|column_name|column_type|aligned|num_slices|partitioning_func_schema|partitioning_func |interval_length|compress_interval_length|integer_now_func_schema|integer_now_func
--+-------------+-----------+-----------+-------+----------+------------------------+------------------+---------------+------------------------+-----------------------+----------------
15|            9|device     |integer    |f      |         3|_timescaledb_internal   |get_partition_hash|               |                        |                       |                
14|            9|time       |bigint     |t      |          |                        |                  |     2000000000|                        |public                 |dummy_now       
(2 rows)


 remote_exec 
-------------
 
(1 row)

-- Tests for using tablespaces with distributed hypertables
\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER;
--Ensure INSERTs use DataNodeDispatch.
SET timescaledb.enable_distributed_insert_with_copy=false;
CREATE TABLESPACE :TABLESPACE_1 OWNER :ROLE_1 LOCATION :'spc1path';
CREATE TABLESPACE :TABLESPACE_2 OWNER :ROLE_1 LOCATION :'spc2path';
\set ON_ERROR_STOP 0
SELECT attach_tablespace(:'TABLESPACE_1', 'disttable');
ERROR:  cannot attach tablespace to distributed hypertable
SELECT detach_tablespace(:'TABLESPACE_1', 'disttable');
ERROR:  tablespace "db_dist_hypertable_1" is not attached to hypertable "disttable"
\set ON_ERROR_STOP 1
SELECT detach_tablespaces('disttable');
 detach_tablespaces 
--------------------
                  0
(1 row)

-- Continue to use previously attached tablespace, but block attach/detach
CREATE TABLE disttable2(time timestamptz, device int, temp float) TABLESPACE :TABLESPACE_1;
SELECT create_distributed_hypertable('disttable2', 'time', chunk_time_interval => 1000000::bigint);
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (13,public,disttable2,t)
(1 row)

-- Ensure that table is created on the data nodes without a tablespace
CALL distributed_exec($$
SELECT * FROM show_tablespaces('disttable2');
$$);
INSERT INTO disttable2 VALUES ('2017-01-01 06:01', 1, 1.1);
SELECT * FROM show_chunks('disttable2');
                  show_chunks                  
-----------------------------------------------
 _timescaledb_internal._dist_hyper_13_35_chunk
(1 row)

-- Ensure tablespace oid is set to 0 for a foreign table
SELECT reltablespace
FROM pg_class cl, (SELECT show_chunks AS chunk FROM show_chunks('disttable2')) ch
WHERE cl.oid = ch.chunk::regclass;
 reltablespace 
---------------
             0
(1 row)

\set ON_ERROR_STOP 0
SELECT attach_tablespace(:'TABLESPACE_2', 'disttable2');
ERROR:  cannot attach tablespace to distributed hypertable
SELECT detach_tablespace(:'TABLESPACE_2', 'disttable2');
ERROR:  tablespace "db_dist_hypertable_2" is not attached to hypertable "disttable2"
\set ON_ERROR_STOP 1
SELECT detach_tablespaces('disttable2');
 detach_tablespaces 
--------------------
                  0
(1 row)

SELECT * FROM show_tablespaces('disttable2');
 show_tablespaces 
------------------
(0 rows)

-- Ensure tablespace API works for data nodes
CALL distributed_exec(format($$
SELECT attach_tablespace(%L, 'disttable2');
$$, :'TABLESPACE_2'));
CALL distributed_exec(format($$
SELECT detach_tablespace(%L, 'disttable2');
$$, :'TABLESPACE_2'));
CALL distributed_exec(format($$
SELECT attach_tablespace(%L, 'disttable2');
$$, :'TABLESPACE_2'));
CALL distributed_exec($$
SELECT detach_tablespaces('disttable2');
$$);
DROP TABLE disttable2;
CREATE TABLE disttable2(time timestamptz, device int, temp float) TABLESPACE :TABLESPACE_1;
SELECT create_hypertable('disttable2', 'time', chunk_time_interval => 1000000::bigint, replication_factor => 1);
NOTICE:  adding not-null constraint to column "time"
    create_hypertable     
--------------------------
 (14,public,disttable2,t)
(1 row)

-- Ensure that table is created on the data nodes without a tablespace
CALL distributed_exec($$
SELECT * FROM show_tablespaces('disttable2');
$$);
INSERT INTO disttable2 VALUES ('2017-01-01 06:01', 1, 1.1);
SELECT * FROM show_chunks('disttable2');
                  show_chunks                  
-----------------------------------------------
 _timescaledb_internal._dist_hyper_14_36_chunk
(1 row)

-- Ensure tablespace oid is set to 0 for a foreign table
SELECT reltablespace
FROM pg_class cl, (SELECT show_chunks AS chunk FROM show_chunks('disttable2')) ch
WHERE cl.oid = ch.chunk::regclass;
 reltablespace 
---------------
             0
(1 row)

\set ON_ERROR_STOP 0
SELECT attach_tablespace(:'TABLESPACE_2', 'disttable2');
ERROR:  cannot attach tablespace to distributed hypertable
SELECT detach_tablespace(:'TABLESPACE_2', 'disttable2');
ERROR:  tablespace "db_dist_hypertable_2" is not attached to hypertable "disttable2"
\set ON_ERROR_STOP 1
SELECT * FROM show_tablespaces('disttable2');
 show_tablespaces 
------------------
(0 rows)

DROP TABLE disttable2;
DROP TABLESPACE :TABLESPACE_1;
DROP TABLESPACE :TABLESPACE_2;
-- Make sure table qualified name is used in chunks_in function. Otherwise having a table name same as a column name might yield an error
CREATE TABLE dist_device(time timestamptz, dist_device int, temp float);
SELECT * FROM create_distributed_hypertable('dist_device', 'time');
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name | table_name  | created 
---------------+-------------+-------------+---------
            15 | public      | dist_device | t
(1 row)

INSERT INTO dist_device VALUES
       ('2017-01-01 06:01', 1, 1.1),
       ('2017-01-01 09:11', 3, 2.1),
       ('2017-01-01 08:01', 1, 1.2);
EXPLAIN (VERBOSE, COSTS OFF)
SELECT * FROM dist_device;
                                                                  QUERY PLAN                                                                   
-----------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (DataNodeScan) on public.dist_device
   Output: dist_device."time", dist_device.dist_device, dist_device.temp
   Data node: db_dist_hypertable_1
   Chunks: _dist_hyper_15_37_chunk
   Remote SQL: SELECT "time", dist_device, temp FROM public.dist_device WHERE _timescaledb_internal.chunks_in(public.dist_device.*, ARRAY[22])
(5 rows)

-- Check that datanodes use ChunkAppend plans with chunks_in function in the
-- "Remote SQL" when only time partitioning is being used.
SET timescaledb.enable_remote_explain = ON;
EXPLAIN (VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
SELECT "time", dist_device, temp FROM public.dist_device ORDER BY "time" ASC NULLS LAST;
                                                                                  QUERY PLAN                                                                                  
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (DataNodeScan) on public.dist_device
   Output: dist_device."time", dist_device.dist_device, dist_device.temp
   Data node: db_dist_hypertable_1
   Chunks: _dist_hyper_15_37_chunk
   Remote SQL: SELECT "time", dist_device, temp FROM public.dist_device WHERE _timescaledb_internal.chunks_in(public.dist_device.*, ARRAY[22]) ORDER BY "time" ASC NULLS LAST
   Remote EXPLAIN: 
     Custom Scan (ChunkAppend) on public.dist_device
       Output: dist_device."time", dist_device.dist_device, dist_device.temp
       Order: dist_device."time"
       Startup Exclusion: false
       Runtime Exclusion: false
       ->  Index Scan Backward using _dist_hyper_15_37_chunk_dist_device_time_idx on _timescaledb_internal._dist_hyper_15_37_chunk
             Output: _dist_hyper_15_37_chunk."time", _dist_hyper_15_37_chunk.dist_device, _dist_hyper_15_37_chunk.temp
 
(14 rows)

SELECT * FROM dist_device;
             time             | dist_device | temp 
------------------------------+-------------+------
 Sun Jan 01 06:01:00 2017 PST |           1 |  1.1
 Sun Jan 01 09:11:00 2017 PST |           3 |  2.1
 Sun Jan 01 08:01:00 2017 PST |           1 |  1.2
(3 rows)

-- Test estimating relation size without stats
CREATE TABLE hyper_estimate(time timestamptz, device int, temp float);
SELECT * FROM create_distributed_hypertable('hyper_estimate', 'time', 'device', number_partitions => 3, replication_factor => 1, chunk_time_interval => INTERVAL '7 days');
NOTICE:  adding not-null constraint to column "time"
 hypertable_id | schema_name |   table_name   | created 
---------------+-------------+----------------+---------
            16 | public      | hyper_estimate | t
(1 row)

-- This will enable us to more easily see estimates per chunk
SET timescaledb.enable_per_data_node_queries = false;
-- Estimating chunk progress uses current timestamp so we override it for test purposes
SELECT test.tsl_override_current_timestamptz('2017-11-11 00:00'::timestamptz);
 tsl_override_current_timestamptz 
----------------------------------
 
(1 row)

-- Test estimates when backfilling. 3 chunks should be historical and 3 should be considered current when estimating.
-- Note that estimate numbers are way off since we are using shared buffer size as starting point. This will not be
-- an issue in 'production' like env since chunk size should be similar to shared buffer size.
INSERT INTO hyper_estimate VALUES
       ('2017-01-01 06:01', 1, 1.1),
       ('2017-01-01 09:11', 1, 2.1),
       ('2017-01-01 08:01', 1, 1.2),
       ('2017-01-02 08:01', 1, 1.3),
       ('2017-01-02 08:01', 2, 1.6),
       ('2017-01-02 06:01', 2, 1.4),
       ('2017-01-03 01:01', 3, 2),
       ('2017-01-03 01:16', 3, 3),
       ('2017-01-03 01:17', 3, 4),
       ('2018-01-13 01:01', 1, 2),
       ('2018-01-13 01:10', 1, 0.4),
       ('2018-01-13 02:10', 2, 1.4),
       ('2018-01-13 05:01', 2, 2),
       ('2018-01-13 05:50', 2, 4),
       ('2018-01-13 16:01', 3, 2);
-- This will calculate the stats
ANALYZE hyper_estimate;
EXPLAIN (COSTS ON)
SELECT *
FROM hyper_estimate;
                                      QUERY PLAN                                      
--------------------------------------------------------------------------------------
 Append  (cost=100.00..607.58 rows=15 width=20)
   ->  Foreign Scan on _dist_hyper_16_38_chunk  (cost=100.00..101.40 rows=4 width=20)
   ->  Foreign Scan on _dist_hyper_16_39_chunk  (cost=100.00..101.20 rows=2 width=20)
   ->  Foreign Scan on _dist_hyper_16_40_chunk  (cost=100.00..101.30 rows=3 width=20)
   ->  Foreign Scan on _dist_hyper_16_41_chunk  (cost=100.00..101.20 rows=2 width=20)
   ->  Foreign Scan on _dist_hyper_16_42_chunk  (cost=100.00..101.30 rows=3 width=20)
   ->  Foreign Scan on _dist_hyper_16_43_chunk  (cost=100.00..101.10 rows=1 width=20)
(7 rows)

-- Let's insert data into a new chunk. This will result in chunk creation.
INSERT INTO hyper_estimate VALUES ('2019-11-11 06:01', 1, 1.1);
-- We have stats for previous chunks so we can interpolate number of rows for the new chunk
EXPLAIN (COSTS ON)
SELECT *
FROM hyper_estimate;
                                      QUERY PLAN                                      
--------------------------------------------------------------------------------------
 Append  (cost=100.00..707.77 rows=17 width=20)
   ->  Foreign Scan on _dist_hyper_16_38_chunk  (cost=100.00..101.40 rows=4 width=20)
   ->  Foreign Scan on _dist_hyper_16_39_chunk  (cost=100.00..101.20 rows=2 width=20)
   ->  Foreign Scan on _dist_hyper_16_40_chunk  (cost=100.00..101.30 rows=3 width=20)
   ->  Foreign Scan on _dist_hyper_16_41_chunk  (cost=100.00..101.20 rows=2 width=20)
   ->  Foreign Scan on _dist_hyper_16_42_chunk  (cost=100.00..101.30 rows=3 width=20)
   ->  Foreign Scan on _dist_hyper_16_43_chunk  (cost=100.00..101.10 rows=1 width=20)
   ->  Foreign Scan on _dist_hyper_16_44_chunk  (cost=100.00..100.19 rows=2 width=20)
(8 rows)

CREATE TABLE devices (
       device_id INTEGER PRIMARY KEY,
       device_name VARCHAR(10)
);
CALL distributed_exec($$
  CREATE TABLE devices(device_id INTEGER PRIMARY KEY, device_name VARCHAR(10))
$$);
INSERT INTO devices VALUES
  (1, 'A001'), (2, 'B015'), (3, 'D821'), (4, 'C561'), (5, 'D765');
CALL distributed_exec($$
  INSERT INTO devices VALUES
    (1, 'A001'), (2, 'B015'), (3, 'D821'), (4, 'C561'), (5, 'D765')
$$);
CREATE TABLE hyper (
  time TIMESTAMPTZ NOT NULL,
  device INTEGER REFERENCES devices(device_id),
  temp FLOAT
);
SELECT * FROM create_distributed_hypertable('hyper', 'time', 'device', 3,
       chunk_time_interval => interval '18 hours'
);
WARNING:  distributed hypertable "hyper" has a foreign key to a non-distributed table
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            17 | public      | hyper      | t
(1 row)

-- Inserting some values should succeed.
INSERT INTO hyper VALUES
       ('2017-01-01 06:01', 1, 1.1),
       ('2017-01-01 09:11', 1, 2.1),
       ('2017-01-01 08:01', 1, 1.2),
       ('2017-01-02 08:01', 1, 1.3),
       ('2017-01-02 08:01', 2, 1.6),
       ('2017-01-02 06:01', 2, 1.4),
       ('2017-01-03 01:01', 3, 2),
       ('2017-01-03 01:16', 3, 3),
       ('2017-01-03 01:17', 3, 4),
       ('2018-01-13 01:01', 1, 2),
       ('2018-01-13 01:10', 1, 0.4),
       ('2018-01-13 02:10', 2, 1.4),
       ('2018-01-13 05:01', 2, 2),
       ('2018-01-13 05:50', 2, 4),
       ('2018-01-13 16:01', 3, 2);
SELECT time_bucket('3 hours', time) AS time, device, avg(temp) AS avg_temp
FROM hyper
GROUP BY 1, 2
HAVING avg(temp) > 1.2
ORDER BY 1;
             time             | device | avg_temp 
------------------------------+--------+----------
 Sun Jan 01 07:00:00 2017 PST |      1 |     1.65
 Mon Jan 02 04:00:00 2017 PST |      2 |      1.4
 Mon Jan 02 07:00:00 2017 PST |      2 |      1.6
 Mon Jan 02 07:00:00 2017 PST |      1 |      1.3
 Tue Jan 03 01:00:00 2017 PST |      3 |        3
 Sat Jan 13 01:00:00 2018 PST |      2 |      1.4
 Sat Jan 13 04:00:00 2018 PST |      2 |        3
 Sat Jan 13 16:00:00 2018 PST |      3 |        2
(8 rows)

-- Add some devices on the access node only. Inserts should then fail.
INSERT INTO devices VALUES (6, 'E999');
\set ON_ERROR_STOP 0
INSERT INTO hyper VALUES ('2017-01-01 06:01', 6, 1.1);
ERROR:  [db_dist_hypertable_1]: insert or update on table "_dist_hyper_17_45_chunk" violates foreign key constraint "26_17_hyper_device_fkey"
\set ON_ERROR_STOP 1
-- Test alter replication factor with data
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper');
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      26|           16|_timescaledb_internal|_dist_hyper_17_45_chunk|r      |{"time": [1483272000000000, 1483336800000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_46_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [-9223372036854775808, 715827882]}
      28|           16|_timescaledb_internal|_dist_hyper_17_49_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                           
--------+-------------+---------------------+-----------------------+-------+---------------------------------------------------------------------------------
      23|           16|_timescaledb_internal|_dist_hyper_17_47_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [715827882, 1431655764]}
      24|           16|_timescaledb_internal|_dist_hyper_17_50_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [715827882, 1431655764]}
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           14|_timescaledb_internal|_dist_hyper_17_48_chunk|r      |{"time": [1483401600000000, 1483466400000000], "device": [1431655764, 9223372036854775807]}
      24|           14|_timescaledb_internal|_dist_hyper_17_51_chunk|r      |{"time": [1515866400000000, 1515931200000000], "device": [1431655764, 9223372036854775807]}
(2 rows)


 remote_exec 
-------------
 
(1 row)

-- Dimension partitions should be updated to account for replication
-- to additional data nodes
SELECT * FROM hypertable_partitions WHERE table_name = 'hyper';
 table_name | dimension_id |     range_start      |       data_nodes       
------------+--------------+----------------------+------------------------
 hyper      |           29 | -9223372036854775808 | {db_dist_hypertable_1}
 hyper      |           29 |            715827882 | {db_dist_hypertable_2}
 hyper      |           29 |           1431655764 | {db_dist_hypertable_3}
(3 rows)

SELECT * FROM set_replication_factor('hyper', 3);
WARNING:  hypertable "hyper" is under-replicated
 set_replication_factor 
------------------------
 
(1 row)

SELECT * FROM hypertable_partitions WHERE table_name = 'hyper';
 table_name | dimension_id |     range_start      |                            data_nodes                            
------------+--------------+----------------------+------------------------------------------------------------------
 hyper      |           29 | -9223372036854775808 | {db_dist_hypertable_1,db_dist_hypertable_2,db_dist_hypertable_3}
 hyper      |           29 |            715827882 | {db_dist_hypertable_2,db_dist_hypertable_3,db_dist_hypertable_1}
 hyper      |           29 |           1431655764 | {db_dist_hypertable_3,db_dist_hypertable_1,db_dist_hypertable_2}
(3 rows)

INSERT INTO hyper VALUES ('2017-01-02 07:11', 1, 1.7);
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper');
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      26|           16|_timescaledb_internal|_dist_hyper_17_45_chunk|r      |{"time": [1483272000000000, 1483336800000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_46_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [-9223372036854775808, 715827882]}
      28|           16|_timescaledb_internal|_dist_hyper_17_49_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                           
--------+-------------+---------------------+-----------------------+-------+---------------------------------------------------------------------------------
      23|           16|_timescaledb_internal|_dist_hyper_17_47_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [715827882, 1431655764]}
      24|           16|_timescaledb_internal|_dist_hyper_17_50_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [715827882, 1431655764]}
(2 rows)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           14|_timescaledb_internal|_dist_hyper_17_48_chunk|r      |{"time": [1483401600000000, 1483466400000000], "device": [1431655764, 9223372036854775807]}
      24|           14|_timescaledb_internal|_dist_hyper_17_51_chunk|r      |{"time": [1515866400000000, 1515931200000000], "device": [1431655764, 9223372036854775807]}
(2 rows)


 remote_exec 
-------------
 
(1 row)

INSERT INTO hyper VALUES ('2017-02-01 06:01', 1, 5.1);
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper');
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      26|           16|_timescaledb_internal|_dist_hyper_17_45_chunk|r      |{"time": [1483272000000000, 1483336800000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_46_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [-9223372036854775808, 715827882]}
      28|           16|_timescaledb_internal|_dist_hyper_17_49_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [-9223372036854775808, 715827882]}
      29|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
(4 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           16|_timescaledb_internal|_dist_hyper_17_47_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [715827882, 1431655764]}          
      24|           16|_timescaledb_internal|_dist_hyper_17_50_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [715827882, 1431655764]}          
      25|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           14|_timescaledb_internal|_dist_hyper_17_48_chunk|r      |{"time": [1483401600000000, 1483466400000000], "device": [1431655764, 9223372036854775807]}
      24|           14|_timescaledb_internal|_dist_hyper_17_51_chunk|r      |{"time": [1515866400000000, 1515931200000000], "device": [1431655764, 9223372036854775807]}
      25|           14|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM set_replication_factor('hyper', 2);
WARNING:  hypertable "hyper" is under-replicated
 set_replication_factor 
------------------------
 
(1 row)

SELECT * FROM hypertable_partitions WHERE table_name = 'hyper';
 table_name | dimension_id |     range_start      |                 data_nodes                  
------------+--------------+----------------------+---------------------------------------------
 hyper      |           29 | -9223372036854775808 | {db_dist_hypertable_1,db_dist_hypertable_2}
 hyper      |           29 |            715827882 | {db_dist_hypertable_2,db_dist_hypertable_3}
 hyper      |           29 |           1431655764 | {db_dist_hypertable_3,db_dist_hypertable_1}
(3 rows)

INSERT INTO hyper VALUES ('2017-03-01 06:01', 1, 15.1);
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper');
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      26|           16|_timescaledb_internal|_dist_hyper_17_45_chunk|r      |{"time": [1483272000000000, 1483336800000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_46_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [-9223372036854775808, 715827882]}
      28|           16|_timescaledb_internal|_dist_hyper_17_49_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [-9223372036854775808, 715827882]}
      29|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
      30|           16|_timescaledb_internal|_dist_hyper_17_53_chunk|r      |{"time": [1488326400000000, 1488391200000000], "device": [-9223372036854775808, 715827882]}
(5 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           16|_timescaledb_internal|_dist_hyper_17_47_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [715827882, 1431655764]}          
      24|           16|_timescaledb_internal|_dist_hyper_17_50_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [715827882, 1431655764]}          
      25|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
      26|           16|_timescaledb_internal|_dist_hyper_17_53_chunk|r      |{"time": [1488326400000000, 1488391200000000], "device": [-9223372036854775808, 715827882]}
(4 rows)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           14|_timescaledb_internal|_dist_hyper_17_48_chunk|r      |{"time": [1483401600000000, 1483466400000000], "device": [1431655764, 9223372036854775807]}
      24|           14|_timescaledb_internal|_dist_hyper_17_51_chunk|r      |{"time": [1515866400000000, 1515931200000000], "device": [1431655764, 9223372036854775807]}
      25|           14|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
(3 rows)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM set_replication_factor('hyper', replication_factor => 2);
WARNING:  hypertable "hyper" is under-replicated
 set_replication_factor 
------------------------
 
(1 row)

INSERT INTO hyper VALUES ('2017-04-01 06:01', 2, 45.1);
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper');
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_1]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      26|           16|_timescaledb_internal|_dist_hyper_17_45_chunk|r      |{"time": [1483272000000000, 1483336800000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_46_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [-9223372036854775808, 715827882]}
      28|           16|_timescaledb_internal|_dist_hyper_17_49_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [-9223372036854775808, 715827882]}
      29|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
      30|           16|_timescaledb_internal|_dist_hyper_17_53_chunk|r      |{"time": [1488326400000000, 1488391200000000], "device": [-9223372036854775808, 715827882]}
(5 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_2]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           16|_timescaledb_internal|_dist_hyper_17_47_chunk|r      |{"time": [1483336800000000, 1483401600000000], "device": [715827882, 1431655764]}          
      24|           16|_timescaledb_internal|_dist_hyper_17_50_chunk|r      |{"time": [1515801600000000, 1515866400000000], "device": [715827882, 1431655764]}          
      25|           16|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
      26|           16|_timescaledb_internal|_dist_hyper_17_53_chunk|r      |{"time": [1488326400000000, 1488391200000000], "device": [-9223372036854775808, 715827882]}
      27|           16|_timescaledb_internal|_dist_hyper_17_54_chunk|r      |{"time": [1491048000000000, 1491112800000000], "device": [715827882, 1431655764]}          
(5 rows)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
    FROM show_chunks('hyper')
NOTICE:  [db_dist_hypertable_3]:
chunk_id|hypertable_id|schema_name          |table_name             |relkind|slices                                                                                     
--------+-------------+---------------------+-----------------------+-------+-------------------------------------------------------------------------------------------
      23|           14|_timescaledb_internal|_dist_hyper_17_48_chunk|r      |{"time": [1483401600000000, 1483466400000000], "device": [1431655764, 9223372036854775807]}
      24|           14|_timescaledb_internal|_dist_hyper_17_51_chunk|r      |{"time": [1515866400000000, 1515931200000000], "device": [1431655764, 9223372036854775807]}
      25|           14|_timescaledb_internal|_dist_hyper_17_52_chunk|r      |{"time": [1485928800000000, 1485993600000000], "device": [-9223372036854775808, 715827882]}
      26|           14|_timescaledb_internal|_dist_hyper_17_54_chunk|r      |{"time": [1491048000000000, 1491112800000000], "device": [715827882, 1431655764]}          
(4 rows)


 remote_exec 
-------------
 
(1 row)

\set ON_ERROR_STOP 0
SELECT * FROM set_replication_factor('hyper', replication_factor => 4);
ERROR:  replication factor too large for hypertable "hyper"
\set ON_ERROR_STOP 1
DROP TABLE hyper;
CALL distributed_exec($$
    DROP TABLE devices;
$$);
DROP TABLE devices;
-- Test storage options are distributed to data nodes
--
-- Make sure that options used during CREATE TABLE WITH and CREATE INDEX WITH
-- are properly distributed.
--
CREATE TABLE disttable_with_relopts_1(time timestamptz NOT NULL, device int) WITH (fillfactor=10);
CREATE TABLE disttable_with_relopts_2(time timestamptz NOT NULL, device int) WITH (fillfactor=10, parallel_workers=1);
CREATE TABLE disttable_with_relopts_3(time timestamptz NOT NULL, device int);
CREATE INDEX disttable_with_relopts_3_idx ON disttable_with_relopts_3(device) WITH (fillfactor=20);
SELECT * FROM create_distributed_hypertable('disttable_with_relopts_1', 'time');
 hypertable_id | schema_name |        table_name        | created 
---------------+-------------+--------------------------+---------
            18 | public      | disttable_with_relopts_1 | t
(1 row)

SELECT * FROM create_distributed_hypertable('disttable_with_relopts_2', 'time');
 hypertable_id | schema_name |        table_name        | created 
---------------+-------------+--------------------------+---------
            19 | public      | disttable_with_relopts_2 | t
(1 row)

SELECT * FROM create_distributed_hypertable('disttable_with_relopts_3', 'time');
 hypertable_id | schema_name |        table_name        | created 
---------------+-------------+--------------------------+---------
            20 | public      | disttable_with_relopts_3 | t
(1 row)

INSERT INTO disttable_with_relopts_1 VALUES
       ('2017-01-01 06:01', 1),
       ('2017-01-01 09:11', 3),
       ('2017-01-01 08:01', 1),
       ('2017-01-02 08:01', 2),
       ('2018-07-02 08:01', 87);
INSERT INTO disttable_with_relopts_2 VALUES
       ('2017-01-01 06:01', 1),
       ('2017-01-01 09:11', 3),
       ('2017-01-01 08:01', 1),
       ('2017-01-02 08:01', 2),
       ('2018-07-02 08:01', 87);
SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname;
         relname          |   reloptions    
--------------------------+-----------------
 disttable_with_relopts_1 | {fillfactor=10}
(1 row)

SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_2' ORDER BY relname;
         relname          |             reloptions             
--------------------------+------------------------------------
 disttable_with_relopts_2 | {fillfactor=10,parallel_workers=1}
(1 row)

SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3' ORDER BY relname;
         relname          | reloptions 
--------------------------+------------
 disttable_with_relopts_3 | 
(1 row)

SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3_idx' ORDER BY relname;
           relname            |   reloptions    
------------------------------+-----------------
 disttable_with_relopts_3_idx | {fillfactor=20}
(1 row)

-- Ensure reloptions are not set for distributed hypertable chunks on the AN
SELECT relname, reloptions FROM pg_class WHERE relname IN
(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
ORDER BY relname;
         relname         | reloptions 
-------------------------+------------
 _dist_hyper_18_55_chunk | 
 _dist_hyper_18_56_chunk | 
(2 rows)

SELECT relname, reloptions FROM pg_class WHERE relname IN
(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_2'))
ORDER BY relname;
         relname         | reloptions 
-------------------------+------------
 _dist_hyper_19_57_chunk | 
 _dist_hyper_19_58_chunk | 
(2 rows)

-- Ensure parent tables has proper storage options
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                 |reloptions     
------------------------+---------------
disttable_with_relopts_1|{fillfactor=10}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                 |reloptions     
------------------------+---------------
disttable_with_relopts_1|{fillfactor=10}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname                 |reloptions     
------------------------+---------------
disttable_with_relopts_1|{fillfactor=10}
(1 row)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_2' ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_2' ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                 |reloptions                        
------------------------+----------------------------------
disttable_with_relopts_2|{fillfactor=10,parallel_workers=1}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_2' ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                 |reloptions                        
------------------------+----------------------------------
disttable_with_relopts_2|{fillfactor=10,parallel_workers=1}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_2' ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname                 |reloptions                        
------------------------+----------------------------------
disttable_with_relopts_2|{fillfactor=10,parallel_workers=1}
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Ensure index has proper storage options set
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3_idx' ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3_idx' ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                     |reloptions     
----------------------------+---------------
disttable_with_relopts_3_idx|{fillfactor=20}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3_idx' ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                     |reloptions     
----------------------------+---------------
disttable_with_relopts_3_idx|{fillfactor=20}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_3_idx' ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname                     |reloptions     
----------------------------+---------------
disttable_with_relopts_3_idx|{fillfactor=20}
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Make sure chunks derive parent reloptions
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                |reloptions     
-----------------------+---------------
_dist_hyper_18_55_chunk|{fillfactor=10}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                |reloptions     
-----------------------+---------------
_dist_hyper_18_56_chunk|{fillfactor=10}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname|reloptions
-------+----------
(0 rows)


 remote_exec 
-------------
 
(1 row)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_2'))
    ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_2'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname|reloptions
-------+----------
(0 rows)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_2'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                |reloptions                        
-----------------------+----------------------------------
_dist_hyper_19_57_chunk|{fillfactor=10,parallel_workers=1}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_2'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname                |reloptions                        
-----------------------+----------------------------------
_dist_hyper_19_58_chunk|{fillfactor=10,parallel_workers=1}
(1 row)


 remote_exec 
-------------
 
(1 row)

-- ALTER TABLE SET/RESET support for distributed hypertable
--
-- SET
ALTER TABLE disttable_with_relopts_1 SET (fillfactor=40);
SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname;
         relname          |   reloptions    
--------------------------+-----------------
 disttable_with_relopts_1 | {fillfactor=40}
(1 row)

-- Ensure chunks are not affected on the AN
SELECT relname, reloptions FROM pg_class WHERE relname IN
(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
ORDER BY relname;
         relname         | reloptions 
-------------------------+------------
 _dist_hyper_18_55_chunk | 
 _dist_hyper_18_56_chunk | 
(2 rows)

-- Ensure data node chunks has proper options set
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                |reloptions     
-----------------------+---------------
_dist_hyper_18_55_chunk|{fillfactor=40}
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                |reloptions     
-----------------------+---------------
_dist_hyper_18_56_chunk|{fillfactor=40}
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname|reloptions
-------+----------
(0 rows)


 remote_exec 
-------------
 
(1 row)

-- RESET
ALTER TABLE disttable_with_relopts_1 RESET (fillfactor);
SELECT relname, reloptions FROM pg_class WHERE relname = 'disttable_with_relopts_1' ORDER BY relname;
         relname          | reloptions 
--------------------------+------------
 disttable_with_relopts_1 | 
(1 row)

-- Ensure chunks are not affected on the AN
SELECT relname, reloptions FROM pg_class WHERE relname IN
(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
ORDER BY relname;
         relname         | reloptions 
-------------------------+------------
 _dist_hyper_18_55_chunk | 
 _dist_hyper_18_56_chunk | 
(2 rows)

-- Ensure data node chunks has proper options set
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname;
$$);
NOTICE:  [db_dist_hypertable_1]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_1]:
relname                |reloptions
-----------------------+----------
_dist_hyper_18_55_chunk|          
(1 row)


NOTICE:  [db_dist_hypertable_2]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_2]:
relname                |reloptions
-----------------------+----------
_dist_hyper_18_56_chunk|          
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
    SELECT relname, reloptions FROM pg_class WHERE relname IN
    (SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable_with_relopts_1'))
    ORDER BY relname
NOTICE:  [db_dist_hypertable_3]:
relname|reloptions
-------+----------
(0 rows)


 remote_exec 
-------------
 
(1 row)

-- WITH OIDS and WITHOUT OIDS are tested in a separate test since those
-- options were dropped in PG12
DROP TABLE disttable_with_relopts_1;
DROP TABLE disttable_with_relopts_2;
DROP TABLE disttable_with_relopts_3;
-- Test SERIAL type column support for distributed hypertables
--
CREATE TABLE disttable_serial(time timestamptz NOT NULL, device int, id1 SERIAL, id2 SMALLSERIAL, id3 BIGSERIAL);
SELECT create_distributed_hypertable('disttable_serial', 'time', 'device');
 create_distributed_hypertable  
--------------------------------
 (21,public,disttable_serial,t)
(1 row)

-- Show created columns (AN and DN tables must be exact)
SELECT * FROM test.show_columns('disttable_serial');
 Column |           Type           | NotNull 
--------+--------------------------+---------
 time   | timestamp with time zone | t
 device | integer                  | f
 id1    | integer                  | t
 id2    | smallint                 | t
 id3    | bigint                   | t
(5 rows)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
	SELECT * FROM test.show_columns('disttable_serial');
$$);
NOTICE:  [db_dist_hypertable_1]: 
	SELECT * FROM test.show_columns('disttable_serial')
NOTICE:  [db_dist_hypertable_1]:
Column|Type                    |NotNull
------+------------------------+-------
time  |timestamp with time zone|t      
device|integer                 |f      
id1   |integer                 |t      
id2   |smallint                |t      
id3   |bigint                  |t      
(5 rows)


NOTICE:  [db_dist_hypertable_2]: 
	SELECT * FROM test.show_columns('disttable_serial')
NOTICE:  [db_dist_hypertable_2]:
Column|Type                    |NotNull
------+------------------------+-------
time  |timestamp with time zone|t      
device|integer                 |f      
id1   |integer                 |t      
id2   |smallint                |t      
id3   |bigint                  |t      
(5 rows)


NOTICE:  [db_dist_hypertable_3]: 
	SELECT * FROM test.show_columns('disttable_serial')
NOTICE:  [db_dist_hypertable_3]:
Column|Type                    |NotNull
------+------------------------+-------
time  |timestamp with time zone|t      
device|integer                 |f      
id1   |integer                 |t      
id2   |smallint                |t      
id3   |bigint                  |t      
(5 rows)


 remote_exec 
-------------
 
(1 row)

-- Ensure DEFAULT expression is applied on the AN only
SELECT column_name, column_default
FROM information_schema.columns
WHERE table_name  = 'disttable_serial';
 column_name |                column_default                 
-------------+-----------------------------------------------
 time        | 
 device      | 
 id1         | nextval('disttable_serial_id1_seq'::regclass)
 id2         | nextval('disttable_serial_id2_seq'::regclass)
 id3         | nextval('disttable_serial_id3_seq'::regclass)
(5 rows)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
	SELECT column_name, column_default
	FROM information_schema.columns
	WHERE table_name  = 'disttable_serial';
$$);
NOTICE:  [db_dist_hypertable_1]: 
	SELECT column_name, column_default
	FROM information_schema.columns
	WHERE table_name  = 'disttable_serial'
NOTICE:  [db_dist_hypertable_1]:
column_name|column_default
-----------+--------------
time       |              
device     |              
id1        |              
id2        |              
id3        |              
(5 rows)


NOTICE:  [db_dist_hypertable_2]: 
	SELECT column_name, column_default
	FROM information_schema.columns
	WHERE table_name  = 'disttable_serial'
NOTICE:  [db_dist_hypertable_2]:
column_name|column_default
-----------+--------------
time       |              
device     |              
id1        |              
id2        |              
id3        |              
(5 rows)


NOTICE:  [db_dist_hypertable_3]: 
	SELECT column_name, column_default
	FROM information_schema.columns
	WHERE table_name  = 'disttable_serial'
NOTICE:  [db_dist_hypertable_3]:
column_name|column_default
-----------+--------------
time       |              
device     |              
id1        |              
id2        |              
id3        |              
(5 rows)


 remote_exec 
-------------
 
(1 row)

-- Ensure sequences were created on the AN only
INSERT INTO disttable_serial VALUES
       ('2017-01-01 06:01', 1),
       ('2017-01-01 09:11', 3),
       ('2017-01-01 08:01', 1),
       ('2017-01-02 08:01', 2),
       ('2018-07-02 08:01', 87);
SELECT currval('disttable_serial_id1_seq'::regclass),
       currval('disttable_serial_id2_seq'::regclass),
       currval('disttable_serial_id3_seq'::regclass);
 currval | currval | currval 
---------+---------+---------
       5 |       5 |       5
(1 row)

\set ON_ERROR_STOP 0
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1'], $$
	SELECT currval('disttable_serial_id1_seq'::regclass);
$$);
NOTICE:  [db_dist_hypertable_1]: 
	SELECT currval('disttable_serial_id1_seq'::regclass)
ERROR:  [db_dist_hypertable_1]: relation "disttable_serial_id1_seq" does not exist
\set ON_ERROR_STOP 1
-- Verify that the data is getting spread over multiple data nodes with the
-- serial values being set correctly
SELECT * from disttable_serial ORDER BY id1;
             time             | device | id1 | id2 | id3 
------------------------------+--------+-----+-----+-----
 Sun Jan 01 06:01:00 2017 PST |      1 |   1 |   1 |   1
 Sun Jan 01 09:11:00 2017 PST |      3 |   2 |   2 |   2
 Sun Jan 01 08:01:00 2017 PST |      1 |   3 |   3 |   3
 Mon Jan 02 08:01:00 2017 PST |      2 |   4 |   4 |   4
 Mon Jul 02 08:01:00 2018 PDT |     87 |   5 |   5 |   5
(5 rows)

SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
	SELECT * from disttable_serial ORDER BY id1;
$$);
NOTICE:  [db_dist_hypertable_1]: 
	SELECT * from disttable_serial ORDER BY id1
NOTICE:  [db_dist_hypertable_1]:
time                        |device|id1|id2|id3
----------------------------+------+---+---+---
Sun Jan 01 06:01:00 2017 PST|     1|  1|  1|  1
Sun Jan 01 08:01:00 2017 PST|     1|  3|  3|  3
Mon Jul 02 08:01:00 2018 PDT|    87|  5|  5|  5
(3 rows)


NOTICE:  [db_dist_hypertable_2]: 
	SELECT * from disttable_serial ORDER BY id1
NOTICE:  [db_dist_hypertable_2]:
time                        |device|id1|id2|id3
----------------------------+------+---+---+---
Mon Jan 02 08:01:00 2017 PST|     2|  4|  4|  4
(1 row)


NOTICE:  [db_dist_hypertable_3]: 
	SELECT * from disttable_serial ORDER BY id1
NOTICE:  [db_dist_hypertable_3]:
time                        |device|id1|id2|id3
----------------------------+------+---+---+---
Sun Jan 01 09:11:00 2017 PST|     3|  2|  2|  2
(1 row)


 remote_exec 
-------------
 
(1 row)

DROP TABLE disttable_serial;
-- Test insert batching case which will hit the limit of arguments for 
-- prepared statements (65k).
--
-- Issue: #1702
-- distributed hypertable insert fails when # of columns are more than 65
--
-- Use default value
SET timescaledb.max_insert_batch_size TO 1000;
CREATE TABLE test_1702 (
	id varchar(100) NOT NULL,
	time timestamp NOT NULL,
	dummy1	int	,
	dummy2	int	,
	dummy4	int	,
	dummy5	int	,
	dummy6	int	,
	dummy7	int	,
	dummy8	int	,
	dummy9	int	,
	dummy10	int	,
	dummy11	int	,
	dummy12	int	,
	dummy13	int	,
	dummy14	int	,
	dummy15	int	,
	dummy16	int	,
	dummy17	int	,
	dummy18	int	,
	dummy19	int	,
	dummy20	int	,
	dummy21	int	,
	dummy22	int	,
	dummy23	int	,
	dummy24	int	,
	dummy25	int	,
	dummy26	int	,
	dummy27	int	,
	dummy28	int	,
	dummy29	int	,
	dummy30	int	,
	dummy31	int	,
	dummy32	int	,
	dummy33	int	,
	dummy34	int	,
	dummy35	int	,
	dummy36	int	,
	dummy37	int	,
	dummy38	int	,
	dummy39	int	,
	dummy40	int	,
	dummy41	int	,
	dummy42	int	,
	dummy43	int	,
	dummy44	int	,
	dummy45	int	,
	dummy46	int	,
	dummy47	int	,
	dummy48	int	,
	dummy49	int	,
	dummy50	int	,
	dummy51	int	,
	dummy52	int	,
	dummy53	int	,
	dummy54	int	,
	dummy55	int	,
	dummy56	int	,
	dummy57	int	,
	dummy58	int	,
	dummy59	int	,
	dummy60	int	,
	dummy61	int	,
	dummy62	int	,
	dummy63	int	,
	dummy64	int	,
	dummy65	int	,
	dummy66	int	,
	dummy67	int	,
	dummy68	int	,
	dummy69	int	,
	dummy70	int	,
	dummy71	int	
);
SELECT create_distributed_hypertable('test_1702', 'time', 'id');
WARNING:  column type "character varying" used for "id" does not follow best practices
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
 create_distributed_hypertable 
-------------------------------
 (22,public,test_1702,t)
(1 row)

-- Original issue case
--
-- Expect batch size to be lower than defined max_insert_batch_size
--
EXPLAIN (COSTS OFF) INSERT INTO test_1702(id, time) VALUES('1', current_timestamp);
                  QUERY PLAN                   
-----------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable test_1702
   ->  Insert on test_1702
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 910
               ->  Custom Scan (ChunkDispatch)
                     ->  Result
(7 rows)

INSERT INTO test_1702(id, time) VALUES('1', current_timestamp);
EXPLAIN (COSTS OFF) INSERT INTO test_1702(id, time) SELECT generate_series(2, 1500), current_timestamp;
                     QUERY PLAN                      
-----------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable test_1702
   ->  Insert on test_1702
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 910
               ->  Custom Scan (ChunkDispatch)
                     ->  Subquery Scan on "*SELECT*"
                           ->  ProjectSet
                                 ->  Result
(9 rows)

INSERT INTO test_1702(id, time) SELECT generate_series(2, 1500), current_timestamp;
SELECT count(*) from test_1702;
 count 
-------
  1500
(1 row)

DROP TABLE test_1702;
--
-- Expect batch size to be similair to max_insert_batch_size
--
CREATE TABLE test_1702 (
	id varchar(100) NOT NULL,
	time timestamp NOT NULL,
	dummy1	int	,
	dummy2	int	,
	dummy4	int	,
	dummy5	int
	);
SELECT create_distributed_hypertable('test_1702', 'time', 'id');
WARNING:  column type "character varying" used for "id" does not follow best practices
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
 create_distributed_hypertable 
-------------------------------
 (23,public,test_1702,t)
(1 row)

EXPLAIN (COSTS OFF) INSERT INTO test_1702(id, time) VALUES('1', current_timestamp);
                  QUERY PLAN                   
-----------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable test_1702
   ->  Insert on test_1702
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 1000
               ->  Custom Scan (ChunkDispatch)
                     ->  Result
(7 rows)

DROP TABLE test_1702;
--
-- Test that creating a hypertable with a space dimension and
-- if_not_exists works as expected, that is, the second call does not
-- generate an error (and does not crash).
--
CREATE TABLE whatever (
    timestamp TIMESTAMPTZ NOT NULL,
    user_id   INT         NOT NULL,
    data      JSONB
);
SELECT * FROM create_distributed_hypertable('whatever', 'timestamp', 'user_id',
                                            if_not_exists => true, chunk_time_interval => INTERVAL '1 day');
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            24 | public      | whatever   | t
(1 row)

-- Check the hypertable sequence before and after call to ensure that
-- the hypertable sequence was not increased with the second call.
SELECT last_value FROM _timescaledb_catalog.hypertable_id_seq;
 last_value 
------------
         24
(1 row)

SELECT * FROM create_distributed_hypertable('whatever', 'timestamp', 'user_id',
                                            if_not_exists => true, chunk_time_interval => INTERVAL '1 day');
NOTICE:  table "whatever" is already a hypertable, skipping
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            24 | public      | whatever   | f
(1 row)

SELECT last_value FROM _timescaledb_catalog.hypertable_id_seq;
 last_value 
------------
         24
(1 row)

-- Test that creating a distributed hypertable from a table with data
-- fails, and that migrate_data blocked.
CREATE TABLE dist_hypertable_1 (
  time TIMESTAMPTZ NOT NULL,
  device INTEGER,
  temp FLOAT
);
INSERT INTO dist_hypertable_1 VALUES
       ('2017-01-01 06:01', 1),
       ('2017-01-01 09:11', 3),
       ('2017-01-01 08:01', 1),
       ('2017-01-02 08:01', 2),
       ('2018-07-02 08:01', 87);
\set ON_ERROR_STOP 0
SELECT * FROM create_distributed_hypertable('dist_hypertable_1', 'time', 'device', 3,
       migrate_data => FALSE);
ERROR:  table "dist_hypertable_1" is not empty
SELECT * FROM create_distributed_hypertable('dist_hypertable_1', 'time', 'device', 3,
       migrate_data => TRUE);
ERROR:  cannot migrate data for distributed hypertable
\set ON_ERROR_STOP 1
-- Test creating index with transaction per chunk on a distributed hypertable
--
DROP TABLE disttable;
CREATE TABLE disttable(
    time timestamptz NOT NULL, 
    device int, 
    value float
);
SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 3);
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            25 | public      | disttable  | t
(1 row)

INSERT INTO disttable VALUES
       ('2017-01-01 06:01', 1, 1.2),
       ('2017-01-01 09:11', 3, 4.3),
       ('2017-01-01 08:01', 1, 7.3),
       ('2017-01-02 08:01', 2, 0.23),
       ('2018-07-02 08:01', 87, 0.0),
       ('2018-07-01 06:01', 13, 3.1),
       ('2018-07-01 09:11', 90, 10303.12),
       ('2018-07-01 08:01', 29, 64);
\set ON_ERROR_STOP 0
CREATE INDEX disttable_time_device_idx ON disttable (time, device) WITH (timescaledb.transaction_per_chunk);
ERROR:  cannot use timescaledb.transaction_per_chunk with distributed hypertable
\set ON_ERROR_STOP 1
-- Test using system columns with distributed hypertable
--
CREATE TABLE dist_syscol(time timestamptz NOT NULL, color int, temp float);
SELECT * FROM create_distributed_hypertable('dist_syscol', 'time', 'color');
 hypertable_id | schema_name | table_name  | created 
---------------+-------------+-------------+---------
            26 | public      | dist_syscol | t
(1 row)

INSERT INTO dist_syscol VALUES
	('2017-02-01 06:01', 1, 1.1),
	('2017-02-01 08:01', 1, 1.2),
	('2018-02-02 08:01', 2, 1.3),
	('2019-02-01 09:11', 3, 2.1),
	('2019-02-02 09:11', 3, 2.1),
	('2019-02-02 10:01', 5, 1.2),
	('2019-02-03 11:11', 6, 3.5),
	('2019-02-04 08:21', 4, 6.6),
	('2019-02-04 10:11', 7, 7.4),
	('2019-02-04 12:11', 8, 2.1),
	('2019-02-05 13:31', 8, 6.3),
	('2019-02-06 02:11', 5, 1.8),
	('2019-02-06 01:13', 7, 7.9),
	('2019-02-06 19:24', 9, 5.9),
	('2019-02-07 18:44', 5, 9.7),
	('2019-02-07 20:24', 6, NULL),
	('2019-02-07 09:33', 7, 9.5),
	('2019-02-08 08:54', 1, 7.3),
	('2019-02-08 18:14', 4, 8.2),
	('2019-02-09 19:23', 8, 9.1);
-- Return chunk table as a source
SET timescaledb.enable_per_data_node_queries = false;
SELECT tableoid::regclass, * FROM dist_syscol;
                   tableoid                    |             time             | color | temp 
-----------------------------------------------+------------------------------+-------+------
 _timescaledb_internal._dist_hyper_26_72_chunk | Wed Feb 01 06:01:00 2017 PST |     1 |  1.1
 _timescaledb_internal._dist_hyper_26_72_chunk | Wed Feb 01 08:01:00 2017 PST |     1 |  1.2
 _timescaledb_internal._dist_hyper_26_73_chunk | Fri Feb 02 08:01:00 2018 PST |     2 |  1.3
 _timescaledb_internal._dist_hyper_26_74_chunk | Fri Feb 01 09:11:00 2019 PST |     3 |  2.1
 _timescaledb_internal._dist_hyper_26_74_chunk | Sat Feb 02 09:11:00 2019 PST |     3 |  2.1
 _timescaledb_internal._dist_hyper_26_75_chunk | Sat Feb 02 10:01:00 2019 PST |     5 |  1.2
 _timescaledb_internal._dist_hyper_26_75_chunk | Mon Feb 04 08:21:00 2019 PST |     4 |  6.6
 _timescaledb_internal._dist_hyper_26_75_chunk | Mon Feb 04 10:11:00 2019 PST |     7 |  7.4
 _timescaledb_internal._dist_hyper_26_75_chunk | Wed Feb 06 02:11:00 2019 PST |     5 |  1.8
 _timescaledb_internal._dist_hyper_26_75_chunk | Wed Feb 06 01:13:00 2019 PST |     7 |  7.9
 _timescaledb_internal._dist_hyper_26_76_chunk | Sun Feb 03 11:11:00 2019 PST |     6 |  3.5
 _timescaledb_internal._dist_hyper_26_76_chunk | Mon Feb 04 12:11:00 2019 PST |     8 |  2.1
 _timescaledb_internal._dist_hyper_26_76_chunk | Tue Feb 05 13:31:00 2019 PST |     8 |  6.3
 _timescaledb_internal._dist_hyper_26_77_chunk | Wed Feb 06 19:24:00 2019 PST |     9 |  5.9
 _timescaledb_internal._dist_hyper_26_78_chunk | Thu Feb 07 18:44:00 2019 PST |     5 |  9.7
 _timescaledb_internal._dist_hyper_26_78_chunk | Thu Feb 07 09:33:00 2019 PST |     7 |  9.5
 _timescaledb_internal._dist_hyper_26_78_chunk | Fri Feb 08 18:14:00 2019 PST |     4 |  8.2
 _timescaledb_internal._dist_hyper_26_79_chunk | Thu Feb 07 20:24:00 2019 PST |     6 |     
 _timescaledb_internal._dist_hyper_26_79_chunk | Fri Feb 08 08:54:00 2019 PST |     1 |  7.3
 _timescaledb_internal._dist_hyper_26_79_chunk | Sat Feb 09 19:23:00 2019 PST |     8 |  9.1
(20 rows)

-- Produce an error
SET timescaledb.enable_per_data_node_queries = true;
\set ON_ERROR_STOP 0
SELECT tableoid::regclass, * FROM dist_syscol;
ERROR:  system columns are not accessible on distributed hypertables with current settings
\set ON_ERROR_STOP 1
-----------------------
-- Test DataNodeCopy --
-----------------------
SET timescaledb.enable_distributed_insert_with_copy=true;
DROP TABLE disttable;
-- Add serial (autoincrement) and DEFAULT value columns to test that
-- these work with DataNodeCopy
CREATE TABLE disttable(
    id serial,
    time timestamptz NOT NULL,
    device int DEFAULT 100,
    temp_c float
);
SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device');
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            27 | public      | disttable  | t
(1 row)

-- Create a datatable to source data from. Add array of composite data
-- type to test switching to text mode below. Arrays include the type
-- Oid when serialized in binary format. Since the Oid of a
-- user-created type can differ across data nodes, such serialization
-- is not safe.
CREATE TABLE datatable (LIKE disttable);
CREATE TYPE highlow AS (high int, low int);
CALL distributed_exec($$ CREATE TYPE highlow AS (high int, low int) $$);
ALTER TABLE datatable ADD COLUMN minmaxes highlow[];
INSERT INTO datatable (id, time, device, temp_c, minmaxes) VALUES
       (1, '2017-01-01 06:01', 1, 1.2, ARRAY[(1,2)::highlow]),
       (2, '2017-01-01 09:11', 3, 4.3, ARRAY[(2,3)::highlow]),
       (3, '2017-01-01 08:01', 1, 7.3, ARRAY[(4,5)::highlow]),
       (4, '2017-01-02 08:01', 2, 0.23, ARRAY[(6,7)::highlow]),
       (5, '2018-07-02 08:01', 87, 0.0, ARRAY[(8,9)::highlow]),
       (6, '2018-07-01 06:01', 13, 3.1, ARRAY[(10,11)::highlow]),
       (7, '2018-07-01 09:11', 90, 10303.12, ARRAY[(12,13)::highlow]),
       (8, '2018-07-01 08:01', 29, 64, ARRAY[(14,15)::highlow]);
-- Show that DataNodeCopy is used instead of DataNodeDispatch. Should
-- default to FORMAT binary in the remote SQL. Add RETURNING to show
-- that it works.
EXPLAIN VERBOSE
INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM datatable
RETURNING *;
                                                              QUERY PLAN                                                              
--------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)  (cost=0.00..24.55 rows=970 width=24)
   Output: disttable.id, disttable."time", disttable.device, disttable.temp_c
 Insert on distributed hypertable public.disttable
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.disttable  (cost=0.00..24.55 rows=970 width=24)
         Output: disttable.id, disttable."time", disttable.device, disttable.temp_c
         ->  Custom Scan (DataNodeCopy)  (cost=0.00..24.55 rows=970 width=24)
               Output: ((nextval('disttable_id_seq'::regclass))::integer), datatable."time", datatable.device, datatable.temp_c
               Remote SQL: COPY public.disttable (id, "time", device, temp_c) FROM STDIN WITH (FORMAT binary)
               ->  Custom Scan (ChunkDispatch)  (cost=0.00..24.55 rows=970 width=24)
                     Output: ((nextval('disttable_id_seq'::regclass))::integer), datatable."time", datatable.device, datatable.temp_c
                     ->  Seq Scan on public.datatable  (cost=0.00..24.55 rows=970 width=24)
                           Output: nextval('disttable_id_seq'::regclass), datatable."time", datatable.device, datatable.temp_c
(13 rows)

-- Perform the actual insert
INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM datatable
RETURNING *;
 id |             time             | device |  temp_c  
----+------------------------------+--------+----------
  1 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2
  2 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3
  3 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3
  4 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23
  5 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0
  6 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1
  7 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12
  8 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64
(8 rows)

-- Show that the data was added:
SELECT * FROM disttable ORDER BY 1;
 id |             time             | device |  temp_c  
----+------------------------------+--------+----------
  1 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2
  2 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3
  3 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3
  4 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23
  5 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0
  6 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1
  7 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12
  8 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64
(8 rows)

SELECT count(*) FROM disttable;
 count 
-------
     8
(1 row)

-- Add an array of a composite type to check that DataNodeCopy
-- switches to text format if we use a table with an array of a custom
-- type. There should be no "FORMAT binary" in the remote explain.
ALTER TABLE disttable ADD COLUMN minmaxes highlow[];
EXPLAIN VERBOSE
INSERT INTO disttable (time, device, temp_c, minmaxes)
SELECT time, device, temp_c, minmaxes FROM datatable;
                                                                        QUERY PLAN                                                                        
----------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)  (cost=0.00..24.55 rows=970 width=56)
 Insert on distributed hypertable public.disttable
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.disttable  (cost=0.00..24.55 rows=970 width=56)
         ->  Custom Scan (DataNodeCopy)  (cost=0.00..24.55 rows=970 width=56)
               Output: ((nextval('disttable_id_seq'::regclass))::integer), datatable."time", datatable.device, datatable.temp_c, datatable.minmaxes
               Remote SQL: COPY public.disttable (id, "time", device, temp_c, minmaxes) FROM STDIN
               ->  Custom Scan (ChunkDispatch)  (cost=0.00..24.55 rows=970 width=56)
                     Output: ((nextval('disttable_id_seq'::regclass))::integer), datatable."time", datatable.device, datatable.temp_c, datatable.minmaxes
                     ->  Seq Scan on public.datatable  (cost=0.00..24.55 rows=970 width=56)
                           Output: nextval('disttable_id_seq'::regclass), datatable."time", datatable.device, datatable.temp_c, datatable.minmaxes
(11 rows)

INSERT INTO disttable (time, device, temp_c, minmaxes)
SELECT time, device, temp_c, minmaxes FROM datatable;
-- Should have double amount of rows compared to before and half of
-- them values in the new column.  Note, must use TEXT format on the
-- connection to make query work with custom type array.
SET timescaledb.enable_connection_binary_data=false;
SELECT * FROM disttable ORDER BY 1;
 id |             time             | device |  temp_c  |  minmaxes   
----+------------------------------+--------+----------+-------------
  1 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | 
  2 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | 
  3 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | 
  4 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | 
  5 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | 
  6 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | 
  7 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | 
  8 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | 
  9 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | {"(1,2)"}
 10 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | {"(2,3)"}
 11 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | {"(4,5)"}
 12 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | {"(6,7)"}
 13 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | {"(8,9)"}
 14 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | {"(10,11)"}
 15 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | {"(12,13)"}
 16 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | {"(14,15)"}
(16 rows)

SELECT count(*) FROM disttable;
 count 
-------
    16
(1 row)

-- Binary format should lead to data incompatibility in PG 13 and earlier,
-- because the highlow data type has different oids on data and access nodes.
-- Use this to test the deserialization error reporting. Newer PG version
-- ignore this oid mismatch for non-builtin types.
SET timescaledb.enable_connection_binary_data=true;
\set ON_ERROR_STOP 0
SET timescaledb.remote_data_fetcher = 'copy';
SELECT * FROM disttable ORDER BY 1;
 id |             time             | device |  temp_c  |  minmaxes   
----+------------------------------+--------+----------+-------------
  1 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | 
  2 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | 
  3 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | 
  4 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | 
  5 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | 
  6 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | 
  7 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | 
  8 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | 
  9 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | {"(1,2)"}
 10 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | {"(2,3)"}
 11 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | {"(4,5)"}
 12 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | {"(6,7)"}
 13 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | {"(8,9)"}
 14 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | {"(10,11)"}
 15 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | {"(12,13)"}
 16 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | {"(14,15)"}
(16 rows)

SET timescaledb.remote_data_fetcher = 'cursor';
SELECT * FROM disttable ORDER BY 1;
 id |             time             | device |  temp_c  |  minmaxes   
----+------------------------------+--------+----------+-------------
  1 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | 
  2 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | 
  3 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | 
  4 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | 
  5 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | 
  6 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | 
  7 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | 
  8 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | 
  9 | Sun Jan 01 06:01:00 2017 PST |      1 |      1.2 | {"(1,2)"}
 10 | Sun Jan 01 09:11:00 2017 PST |      3 |      4.3 | {"(2,3)"}
 11 | Sun Jan 01 08:01:00 2017 PST |      1 |      7.3 | {"(4,5)"}
 12 | Mon Jan 02 08:01:00 2017 PST |      2 |     0.23 | {"(6,7)"}
 13 | Mon Jul 02 08:01:00 2018 PDT |     87 |        0 | {"(8,9)"}
 14 | Sun Jul 01 06:01:00 2018 PDT |     13 |      3.1 | {"(10,11)"}
 15 | Sun Jul 01 09:11:00 2018 PDT |     90 | 10303.12 | {"(12,13)"}
 16 | Sun Jul 01 08:01:00 2018 PDT |     29 |       64 | {"(14,15)"}
(16 rows)

\set ON_ERROR_STOP 1
RESET timescaledb.remote_data_fetcher;
-- Show that DataNodeCopy is NOT used when source hypertable and target hypertable
-- of the SELECT are both distributed. Try subselects with LIMIT, RETURNING and
-- different distributed hypertable as source
EXPLAIN (COSTS OFF)
INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM disttable;
                                       QUERY PLAN                                        
-----------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable disttable
   ->  Insert on disttable
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 1000
               ->  Custom Scan (ChunkDispatch)
                     ->  Append
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_2
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_3
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_4
(13 rows)

EXPLAIN (COSTS OFF)
INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM disttable LIMIT 1;
                                             QUERY PLAN                                              
-----------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable disttable
   ->  Insert on disttable
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 1000
               ->  Custom Scan (ChunkDispatch)
                     ->  Subquery Scan on "*SELECT*"
                           ->  Limit
                                 ->  Custom Scan (AsyncAppend)
                                       ->  Append
                                             ->  Custom Scan (DataNodeScan) on disttable disttable_2
                                             ->  Custom Scan (DataNodeScan) on disttable disttable_3
                                             ->  Custom Scan (DataNodeScan) on disttable disttable_4
(13 rows)

EXPLAIN (COSTS OFF)
INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM disttable RETURNING *;
                                       QUERY PLAN                                        
-----------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)
 Insert on distributed hypertable disttable
   ->  Insert on disttable
         ->  Custom Scan (DataNodeDispatch)
               Batch size: 1000
               ->  Custom Scan (ChunkDispatch)
                     ->  Append
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_2
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_3
                           ->  Result
                                 ->  Custom Scan (DataNodeScan) on disttable disttable_4
(13 rows)

INSERT INTO disttable (time, device, temp_c)
SELECT time, device, temp_c FROM disttable;
INSERT INTO disttable (time, device, temp_c)
SELECT * FROM hyper_estimate LIMIT 2;
SELECT count(*) FROM disttable;
 count 
-------
    34
(1 row)

-- REMOVE a column on data nodes to check how errors are handled:
CALL distributed_exec($$ ALTER TABLE disttable DROP COLUMN minmaxes $$);
\set ON_ERROR_STOP 0
INSERT INTO disttable SELECT * FROM datatable;
ERROR:  [db_dist_hypertable_1]: column "minmaxes" of relation "disttable" does not exist
\set ON_ERROR_STOP 1
DROP TABLE disttable;
-- Create a new table access method by reusing heap handler
CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler;
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$
CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler;
$$);
NOTICE:  [db_dist_hypertable_1]: 
CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler
NOTICE:  [db_dist_hypertable_2]: 
CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler
NOTICE:  [db_dist_hypertable_3]: 
CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler
 remote_exec 
-------------
 
(1 row)

-- Create distributed hypertable using non-default access method
CREATE TABLE disttable(time timestamptz NOT NULL, device int, temp_c float, temp_f float GENERATED ALWAYS AS (temp_c * 9 / 5 + 32) STORED) USING test_am;
SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 3);
 hypertable_id | schema_name | table_name | created 
---------------+-------------+------------+---------
            28 | public      | disttable  | t
(1 row)

-- Make sure that distributed hypertable created on data nodes is
-- using the correct table access method
SELECT * FROM test.remote_exec(ARRAY[:'DATA_NODE_1', :'DATA_NODE_2', :'DATA_NODE_3'], $$

SELECT amname AS hypertable_amname
FROM pg_class cl, pg_am am
WHERE cl.oid = 'disttable'::regclass
AND cl.relam = am.oid;
$$);
NOTICE:  [db_dist_hypertable_1]: 

SELECT amname AS hypertable_amname
FROM pg_class cl, pg_am am
WHERE cl.oid = 'disttable'::regclass
AND cl.relam = am.oid
NOTICE:  [db_dist_hypertable_1]:
hypertable_amname
-----------------
test_am          
(1 row)


NOTICE:  [db_dist_hypertable_2]: 

SELECT amname AS hypertable_amname
FROM pg_class cl, pg_am am
WHERE cl.oid = 'disttable'::regclass
AND cl.relam = am.oid
NOTICE:  [db_dist_hypertable_2]:
hypertable_amname
-----------------
test_am          
(1 row)


NOTICE:  [db_dist_hypertable_3]: 

SELECT amname AS hypertable_amname
FROM pg_class cl, pg_am am
WHERE cl.oid = 'disttable'::regclass
AND cl.relam = am.oid
NOTICE:  [db_dist_hypertable_3]:
hypertable_amname
-----------------
test_am          
(1 row)


 remote_exec 
-------------
 
(1 row)

-- Check that basic operations are working as expected
INSERT INTO disttable VALUES
       ('2017-01-01 06:01', 1, -10.0),
       ('2017-01-01 09:11', 3, -5.0),
       ('2017-01-01 08:01', 1, 1.0),
       ('2017-01-02 08:01', 2, 5.0),
       ('2018-07-02 08:01', 87, 10.0),
       ('2018-07-01 06:01', 13, 15.0),
       ('2018-07-01 09:11', 90, 20.0),
       ('2018-07-01 08:01', 29, 24.0);
SELECT * FROM disttable ORDER BY time;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Sun Jan 01 06:01:00 2017 PST |      1 |    -10 |     14
 Sun Jan 01 08:01:00 2017 PST |      1 |      1 |   33.8
 Sun Jan 01 09:11:00 2017 PST |      3 |     -5 |     23
 Mon Jan 02 08:01:00 2017 PST |      2 |      5 |     41
 Sun Jul 01 06:01:00 2018 PDT |     13 |     15 |     59
 Sun Jul 01 08:01:00 2018 PDT |     29 |     24 |   75.2
 Sun Jul 01 09:11:00 2018 PDT |     90 |     20 |     68
 Mon Jul 02 08:01:00 2018 PDT |     87 |     10 |     50
(8 rows)

-- Show that GENERATED columns work for INSERT with RETURNING clause
-- (should use DataNodeCopy)
TRUNCATE disttable;
EXPLAIN VERBOSE
INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *;
                                                                  QUERY PLAN                                                                   
-----------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)  (cost=0.00..0.01 rows=1 width=28)
   Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f
 Insert on distributed hypertable public.disttable
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.disttable  (cost=0.00..0.01 rows=1 width=28)
         Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f
         ->  Custom Scan (DataNodeCopy)  (cost=0.00..0.01 rows=1 width=28)
               Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision
               Remote SQL: COPY public.disttable ("time", device, temp_c) FROM STDIN WITH (FORMAT binary)
               ->  Custom Scan (ChunkDispatch)  (cost=0.00..0.01 rows=1 width=28)
                     Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision
                     ->  Result  (cost=0.00..0.01 rows=1 width=28)
                           Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision
(13 rows)

INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Tue Aug 01 06:01:00 2017 PDT |      1 |     35 |     95
(1 row)

-- Same values returned with SELECT:
SELECT * FROM disttable ORDER BY 1;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Tue Aug 01 06:01:00 2017 PDT |      1 |     35 |     95
(1 row)

UPDATE disttable SET temp_c=40.0 WHERE device=1;
SELECT * FROM disttable ORDER BY 1;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Tue Aug 01 06:01:00 2017 PDT |      1 |     40 |    104
(1 row)

-- Insert another value
INSERT INTO disttable VALUES ('2017-09-01 06:01', 2, 30.0);
SELECT * FROM disttable ORDER BY 1;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Tue Aug 01 06:01:00 2017 PDT |      1 |     40 |    104
 Fri Sep 01 06:01:00 2017 PDT |      2 |     30 |     86
(2 rows)

-- Delete a value based on the generated column
DELETE FROM disttable WHERE temp_f=104;
SELECT * FROM disttable ORDER BY 1;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Fri Sep 01 06:01:00 2017 PDT |      2 |     30 |     86
(1 row)

-- Test also with DataNodeDispatch
TRUNCATE disttable;
SET timescaledb.enable_distributed_insert_with_copy=false;
EXPLAIN VERBOSE
INSERT INTO disttable VALUES ('2017-09-01 06:01', 5, 40.0) RETURNING *;
                                                                               QUERY PLAN                                                                                
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (HypertableModify)  (cost=0.00..0.01 rows=1 width=28)
   Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f
 Insert on distributed hypertable public.disttable
   Data nodes: db_dist_hypertable_1, db_dist_hypertable_2, db_dist_hypertable_3
   ->  Insert on public.disttable  (cost=0.00..0.01 rows=1 width=28)
         Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f
         ->  Custom Scan (DataNodeDispatch)  (cost=0.00..0.01 rows=1 width=28)
               Output: 'Fri Sep 01 06:01:00 2017 PDT'::timestamp with time zone, 5, '40'::double precision, NULL::double precision
               Batch size: 1000
               Remote SQL: INSERT INTO public.disttable("time", device, temp_c) VALUES ($1, $2, $3), ..., ($2998, $2999, $3000) RETURNING "time", device, temp_c, temp_f
               ->  Custom Scan (ChunkDispatch)  (cost=0.00..0.01 rows=1 width=28)
                     Output: 'Fri Sep 01 06:01:00 2017 PDT'::timestamp with time zone, 5, '40'::double precision, NULL::double precision
                     ->  Result  (cost=0.00..0.01 rows=1 width=28)
                           Output: 'Fri Sep 01 06:01:00 2017 PDT'::timestamp with time zone, 5, '40'::double precision, NULL::double precision
(14 rows)

INSERT INTO disttable VALUES ('2017-09-01 06:01', 5, 40.0) RETURNING *;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Fri Sep 01 06:01:00 2017 PDT |      5 |     40 |    104
(1 row)

-- Generated columns with SELECT
SELECT * FROM disttable ORDER BY 1;
             time             | device | temp_c | temp_f 
------------------------------+--------+--------+--------
 Fri Sep 01 06:01:00 2017 PDT |      5 |     40 |    104
(1 row)

-- Check distributed hypertable within procedure properly drops remote tables
--
-- #3663 
--
CREATE TABLE test (time timestamp, v int);
SELECT create_distributed_hypertable('test','time');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (29,public,test,t)
(1 row)

CREATE PROCEDURE test_drop() LANGUAGE PLPGSQL AS $$
BEGIN
    DROP TABLE test;
END
$$;
CALL test_drop();
CREATE TABLE test (time timestamp, v int);
SELECT create_distributed_hypertable('test','time');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (30,public,test,t)
(1 row)

DROP TABLE test;
-- Test that stable functions are calculated on the access node.
--
-- As a stable function to test, use the timestamp -> timestamptz conversion
-- that is stable because it uses the current timezone.
-- We have to be careful about `timestamp < timestamptz` comparison. From postgres
-- docs:
--  When comparing a timestamp without time zone to a timestamp with time zone,
--  the former value is assumed to be given in the time zone specified by the
--  TimeZone configuration parameter, and is rotated to UTC for comparison to
--  the latter value (which is already in UTC internally).
-- We don't want this to happen on data node, so we cast the filter value to
-- timestamp, and check that this cast happens on the access node and uses the
-- current timezone.
SELECT test.tsl_override_current_timestamptz(null);
 tsl_override_current_timestamptz 
----------------------------------
 
(1 row)

CREATE TABLE test_tz (time timestamp, v int);
SELECT create_distributed_hypertable('test_tz','time',
	chunk_time_interval => interval '1 hour');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (31,public,test_tz,t)
(1 row)

INSERT INTO test_tz VALUES ('2018-01-02 12:00:00', 2), ('2018-01-02 11:00:00', 1),
	('2018-01-02 13:00:00', 3), ('2018-01-02 14:00:00', 4);
SET TIME ZONE 'Etc/GMT';
SELECT '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
        timestamp         
--------------------------
 Tue Jan 02 12:00:00 2018
(1 row)

-- Normal WHERE clause on baserel
SELECT * FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
           time           | v 
--------------------------+---
 Tue Jan 02 13:00:00 2018 | 3
 Tue Jan 02 14:00:00 2018 | 4
(2 rows)

EXPLAIN (verbose, costs off)
SELECT * FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
                                                                                                  QUERY PLAN                                                                                                   
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: test_tz."time", test_tz.v
   ->  Append
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
               Output: test_tz_1."time", test_tz_1.v
               Data node: db_dist_hypertable_1
               Chunks: _dist_hyper_31_97_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
               Remote EXPLAIN: 
                 Index Scan using _dist_hyper_31_97_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_97_chunk
                   Output: _dist_hyper_31_97_chunk."time", _dist_hyper_31_97_chunk.v
                   Index Cond: (_dist_hyper_31_97_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
               Output: test_tz_2."time", test_tz_2.v
               Data node: db_dist_hypertable_2
               Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
               Remote EXPLAIN: 
                 Append
                   ->  Index Scan using _dist_hyper_31_95_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_95_chunk
                         Output: _dist_hyper_31_95_chunk."time", _dist_hyper_31_95_chunk.v
                         Index Cond: (_dist_hyper_31_95_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
                   ->  Index Scan using _dist_hyper_31_98_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_98_chunk
                         Output: _dist_hyper_31_98_chunk."time", _dist_hyper_31_98_chunk.v
                         Index Cond: (_dist_hyper_31_98_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
(27 rows)

-- Also test different code paths used with aggregation pushdown.
SELECT count(*) FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
 count 
-------
     2
(1 row)

EXPLAIN (verbose, costs off)
SELECT count(*) FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
                                                                                                   QUERY PLAN                                                                                                   
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Aggregate
   Output: count(*)
   ->  Custom Scan (AsyncAppend)
         ->  Append
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
                     Data node: db_dist_hypertable_1
                     Chunks: _dist_hyper_31_97_chunk
                     Remote SQL: SELECT NULL FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Result
                         Output: NULL::text
                         ->  Index Only Scan using _dist_hyper_31_97_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_97_chunk
                               Index Cond: (_dist_hyper_31_97_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
                     Data node: db_dist_hypertable_2
                     Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
                     Remote SQL: SELECT NULL FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Result
                         Output: NULL::text
                         ->  Append
                               ->  Index Only Scan using _dist_hyper_31_95_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_95_chunk
                                     Index Cond: (_dist_hyper_31_95_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
                               ->  Index Only Scan using _dist_hyper_31_98_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_98_chunk
                                     Index Cond: (_dist_hyper_31_98_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
(27 rows)

-- TODO: test HAVING here and in the later now() tests as well.
-- Change the timezone and check that the conversion is applied correctly.
SET TIME ZONE 'Etc/GMT+1';
SELECT '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
        timestamp         
--------------------------
 Tue Jan 02 11:00:00 2018
(1 row)

SELECT * FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
           time           | v 
--------------------------+---
 Tue Jan 02 13:00:00 2018 | 3
 Tue Jan 02 12:00:00 2018 | 2
 Tue Jan 02 14:00:00 2018 | 4
(3 rows)

EXPLAIN (verbose, costs off)
SELECT * FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
                                                                                                  QUERY PLAN                                                                                                   
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: test_tz."time", test_tz.v
   ->  Append
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
               Output: test_tz_1."time", test_tz_1.v
               Data node: db_dist_hypertable_1
               Chunks: _dist_hyper_31_97_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time" > '2018-01-02 11:00:00'::timestamp without time zone))
               Remote EXPLAIN: 
                 Index Scan using _dist_hyper_31_97_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_97_chunk
                   Output: _dist_hyper_31_97_chunk."time", _dist_hyper_31_97_chunk.v
                   Index Cond: (_dist_hyper_31_97_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
 
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
               Output: test_tz_2."time", test_tz_2.v
               Data node: db_dist_hypertable_2
               Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time" > '2018-01-02 11:00:00'::timestamp without time zone))
               Remote EXPLAIN: 
                 Append
                   ->  Index Scan using _dist_hyper_31_95_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_95_chunk
                         Output: _dist_hyper_31_95_chunk."time", _dist_hyper_31_95_chunk.v
                         Index Cond: (_dist_hyper_31_95_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
                   ->  Index Scan using _dist_hyper_31_96_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_96_chunk
                         Output: _dist_hyper_31_96_chunk."time", _dist_hyper_31_96_chunk.v
                         Index Cond: (_dist_hyper_31_96_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
                   ->  Index Scan using _dist_hyper_31_98_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_98_chunk
                         Output: _dist_hyper_31_98_chunk."time", _dist_hyper_31_98_chunk.v
                         Index Cond: (_dist_hyper_31_98_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
 
(30 rows)

SELECT count(*) FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
 count 
-------
     3
(1 row)

EXPLAIN (verbose, costs off)
SELECT count(*) FROM test_tz WHERE time > '2018-01-02 12:00:00 +00'::timestamptz::timestamp;
                                                                                                   QUERY PLAN                                                                                                   
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Aggregate
   Output: count(*)
   ->  Custom Scan (AsyncAppend)
         ->  Append
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
                     Data node: db_dist_hypertable_1
                     Chunks: _dist_hyper_31_97_chunk
                     Remote SQL: SELECT NULL FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time" > '2018-01-02 11:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Result
                         Output: NULL::text
                         ->  Index Only Scan using _dist_hyper_31_97_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_97_chunk
                               Index Cond: (_dist_hyper_31_97_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
 
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
                     Data node: db_dist_hypertable_2
                     Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
                     Remote SQL: SELECT NULL FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time" > '2018-01-02 11:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Result
                         Output: NULL::text
                         ->  Append
                               ->  Index Only Scan using _dist_hyper_31_95_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_95_chunk
                                     Index Cond: (_dist_hyper_31_95_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
                               ->  Index Only Scan using _dist_hyper_31_96_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_96_chunk
                                     Index Cond: (_dist_hyper_31_96_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
                               ->  Index Only Scan using _dist_hyper_31_98_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_98_chunk
                                     Index Cond: (_dist_hyper_31_98_chunk."time" > '2018-01-02 11:00:00'::timestamp without time zone)
 
(29 rows)

-- Conversion to timestamptz cannot be evaluated at the access node, because the
-- argument is a column reference.
SELECT count(*) FROM test_tz WHERE time::timestamptz > now();
 count 
-------
     0
(1 row)

-- According to our docs, JIT is not recommended for use on access node in
-- multi-node environment. Turn it off so that it doesn't ruin EXPLAIN for the
-- next query.
SET jit = 0;
-- Test that operators are evaluated as well. Comparison of timestamp with
-- timestamptz is a stable operator, and comparison of two timestamps is an
-- immutable operator. This also test that immutable functions using these
-- operators as arguments are evaluated.
EXPLAIN (verbose, costs off)
WITH dummy AS (SELECT '2018-01-02 12:00:00 +00'::timestamptz::timestamp x)
SELECT * FROM test_tz, dummy
WHERE time > x
	+ (x = x)::int -- stable
		* (x = '2018-01-02 11:00:00'::timestamp)::int -- immutable
		* INTERVAL '1 hour';
                                                                                                     QUERY PLAN                                                                                                      
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: test_tz."time", test_tz.v, (('Tue Jan 02 11:00:00 2018 -01'::timestamp with time zone)::timestamp without time zone)
   ->  Append
         ->  Result
               Output: test_tz_1."time", test_tz_1.v, ('Tue Jan 02 11:00:00 2018 -01'::timestamp with time zone)::timestamp without time zone
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
                     Output: test_tz_1."time", test_tz_1.v
                     Data node: db_dist_hypertable_1
                     Chunks: _dist_hyper_31_97_chunk
                     Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Index Scan using _dist_hyper_31_97_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_97_chunk
                         Output: _dist_hyper_31_97_chunk."time", _dist_hyper_31_97_chunk.v
                         Index Cond: (_dist_hyper_31_97_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
         ->  Result
               Output: test_tz_2."time", test_tz_2.v, ('Tue Jan 02 11:00:00 2018 -01'::timestamp with time zone)::timestamp without time zone
               ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
                     Output: test_tz_2."time", test_tz_2.v
                     Data node: db_dist_hypertable_2
                     Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
                     Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time" > '2018-01-02 12:00:00'::timestamp without time zone))
                     Remote EXPLAIN: 
                       Append
                         ->  Index Scan using _dist_hyper_31_95_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_95_chunk
                               Output: _dist_hyper_31_95_chunk."time", _dist_hyper_31_95_chunk.v
                               Index Cond: (_dist_hyper_31_95_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
                         ->  Index Scan using _dist_hyper_31_98_chunk_test_tz_time_idx on _timescaledb_internal._dist_hyper_31_98_chunk
                               Output: _dist_hyper_31_98_chunk."time", _dist_hyper_31_98_chunk.v
                               Index Cond: (_dist_hyper_31_98_chunk."time" > '2018-01-02 12:00:00'::timestamp without time zone)
 
(31 rows)

-- Reference value for the above test.
WITH dummy AS (SELECT '2018-01-02 12:00:00 +00'::timestamptz::timestamp x)
SELECT x + (x = x)::int * (x = '2018-01-02 11:00:00'::timestamp)::int * INTERVAL '1 hour'
FROM dummy;
         ?column?         
--------------------------
 Tue Jan 02 12:00:00 2018
(1 row)

-- Exercise some more stable timestamp-related functions.
EXPLAIN (COSTS OFF, VERBOSE)
SELECT * FROM test_tz WHERE date_trunc('month', time) > date_in('2021-01-01')
	AND time::time > '00:00:00'::time
		+ (INTERVAL '1 hour') * date_part('hour', INTERVAL '1 hour');
                                                                                                                                      QUERY PLAN                                                                                                                                      
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 Custom Scan (AsyncAppend)
   Output: test_tz."time", test_tz.v
   ->  Append
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_1
               Output: test_tz_1."time", test_tz_1.v
               Data node: db_dist_hypertable_1
               Chunks: _dist_hyper_31_97_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[45]) AND (("time"::time without time zone > '01:00:00'::time without time zone)) AND ((date_trunc('month'::text, "time") > '2021-01-01'::date))
               Remote EXPLAIN: 
                 Seq Scan on _timescaledb_internal._dist_hyper_31_97_chunk
                   Output: _dist_hyper_31_97_chunk."time", _dist_hyper_31_97_chunk.v
                   Filter: (((_dist_hyper_31_97_chunk."time")::time without time zone > '01:00:00'::time without time zone) AND (date_trunc('month'::text, _dist_hyper_31_97_chunk."time") > '2021-01-01'::date))
 
         ->  Custom Scan (DataNodeScan) on public.test_tz test_tz_2
               Output: test_tz_2."time", test_tz_2.v
               Data node: db_dist_hypertable_2
               Chunks: _dist_hyper_31_95_chunk, _dist_hyper_31_96_chunk, _dist_hyper_31_98_chunk
               Remote SQL: SELECT "time", v FROM public.test_tz WHERE _timescaledb_internal.chunks_in(public.test_tz.*, ARRAY[43, 44, 45]) AND (("time"::time without time zone > '01:00:00'::time without time zone)) AND ((date_trunc('month'::text, "time") > '2021-01-01'::date))
               Remote EXPLAIN: 
                 Append
                   ->  Seq Scan on _timescaledb_internal._dist_hyper_31_95_chunk
                         Output: _dist_hyper_31_95_chunk."time", _dist_hyper_31_95_chunk.v
                         Filter: (((_dist_hyper_31_95_chunk."time")::time without time zone > '01:00:00'::time without time zone) AND (date_trunc('month'::text, _dist_hyper_31_95_chunk."time") > '2021-01-01'::date))
                   ->  Seq Scan on _timescaledb_internal._dist_hyper_31_96_chunk
                         Output: _dist_hyper_31_96_chunk."time", _dist_hyper_31_96_chunk.v
                         Filter: (((_dist_hyper_31_96_chunk."time")::time without time zone > '01:00:00'::time without time zone) AND (date_trunc('month'::text, _dist_hyper_31_96_chunk."time") > '2021-01-01'::date))
                   ->  Seq Scan on _timescaledb_internal._dist_hyper_31_98_chunk
                         Output: _dist_hyper_31_98_chunk."time", _dist_hyper_31_98_chunk.v
                         Filter: (((_dist_hyper_31_98_chunk."time")::time without time zone > '01:00:00'::time without time zone) AND (date_trunc('month'::text, _dist_hyper_31_98_chunk."time") > '2021-01-01'::date))
 
(30 rows)

-- Check that the test function for partly overriding now() works. It's very
-- hacky and only has effect when we estimate some costs or evaluate sTABLE
-- functions in quals on access node, and has no effect in other cases.
-- Consider deleting it altogether.
SELECT test.tsl_override_current_timestamptz('2018-01-02 12:00:00 +00'::timestamptz);
 tsl_override_current_timestamptz 
----------------------------------
 
(1 row)

SELECT count(*) FROM test_tz WHERE time > now();
 count 
-------
     3
(1 row)

SELECT test.tsl_override_current_timestamptz(null);
 tsl_override_current_timestamptz 
----------------------------------
 
(1 row)

RESET TIME ZONE;
DROP TABLE test_tz;
-- Check that now() is evaluated on the access node. Also check that it is evaluated
-- anew on every execution of a prepared statement.
CREATE TABLE test_now (time timestamp, v int);
SELECT create_distributed_hypertable('test_now','time',
	chunk_time_interval => interval '1 hour');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (32,public,test_now,t)
(1 row)

PREPARE test_query as
SELECT count(*) FILTER (WHERE time = now()), count(*) FILTER (WHERE time < now()),
	count(*) FILTER (WHERE time > now()) FROM test_now;
;
BEGIN; -- to fix the value of now();
INSERT INTO test_now VALUES
	(now(), 1), (now() + INTERVAL '1 hour', 1),
	(now() + INTERVAL '2 hour', 2 ), (now() + INTERVAL '3 hour', 3);
SELECT count(*) FILTER (WHERE time = now()), count(*) FILTER (WHERE time < now()),
	count(*) FILTER (WHERE time > now()) FROM test_now;
 count | count | count 
-------+-------+-------
     1 |     0 |     3
(1 row)

EXECUTE test_query;
 count | count | count 
-------+-------+-------
     1 |     0 |     3
(1 row)

-- Also test different code paths used with aggregation pushdown.
-- We can't run EXPLAIN here, because now() is different every time. But the
-- strict equality should be enough to detect if now() is being erroneously
-- evaluated on data node, where it will differ from time to time.
SELECT count(*) FROM test_now WHERE time = now();
 count 
-------
     1
(1 row)

COMMIT;
-- now() will be different in a new transaction.
BEGIN;
SELECT count(*) FILTER (WHERE time = now()), count(*) FILTER (WHERE time < now()),
	count(*) FILTER (WHERE time > now()) FROM test_now;
 count | count | count 
-------+-------+-------
     0 |     1 |     3
(1 row)

EXECUTE test_query;
 count | count | count 
-------+-------+-------
     0 |     1 |     3
(1 row)

SELECT count(*) FROM test_now WHERE time = now();
 count 
-------
     0
(1 row)

COMMIT;
DROP TABLE test_now;
DEALLOCATE test_query;
-- Check enabling distributed compression within a
-- procedure/function works
--
-- #3705
--
CREATE TABLE test (time timestamp, v int);
SELECT create_distributed_hypertable('test','time');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (33,public,test,t)
(1 row)

CREATE PROCEDURE test_set_compression() LANGUAGE PLPGSQL AS $$
BEGIN
	ALTER TABLE test SET (timescaledb.compress);
END
$$;
CALL test_set_compression();
INSERT INTO test VALUES (now(), 0);
SELECT compress_chunk(show_chunks) FROM show_chunks('test');
                 compress_chunk                 
------------------------------------------------
 _timescaledb_internal._dist_hyper_33_103_chunk
(1 row)

DROP TABLE test;
CREATE TABLE test (time timestamp, v int);
SELECT create_distributed_hypertable('test','time');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
NOTICE:  adding not-null constraint to column "time"
 create_distributed_hypertable 
-------------------------------
 (34,public,test,t)
(1 row)

CREATE FUNCTION test_set_compression_func() RETURNS BOOL LANGUAGE PLPGSQL AS $$
BEGIN
	ALTER TABLE test SET (timescaledb.compress);
	RETURN TRUE;
END
$$;
SELECT test_set_compression_func();
 test_set_compression_func 
---------------------------
 t
(1 row)

INSERT INTO test VALUES (now(), 0);
SELECT compress_chunk(show_chunks) FROM show_chunks('test');
                 compress_chunk                 
------------------------------------------------
 _timescaledb_internal._dist_hyper_34_104_chunk
(1 row)

DROP TABLE test;
-- Fix ALTER SET/DROP NULL constraint on distributed hypertable
--
-- #3860
--
CREATE TABLE test (time timestamp NOT NULL, my_column int NOT NULL);
SELECT create_distributed_hypertable('test','time');
WARNING:  column type "timestamp without time zone" used for "time" does not follow best practices
 create_distributed_hypertable 
-------------------------------
 (35,public,test,t)
(1 row)

\set ON_ERROR_STOP 0
INSERT INTO test VALUES (now(), NULL);
ERROR:  [db_dist_hypertable_3]: null value in column "my_column" of relation "_dist_hyper_35_105_chunk" violates not-null constraint
\set ON_ERROR_STOP 1
ALTER TABLE test ALTER COLUMN my_column DROP NOT NULL;
INSERT INTO test VALUES (now(), NULL);
\set ON_ERROR_STOP 0
ALTER TABLE test ALTER COLUMN my_column SET NOT NULL;
ERROR:  [db_dist_hypertable_3]: column "my_column" of relation "_dist_hyper_35_106_chunk" contains null values
\set ON_ERROR_STOP 1
DELETE FROM test;
ALTER TABLE test ALTER COLUMN my_column SET NOT NULL;
DROP TABLE test;
-- Test insert into distributed hypertable with pruned chunks
CREATE TABLE pruned_chunks_1(time TIMESTAMPTZ NOT NULL, sensor_id INTEGER, value FLOAT);
SELECT table_name FROM create_distributed_hypertable('pruned_chunks_1', 'time', 'sensor_id');
   table_name    
-----------------
 pruned_chunks_1
(1 row)

INSERT INTO pruned_chunks_1 VALUES  ('2020-12-09',1,32.2);
CREATE TABLE pruned_chunks_2(time TIMESTAMPTZ NOT NULL, sensor_id INTEGER, value FLOAT);
-- Convert the table to a distributed hypertable
SELECT table_name FROM create_distributed_hypertable('pruned_chunks_2', 'time', 'sensor_id');
   table_name    
-----------------
 pruned_chunks_2
(1 row)

insert into pruned_chunks_2 select * from pruned_chunks_1;
insert into pruned_chunks_2 select * from pruned_chunks_1 WHERE time > '2022-01-01';
-- TEST freeze_chunk api. does not work for distributed chunks
SELECT chunk_schema || '.' ||  chunk_name as "CHNAME"
FROM timescaledb_information.chunks
WHERE hypertable_name = 'pruned_chunks_1' 
ORDER BY chunk_name LIMIT 1
\gset
\set ON_ERROR_STOP 0
SELECT  _timescaledb_internal.freeze_chunk( :'CHNAME');
ERROR:  operation not supported on distributed chunk or foreign table "_dist_hyper_36_107_chunk"
\set ON_ERROR_STOP 1
--TEST freeze_chunk api for regular hypertables. Works only for >= PG14
CREATE TABLE freeze_1(time TIMESTAMPTZ NOT NULL, sensor_id INTEGER, value FLOAT);
SELECT table_name FROM create_hypertable('freeze_1', 'time');
 table_name 
------------
 freeze_1
(1 row)

INSERT INTO freeze_1 VALUES  ('2020-12-09',1,32.2);
\set ON_ERROR_STOP 0
SELECT  _timescaledb_internal.freeze_chunk( ch) FROM ( select show_chunks('freeze_1') ch ) q;
 freeze_chunk 
--------------
 t
(1 row)

\set ON_ERROR_STOP 1
DROP TABLE pruned_chunks_1;
DROP TABLE pruned_chunks_2;
-- Cleanup
DROP DATABASE :DATA_NODE_1;
DROP DATABASE :DATA_NODE_2;
DROP DATABASE :DATA_NODE_3;