Skip to content
Permalink
Browse files
Implement show_chunks in C and have drop_chunks use it
Timescale provides an efficient and easy to use api to drop individual
chunks from timescale database through drop_chunks. This PR builds on
that functionality and through a new show_chunks function gives the
opportunity to see the chunks that would be dropped if drop_chunks was run.
Additionally, it adds a newer_than option to drop_chunks (also supported
by show_chunks) that allows to see/drop chunks in an interval or newer
than a point in time.

This commit includes:
    - Implementation of show_chunks in C
    - Additional helper functions to work with chunks
    - New version of drop_chunks in sql that uses show_chunks. This
      	  also adds a newer_than option to drop_chunks
    - More enhanced tests of drop_chunks and new tests for show_chunks

Among other reasons, show_chunks was implemented in C in order
to be able to have both older_than and newer_than arguments be null. This
was not possible in SQL because the arguments had to have polymorphic types
and whether they are used in function body or not, PL/pgSQL requires these
arguments to typecheck.
  • Loading branch information
Ngalstyan4 authored and cevian committed Nov 28, 2018
1 parent d461959 commit 9a3402809f05e5ac9830797106005dcd127d998e
Show file tree
Hide file tree
Showing 24 changed files with 1,214 additions and 161 deletions.
@@ -83,6 +83,7 @@ set(MOD_FILES
updates/1.0.0-rc2--1.0.0-rc3.sql
updates/1.0.0-rc3--1.0.0.sql
updates/1.0.0--1.0.1-dev.sql
updates/1.0.1--1.1.0.sql
)

set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}")
@@ -27,7 +27,7 @@ CREATE OR REPLACE FUNCTION create_hypertable(
number_partitions INTEGER = NULL,
associated_schema_name NAME = NULL,
associated_table_prefix NAME = NULL,
chunk_time_interval anyelement = NULL::bigint,
chunk_time_interval ANYELEMENT = NULL::bigint,
create_default_indexes BOOLEAN = TRUE,
if_not_exists BOOLEAN = FALSE,
partitioning_func REGPROC = NULL,
@@ -64,69 +64,43 @@ CREATE OR REPLACE FUNCTION set_number_partitions(
dimension_name NAME = NULL
) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_set_num_slices' LANGUAGE C VOLATILE;

-- Drop chunks that are older than a timestamp.
-- Drop chunks that are older than a timestamp or newer than a timestamp.
CREATE OR REPLACE FUNCTION drop_chunks(
older_than anyelement,
older_than ANYELEMENT = NULL,
table_name NAME = NULL,
schema_name NAME = NULL,
cascade BOOLEAN = FALSE
cascade BOOLEAN = FALSE,
newer_than ANYELEMENT = NULL
)
RETURNS VOID LANGUAGE PLPGSQL VOLATILE AS
$BODY$
DECLARE
older_than_internal BIGINT;
newer_than_internal BIGINT;
BEGIN
IF older_than IS NULL THEN
RAISE 'the timestamp provided to drop_chunks cannot be NULL';
IF older_than IS NULL AND newer_than IS NULL THEN
RAISE 'older_than and newer_than timestamps provided to drop_chunks cannot both be NULL';
END IF;

PERFORM _timescaledb_internal.drop_chunks_type_check(pg_typeof(older_than), table_name, schema_name);
SELECT _timescaledb_internal.time_to_internal(older_than) INTO older_than_internal;
PERFORM _timescaledb_internal.drop_chunks_impl(older_than_internal, table_name, schema_name, cascade);
PERFORM _timescaledb_internal.drop_chunks_impl(older_than, table_name, schema_name, cascade,
newer_than_time => newer_than);
END
$BODY$;

-- Drop chunks older than an interval.
CREATE OR REPLACE FUNCTION drop_chunks(
older_than INTERVAL,
table_name NAME = NULL,
schema_name NAME = NULL,
cascade BOOLEAN = false
)
RETURNS VOID LANGUAGE PLPGSQL VOLATILE AS
$BODY$
DECLARE
time_type REGTYPE;
BEGIN

BEGIN
WITH hypertable_ids AS (
SELECT id
FROM _timescaledb_catalog.hypertable h
WHERE (drop_chunks.schema_name IS NULL OR h.schema_name = drop_chunks.schema_name) AND
(drop_chunks.table_name IS NULL OR h.table_name = drop_chunks.table_name)
)
SELECT DISTINCT time_dim.column_type INTO STRICT time_type
FROM hypertable_ids INNER JOIN LATERAL _timescaledb_internal.dimension_get_time(hypertable_ids.id) time_dim ON (true);
EXCEPTION
WHEN NO_DATA_FOUND THEN
RAISE EXCEPTION 'no hypertables found';
WHEN TOO_MANY_ROWS THEN
RAISE EXCEPTION 'cannot use drop_chunks on multiple tables with different time types';
END;


IF time_type = 'TIMESTAMP'::regtype THEN
PERFORM @extschema@.drop_chunks((now() - older_than)::timestamp, table_name, schema_name, cascade);
ELSIF time_type = 'DATE'::regtype THEN
PERFORM @extschema@.drop_chunks((now() - older_than)::date, table_name, schema_name, cascade);
ELSIF time_type = 'TIMESTAMPTZ'::regtype THEN
PERFORM @extschema@.drop_chunks(now() - older_than, table_name, schema_name, cascade);
ELSE
RAISE 'can only use drop_chunks with an INTERVAL for TIMESTAMP, TIMESTAMPTZ, and DATE types';
END IF;
END
$BODY$;
-- show chunks older than or newer than a specific time.
-- `hypertable` argument can be a valid hypertable or NULL.
-- In the latter case the function will try to list all
-- the chunks from all of the hypertables in the database.
-- older_than or newer_than or both can be NULL.
-- if `hypertable` argument is null but a time constraint is specified
-- through older_than or newer_than, the call will succeed
-- if and only if all the hypertables in the database
-- have the same type as the given time constraint argument
CREATE OR REPLACE FUNCTION show_chunks(
hypertable REGCLASS = NULL,
older_than "any" = NULL,
newer_than "any" = NULL
) RETURNS SETOF REGCLASS AS '@MODULE_PATHNAME@', 'ts_chunk_show_chunks'
LANGUAGE C STABLE PARALLEL SAFE;

-- Add a dimension (of partitioning) to a hypertable
--
@@ -3,37 +3,35 @@
-- This file is licensed under the Apache License, see LICENSE-APACHE
-- at the top level directory of the TimescaleDB distribution.

CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_get_time(
hypertable_id INT
)
RETURNS _timescaledb_catalog.dimension LANGUAGE SQL STABLE AS
$BODY$
SELECT *
FROM _timescaledb_catalog.dimension d
WHERE d.hypertable_id = dimension_get_time.hypertable_id AND
d.interval_length IS NOT NULL
$BODY$;
-- show_chunks for internal use only. The only difference
-- from user-facing API is that this one takes a 4th argument
-- specifying the caller name. This makes it easier to taylor
-- error messages to the caller function context.
CREATE OR REPLACE FUNCTION _timescaledb_internal.show_chunks_impl(
hypertable REGCLASS = NULL,
older_than "any" = NULL,
newer_than "any" = NULL,
caller_name NAME = NULL
) RETURNS SETOF REGCLASS AS '@MODULE_PATHNAME@', 'ts_chunk_show_chunks'
LANGUAGE C STABLE PARALLEL SAFE;

-- Drop chunks older than the given timestamp. If a hypertable name is given,
-- drop only chunks associated with this table. Any of the first three arguments
-- can be NULL meaning "all values".
CREATE OR REPLACE FUNCTION _timescaledb_internal.drop_chunks_impl(
older_than_time BIGINT,
older_than_time ANYELEMENT = NULL,
table_name NAME = NULL,
schema_name NAME = NULL,
cascade BOOLEAN = FALSE,
truncate_before BOOLEAN = FALSE
newer_than_time ANYELEMENT = NULL
)
RETURNS VOID LANGUAGE PLPGSQL VOLATILE AS
$BODY$
DECLARE
chunk_row _timescaledb_catalog.chunk;
chunk_row REGCLASS;
cascade_mod TEXT = '';
exist_count INT = 0;
BEGIN
IF older_than_time IS NULL AND table_name IS NULL AND schema_name IS NULL THEN
RAISE 'cannot have all 3 arguments to drop_chunks_older_than be NULL';
END IF;

IF cascade THEN
cascade_mod = 'CASCADE';
@@ -52,71 +50,25 @@ BEGIN
END IF;
END IF;

FOR chunk_row IN SELECT *
FROM _timescaledb_catalog.chunk c
INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = c.hypertable_id)
INNER JOIN _timescaledb_internal.dimension_get_time(h.id) time_dimension ON(true)
INNER JOIN _timescaledb_catalog.dimension_slice ds
ON (ds.dimension_id = time_dimension.id)
INNER JOIN _timescaledb_catalog.chunk_constraint cc
ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id)
WHERE (older_than_time IS NULL OR ds.range_end <= older_than_time)
AND (drop_chunks_impl.schema_name IS NULL OR h.schema_name = drop_chunks_impl.schema_name)
AND (drop_chunks_impl.table_name IS NULL OR h.table_name = drop_chunks_impl.table_name)
FOR schema_name, table_name IN
SELECT hyper.schema_name, hyper.table_name
FROM _timescaledb_catalog.hypertable hyper
WHERE
(drop_chunks_impl.schema_name IS NULL OR hyper.schema_name = drop_chunks_impl.schema_name) AND
(drop_chunks_impl.table_name IS NULL OR hyper.table_name = drop_chunks_impl.table_name)
LOOP
IF truncate_before THEN
FOR chunk_row IN SELECT _timescaledb_internal.show_chunks_impl(schema_name || '.' || table_name, older_than_time, newer_than_time, 'drop_chunks')
LOOP
EXECUTE format(
$$
TRUNCATE %I.%I %s
$$, chunk_row.schema_name, chunk_row.table_name, cascade_mod
$$
DROP TABLE %s %s
$$, chunk_row, cascade_mod
);
END IF;
EXECUTE format(
$$
DROP TABLE %I.%I %s
$$, chunk_row.schema_name, chunk_row.table_name, cascade_mod
);
END LOOP;
END LOOP;
END
$BODY$;

CREATE OR REPLACE FUNCTION _timescaledb_internal.drop_chunks_type_check(
given_type REGTYPE,
table_name NAME,
schema_name NAME
)
RETURNS VOID LANGUAGE PLPGSQL STABLE AS
$BODY$
DECLARE
actual_type regtype;
BEGIN
BEGIN
WITH hypertable_ids AS (
SELECT id
FROM _timescaledb_catalog.hypertable h
WHERE (drop_chunks_type_check.schema_name IS NULL OR h.schema_name = drop_chunks_type_check.schema_name) AND
(drop_chunks_type_check.table_name IS NULL OR h.table_name = drop_chunks_type_check.table_name)
)
SELECT DISTINCT time_dim.column_type INTO STRICT actual_type
FROM hypertable_ids INNER JOIN LATERAL _timescaledb_internal.dimension_get_time(hypertable_ids.id) time_dim ON (true);
EXCEPTION
WHEN NO_DATA_FOUND THEN
RAISE EXCEPTION 'no hypertables found';
WHEN TOO_MANY_ROWS THEN
RAISE EXCEPTION 'cannot use drop_chunks on multiple tables with different time types';
END;

IF given_type IN ('int'::regtype, 'smallint'::regtype, 'bigint'::regtype ) THEN
IF actual_type IN ('int'::regtype, 'smallint'::regtype, 'bigint'::regtype ) THEN
RETURN;
END IF;
END IF;
IF actual_type != given_type THEN
RAISE EXCEPTION 'cannot call drop_chunks with a % on hypertables with a time type of: %', given_type, actual_type;
END IF;
END
$BODY$;

--documentation of these function located in chunk_index.h
CREATE OR REPLACE FUNCTION _timescaledb_internal.chunk_index_clone(chunk_index_oid OID) RETURNS OID
AS '@MODULE_PATHNAME@', 'ts_chunk_index_clone' LANGUAGE C VOLATILE STRICT;
@@ -0,0 +1,5 @@
DROP FUNCTION IF EXISTS drop_chunks(INTERVAL, NAME, NAME, BOOLEAN);
DROP FUNCTION IF EXISTS drop_chunks(ANYELEMENT, NAME, NAME, BOOLEAN);
DROP FUNCTION IF EXISTS _timescaledb_internal.drop_chunks_impl(BIGINT, NAME, NAME, BOOLEAN, BOOLEAN);
DROP FUNCTION IF EXISTS _timescaledb_internal.drop_chunks_type_check(REGTYPE, NAME, NAME);
DROP FUNCTION IF EXISTS _timescaledb_internal.dimension_get_time(INTEGER);
@@ -12,6 +12,7 @@ set(SOURCES
chunk_dispatch_state.c
chunk_index.c
chunk_insert_state.c
compat.c
constraint_aware_append.c
copy.c
dimension.c
@@ -17,7 +17,6 @@
#include <miscadmin.h>
#include <commands/dbcommands.h>
#include <commands/sequence.h>
#include <access/xact.h>

#include "compat.h"
#include "catalog.h"

0 comments on commit 9a34028

Please sign in to comment.