Skip to content

Commit

Permalink
Release 2.0.0-rc4
Browse files Browse the repository at this point in the history
This release candidate contains bugfixes since the previous release
candidate, as well as additional minor features. It improves
validation of configuration changes for background jobs, adds support
for gapfill on distributed tables, contains improvements to the memory
handling for large COPY, and contains improvements to compression for
distributed hypertables.

**Minor Features**
* timescale#2689 Check configuration in alter_job and add_job
* timescale#2696 Support gapfill on distributed hypertable
* timescale#2468 Show more information in get_git_commit
* timescale#2678 Include user actions into job stats view
* timescale#2664 Fix support for complex aggregate expression
* timescale#2672 Add hypertable to continuous aggregates view
* timescale#2662 Save compression metadata settings on access node
* timescale#2707 Introduce additional db for data node bootstrapping

**Bugfixes**
* timescale#2688 Fix crash for concurrent drop and compress chunk
* timescale#2666 Fix timeout handling in async library
* timescale#2683 Fix crash in add_job when given NULL interval
* timescale#2698 Improve memory handling for remote COPY
* timescale#2555 Set metadata for chunks compressed before 2.0
  • Loading branch information
mkindahl committed Dec 3, 2020
1 parent 0560ed2 commit ebadf12
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 161 deletions.
31 changes: 31 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,37 @@
`psql` with the `-X` flag to prevent any `.psqlrc` commands from
accidentally triggering the load of a previous DB version.**

## 2.0.0-rc4 (2020-12-02)

This release candidate contains bugfixes since the previous release
candidate, as well as additional minor features. It improves
validation of configuration changes for background jobs, adds support
for gapfill on distributed tables, contains improvements to the memory
handling for large COPY, and contains improvements to compression for
distributed hypertables.

**Minor Features**
* #2689 Check configuration in alter_job and add_job
* #2696 Support gapfill on distributed hypertable
* #2468 Show more information in get_git_commit
* #2678 Include user actions into job stats view
* #2664 Fix support for complex aggregate expression
* #2672 Add hypertable to continuous aggregates view
* #2662 Save compression metadata settings on access node
* #2707 Introduce additional db for data node bootstrapping

**Bugfixes**
* #2688 Fix crash for concurrent drop and compress chunk
* #2666 Fix timeout handling in async library
* #2683 Fix crash in add_job when given NULL interval
* #2698 Improve memory handling for remote COPY
* #2555 Set metadata for chunks compressed before 2.0

**Thanks**
* @francesco11112 for reporting memory issue on COPY
* @Netskeh for reporting bug on time_bucket problem in continuous
aggregates

## 2.0.0-rc3 (2020-11-12)

This release candidate contains bugfixes since the previous release
Expand Down
1 change: 1 addition & 0 deletions sql/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ set(MOD_FILES
updates/1.7.4--2.0.0-rc1.sql
updates/2.0.0-rc1--2.0.0-rc2.sql
updates/2.0.0-rc2--2.0.0-rc3.sql
updates/2.0.0-rc3--2.0.0-rc4.sql
)

set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}")
Expand Down
159 changes: 159 additions & 0 deletions sql/updates/2.0.0-rc3--2.0.0-rc4.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates;
DROP VIEW IF EXISTS timescaledb_information.job_stats;
DROP FUNCTION IF EXISTS _timescaledb_internal.get_git_commit;

-- Begin Modify hypertable table
-- we make a copy of the data, remove dependencies, drop the table
DROP VIEW IF EXISTS timescaledb_information.hypertables;
DROP VIEW IF EXISTS timescaledb_information.chunks;
DROP VIEW IF EXISTS timescaledb_information.dimensions;
DROP VIEW IF EXISTS timescaledb_information.jobs;
DROP VIEW IF EXISTS timescaledb_information.compression_settings;
DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table;

CREATE TABLE _timescaledb_catalog.hypertable_tmp
AS SELECT * from _timescaledb_catalog.hypertable;

--drop foreign keys on hypertable
ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey ;
ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey ;

CREATE TABLE tmp_hypertable_seq_value AS
SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq;
ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable;
ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq;
DROP TABLE _timescaledb_catalog.hypertable;

CREATE SEQUENCE IF NOT EXISTS _timescaledb_catalog.hypertable_id_seq MINVALUE 1;

-- now create table without self referential foreign key
CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable(
id INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'),
schema_name name NOT NULL CHECK (schema_name != '_timescaledb_catalog'),
table_name name NOT NULL,
associated_schema_name name NOT NULL,
associated_table_prefix name NOT NULL,
num_dimensions smallint NOT NULL,
chunk_sizing_func_schema name NOT NULL,
chunk_sizing_func_name name NOT NULL,
chunk_target_size bigint NOT NULL CHECK (chunk_target_size >= 0), -- size in bytes
compression_state smallint NOT NULL DEFAULT 0,
compressed_hypertable_id integer ,
replication_factor smallint NULL CHECK (replication_factor > 0),
UNIQUE (associated_schema_name, associated_table_prefix),
CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name),
----internal compressed hypertables have compression state = 2
CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2),
CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL))
);
ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id;
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', '');
SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM tmp_hypertable_seq_value;

INSERT INTO _timescaledb_catalog.hypertable
( id, schema_name, table_name, associated_schema_name, associated_table_prefix,
num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name,
chunk_target_size, compression_state, compressed_hypertable_id,
replication_factor)
SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix,
num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name,
chunk_target_size,
CASE WHEN compressed is FALSE AND compressed_hypertable_id IS NOT NULL THEN 1
WHEN compressed is TRUE THEN 2
ELSE 0
END,
compressed_hypertable_id,
replication_factor
FROM _timescaledb_catalog.hypertable_tmp;

-- add self referential foreign key
ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY ( compressed_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id );
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', '');
--cleanup
DROP TABLE _timescaledb_catalog.hypertable_tmp;
DROP TABLE tmp_hypertable_seq_value;

-- add all the other foreign keys
ALTER TABLE _timescaledb_catalog.hypertable_data_node
ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id );
ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id );
ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT
continuous_agg_mat_hypertable_id_fkey FOREIGN KEY ( mat_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT
continuous_agg_raw_hypertable_id_fkey FOREIGN KEY ( raw_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold
ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey
FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT
hypertable_compression_hypertable_id_fkey FOREIGN KEY ( hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;

GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC;
GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC;
--End Modify hypertable table

-- update metadata for chunks compressed before 2.0
DO $$
DECLARE
plain_chunk RECORD;
comp_chunk TEXT;
rowcount_pre BIGINT;
rowcount_post BIGINT;
BEGIN
FOR plain_chunk IN
SELECT
*
FROM
_timescaledb_catalog.chunk comp
WHERE
compressed_chunk_id IS NOT NULL LOOP
SELECT
format('%I.%I', schema_name, table_name) INTO comp_chunk
FROM
_timescaledb_catalog.chunk
WHERE
id = plain_chunk.compressed_chunk_id;
EXECUTE format('SELECT sum(_ts_meta_count), count(*) FROM %s', comp_chunk) INTO rowcount_pre, rowcount_post;
UPDATE
_timescaledb_catalog.compression_chunk_size
SET
numrows_pre_compression = rowcount_pre,
numrows_post_compression = rowcount_post
WHERE
chunk_id = plain_chunk.id;
END LOOP;
END
$$;

159 changes: 0 additions & 159 deletions sql/updates/latest-dev.sql
Original file line number Diff line number Diff line change
@@ -1,159 +0,0 @@
DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates;
DROP VIEW IF EXISTS timescaledb_information.job_stats;
DROP FUNCTION IF EXISTS _timescaledb_internal.get_git_commit;

-- Begin Modify hypertable table
-- we make a copy of the data, remove dependencies, drop the table
DROP VIEW IF EXISTS timescaledb_information.hypertables;
DROP VIEW IF EXISTS timescaledb_information.chunks;
DROP VIEW IF EXISTS timescaledb_information.dimensions;
DROP VIEW IF EXISTS timescaledb_information.jobs;
DROP VIEW IF EXISTS timescaledb_information.compression_settings;
DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table;

CREATE TABLE _timescaledb_catalog.hypertable_tmp
AS SELECT * from _timescaledb_catalog.hypertable;

--drop foreign keys on hypertable
ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey ;
ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey ;
ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey ;

CREATE TABLE tmp_hypertable_seq_value AS
SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq;
ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable;
ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq;
DROP TABLE _timescaledb_catalog.hypertable;

CREATE SEQUENCE IF NOT EXISTS _timescaledb_catalog.hypertable_id_seq MINVALUE 1;

-- now create table without self referential foreign key
CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable(
id INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'),
schema_name name NOT NULL CHECK (schema_name != '_timescaledb_catalog'),
table_name name NOT NULL,
associated_schema_name name NOT NULL,
associated_table_prefix name NOT NULL,
num_dimensions smallint NOT NULL,
chunk_sizing_func_schema name NOT NULL,
chunk_sizing_func_name name NOT NULL,
chunk_target_size bigint NOT NULL CHECK (chunk_target_size >= 0), -- size in bytes
compression_state smallint NOT NULL DEFAULT 0,
compressed_hypertable_id integer ,
replication_factor smallint NULL CHECK (replication_factor > 0),
UNIQUE (associated_schema_name, associated_table_prefix),
CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name),
----internal compressed hypertables have compression state = 2
CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2),
CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL))
);
ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id;
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', '');
SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM tmp_hypertable_seq_value;

INSERT INTO _timescaledb_catalog.hypertable
( id, schema_name, table_name, associated_schema_name, associated_table_prefix,
num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name,
chunk_target_size, compression_state, compressed_hypertable_id,
replication_factor)
SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix,
num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name,
chunk_target_size,
CASE WHEN compressed is FALSE AND compressed_hypertable_id IS NOT NULL THEN 1
WHEN compressed is TRUE THEN 2
ELSE 0
END,
compressed_hypertable_id,
replication_factor
FROM _timescaledb_catalog.hypertable_tmp;

-- add self referential foreign key
ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY ( compressed_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id );
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', '');
--cleanup
DROP TABLE _timescaledb_catalog.hypertable_tmp;
DROP TABLE tmp_hypertable_seq_value;

-- add all the other foreign keys
ALTER TABLE _timescaledb_catalog.hypertable_data_node
ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id );
ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id );
ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey
FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT
continuous_agg_mat_hypertable_id_fkey FOREIGN KEY ( mat_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT
continuous_agg_raw_hypertable_id_fkey FOREIGN KEY ( raw_hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold
ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey
FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;
ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT
hypertable_compression_hypertable_id_fkey FOREIGN KEY ( hypertable_id )
REFERENCES _timescaledb_catalog.hypertable( id )
ON DELETE CASCADE;

GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC;
GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC;
--End Modify hypertable table

-- update metadata for chunks compressed before 2.0
DO $$
DECLARE
plain_chunk RECORD;
comp_chunk TEXT;
rowcount_pre BIGINT;
rowcount_post BIGINT;
BEGIN
FOR plain_chunk IN
SELECT
*
FROM
_timescaledb_catalog.chunk comp
WHERE
compressed_chunk_id IS NOT NULL LOOP
SELECT
format('%I.%I', schema_name, table_name) INTO comp_chunk
FROM
_timescaledb_catalog.chunk
WHERE
id = plain_chunk.compressed_chunk_id;
EXECUTE format('SELECT sum(_ts_meta_count), count(*) FROM %s', comp_chunk) INTO rowcount_pre, rowcount_post;
UPDATE
_timescaledb_catalog.compression_chunk_size
SET
numrows_pre_compression = rowcount_pre,
numrows_post_compression = rowcount_post
WHERE
chunk_id = plain_chunk.id;
END LOOP;
END
$$;

5 changes: 3 additions & 2 deletions version.config
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
version = 2.0.0-rc3
update_from_version = 2.0.0-rc2
version = 2.0.0-rc4
update_from_version = 2.0.0-rc3

0 comments on commit ebadf12

Please sign in to comment.