diff --git a/CHANGELOG.md b/CHANGELOG.md index 997516d3c0a..2a48bebc75f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ `psql` with the `-X` flag to prevent any `.psqlrc` commands from accidentally triggering the load of a previous DB version.** +## 2.0.0-rc4 (2020-12-02) + +This release candidate contains bugfixes since the previous release +candidate, as well as additional minor features. It improves +validation of configuration changes for background jobs, adds support +for gapfill on distributed tables, contains improvements to the memory +handling for large COPY, and contains improvements to compression for +distributed hypertables. + +**Minor Features** +* #2689 Check configuration in alter_job and add_job +* #2696 Support gapfill on distributed hypertable +* #2468 Show more information in get_git_commit +* #2678 Include user actions into job stats view +* #2664 Fix support for complex aggregate expression +* #2672 Add hypertable to continuous aggregates view +* #2662 Save compression metadata settings on access node +* #2707 Introduce additional db for data node bootstrapping + +**Bugfixes** +* #2688 Fix crash for concurrent drop and compress chunk +* #2666 Fix timeout handling in async library +* #2683 Fix crash in add_job when given NULL interval +* #2698 Improve memory handling for remote COPY +* #2555 Set metadata for chunks compressed before 2.0 + +**Thanks** +* @francesco11112 for reporting memory issue on COPY +* @Netskeh for reporting bug on time_bucket problem in continuous + aggregates + ## 2.0.0-rc3 (2020-11-12) This release candidate contains bugfixes since the previous release diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index a5bea956a46..d79f0df4d60 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -101,6 +101,7 @@ set(MOD_FILES updates/1.7.4--2.0.0-rc1.sql updates/2.0.0-rc1--2.0.0-rc2.sql updates/2.0.0-rc2--2.0.0-rc3.sql + updates/2.0.0-rc3--2.0.0-rc4.sql ) set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}") diff --git a/sql/updates/2.0.0-rc3--2.0.0-rc4.sql b/sql/updates/2.0.0-rc3--2.0.0-rc4.sql new file mode 100644 index 00000000000..5af154cff2c --- /dev/null +++ b/sql/updates/2.0.0-rc3--2.0.0-rc4.sql @@ -0,0 +1,159 @@ +DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; +DROP VIEW IF EXISTS timescaledb_information.job_stats; +DROP FUNCTION IF EXISTS _timescaledb_internal.get_git_commit; + +-- Begin Modify hypertable table +-- we make a copy of the data, remove dependencies, drop the table +DROP VIEW IF EXISTS timescaledb_information.hypertables; +DROP VIEW IF EXISTS timescaledb_information.chunks; +DROP VIEW IF EXISTS timescaledb_information.dimensions; +DROP VIEW IF EXISTS timescaledb_information.jobs; +DROP VIEW IF EXISTS timescaledb_information.compression_settings; +DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; +DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; +DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table; + +CREATE TABLE _timescaledb_catalog.hypertable_tmp +AS SELECT * from _timescaledb_catalog.hypertable; + +--drop foreign keys on hypertable +ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey ; +ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey ; +ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey ; + +CREATE TABLE tmp_hypertable_seq_value AS +SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; +ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; +ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; +DROP TABLE _timescaledb_catalog.hypertable; + +CREATE SEQUENCE IF NOT EXISTS _timescaledb_catalog.hypertable_id_seq MINVALUE 1; + +-- now create table without self referential foreign key +CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable( + id INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), + schema_name name NOT NULL CHECK (schema_name != '_timescaledb_catalog'), + table_name name NOT NULL, + associated_schema_name name NOT NULL, + associated_table_prefix name NOT NULL, + num_dimensions smallint NOT NULL, + chunk_sizing_func_schema name NOT NULL, + chunk_sizing_func_name name NOT NULL, + chunk_target_size bigint NOT NULL CHECK (chunk_target_size >= 0), -- size in bytes + compression_state smallint NOT NULL DEFAULT 0, + compressed_hypertable_id integer , + replication_factor smallint NULL CHECK (replication_factor > 0), + UNIQUE (associated_schema_name, associated_table_prefix), + CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), +----internal compressed hypertables have compression state = 2 + CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), + CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)) +); +ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; +SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); +SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM tmp_hypertable_seq_value; + +INSERT INTO _timescaledb_catalog.hypertable +( id, schema_name, table_name, associated_schema_name, associated_table_prefix, + num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, + chunk_target_size, compression_state, compressed_hypertable_id, + replication_factor) +SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, + num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, + chunk_target_size, + CASE WHEN compressed is FALSE AND compressed_hypertable_id IS NOT NULL THEN 1 + WHEN compressed is TRUE THEN 2 + ELSE 0 + END, + compressed_hypertable_id, + replication_factor +FROM _timescaledb_catalog.hypertable_tmp; + +-- add self referential foreign key +ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY ( compressed_hypertable_id ) + REFERENCES _timescaledb_catalog.hypertable( id ); +SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', ''); +--cleanup +DROP TABLE _timescaledb_catalog.hypertable_tmp; +DROP TABLE tmp_hypertable_seq_value; + +-- add all the other foreign keys +ALTER TABLE _timescaledb_catalog.hypertable_data_node +ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ); +ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ); +ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey +FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT +continuous_agg_mat_hypertable_id_fkey FOREIGN KEY ( mat_hypertable_id ) +REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT +continuous_agg_raw_hypertable_id_fkey FOREIGN KEY ( raw_hypertable_id ) +REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold +ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey +FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; +ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT +hypertable_compression_hypertable_id_fkey FOREIGN KEY ( hypertable_id ) +REFERENCES _timescaledb_catalog.hypertable( id ) +ON DELETE CASCADE; + +GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; +GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; +--End Modify hypertable table + +-- update metadata for chunks compressed before 2.0 +DO $$ +DECLARE + plain_chunk RECORD; + comp_chunk TEXT; + rowcount_pre BIGINT; + rowcount_post BIGINT; +BEGIN + FOR plain_chunk IN + SELECT + * + FROM + _timescaledb_catalog.chunk comp + WHERE + compressed_chunk_id IS NOT NULL LOOP + SELECT + format('%I.%I', schema_name, table_name) INTO comp_chunk + FROM + _timescaledb_catalog.chunk + WHERE + id = plain_chunk.compressed_chunk_id; + EXECUTE format('SELECT sum(_ts_meta_count), count(*) FROM %s', comp_chunk) INTO rowcount_pre, rowcount_post; + UPDATE + _timescaledb_catalog.compression_chunk_size + SET + numrows_pre_compression = rowcount_pre, + numrows_post_compression = rowcount_post + WHERE + chunk_id = plain_chunk.id; + END LOOP; +END +$$; + diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 5af154cff2c..e69de29bb2d 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -1,159 +0,0 @@ -DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; -DROP VIEW IF EXISTS timescaledb_information.job_stats; -DROP FUNCTION IF EXISTS _timescaledb_internal.get_git_commit; - --- Begin Modify hypertable table --- we make a copy of the data, remove dependencies, drop the table -DROP VIEW IF EXISTS timescaledb_information.hypertables; -DROP VIEW IF EXISTS timescaledb_information.chunks; -DROP VIEW IF EXISTS timescaledb_information.dimensions; -DROP VIEW IF EXISTS timescaledb_information.jobs; -DROP VIEW IF EXISTS timescaledb_information.compression_settings; -DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; -DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; -DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table; - -CREATE TABLE _timescaledb_catalog.hypertable_tmp -AS SELECT * from _timescaledb_catalog.hypertable; - ---drop foreign keys on hypertable -ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey ; -ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey ; -ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey ; - -CREATE TABLE tmp_hypertable_seq_value AS -SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; -ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; -ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; -DROP TABLE _timescaledb_catalog.hypertable; - -CREATE SEQUENCE IF NOT EXISTS _timescaledb_catalog.hypertable_id_seq MINVALUE 1; - --- now create table without self referential foreign key -CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable( - id INTEGER PRIMARY KEY DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), - schema_name name NOT NULL CHECK (schema_name != '_timescaledb_catalog'), - table_name name NOT NULL, - associated_schema_name name NOT NULL, - associated_table_prefix name NOT NULL, - num_dimensions smallint NOT NULL, - chunk_sizing_func_schema name NOT NULL, - chunk_sizing_func_name name NOT NULL, - chunk_target_size bigint NOT NULL CHECK (chunk_target_size >= 0), -- size in bytes - compression_state smallint NOT NULL DEFAULT 0, - compressed_hypertable_id integer , - replication_factor smallint NULL CHECK (replication_factor > 0), - UNIQUE (associated_schema_name, associated_table_prefix), - CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), -----internal compressed hypertables have compression state = 2 - CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), - CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)) -); -ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; -SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); -SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM tmp_hypertable_seq_value; - -INSERT INTO _timescaledb_catalog.hypertable -( id, schema_name, table_name, associated_schema_name, associated_table_prefix, - num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, - chunk_target_size, compression_state, compressed_hypertable_id, - replication_factor) -SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, - num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, - chunk_target_size, - CASE WHEN compressed is FALSE AND compressed_hypertable_id IS NOT NULL THEN 1 - WHEN compressed is TRUE THEN 2 - ELSE 0 - END, - compressed_hypertable_id, - replication_factor -FROM _timescaledb_catalog.hypertable_tmp; - --- add self referential foreign key -ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY ( compressed_hypertable_id ) - REFERENCES _timescaledb_catalog.hypertable( id ); -SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', ''); ---cleanup -DROP TABLE _timescaledb_catalog.hypertable_tmp; -DROP TABLE tmp_hypertable_seq_value; - --- add all the other foreign keys -ALTER TABLE _timescaledb_catalog.hypertable_data_node -ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ); -ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ); -ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey -FOREIGN KEY ( hypertable_id ) REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT -continuous_agg_mat_hypertable_id_fkey FOREIGN KEY ( mat_hypertable_id ) -REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT -continuous_agg_raw_hypertable_id_fkey FOREIGN KEY ( raw_hypertable_id ) -REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold -ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey -FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; -ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT -hypertable_compression_hypertable_id_fkey FOREIGN KEY ( hypertable_id ) -REFERENCES _timescaledb_catalog.hypertable( id ) -ON DELETE CASCADE; - -GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; -GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; ---End Modify hypertable table - --- update metadata for chunks compressed before 2.0 -DO $$ -DECLARE - plain_chunk RECORD; - comp_chunk TEXT; - rowcount_pre BIGINT; - rowcount_post BIGINT; -BEGIN - FOR plain_chunk IN - SELECT - * - FROM - _timescaledb_catalog.chunk comp - WHERE - compressed_chunk_id IS NOT NULL LOOP - SELECT - format('%I.%I', schema_name, table_name) INTO comp_chunk - FROM - _timescaledb_catalog.chunk - WHERE - id = plain_chunk.compressed_chunk_id; - EXECUTE format('SELECT sum(_ts_meta_count), count(*) FROM %s', comp_chunk) INTO rowcount_pre, rowcount_post; - UPDATE - _timescaledb_catalog.compression_chunk_size - SET - numrows_pre_compression = rowcount_pre, - numrows_post_compression = rowcount_post - WHERE - chunk_id = plain_chunk.id; - END LOOP; -END -$$; - diff --git a/version.config b/version.config index 6855af288bf..eaf8d7395f4 100644 --- a/version.config +++ b/version.config @@ -1,2 +1,3 @@ -version = 2.0.0-rc3 -update_from_version = 2.0.0-rc2 +version = 2.0.0-rc4 +update_from_version = 2.0.0-rc3 +