From d2db9c27f65093e8c948a483a43c692d35b264e9 Mon Sep 17 00:00:00 2001 From: Jillian Tullo Date: Fri, 2 Feb 2018 13:31:16 -0500 Subject: [PATCH 1/2] Add vacuum to job scheduler Add vacuum to job scheduler on a 36 hour schedule. Previously this was done periodically as configured via the appliance console. --- app/models/miq_schedule_worker/jobs.rb | 6 ++++ app/models/miq_schedule_worker/runner.rb | 7 +++++ config/settings.yml | 25 ++++++++++++++++ lib/extensions/ar_base.rb | 6 ++++ .../models/miq_schedule_worker/runner_spec.rb | 30 ++++++++++++++++--- 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/app/models/miq_schedule_worker/jobs.rb b/app/models/miq_schedule_worker/jobs.rb index 4bebb524410..a870b8002ee 100644 --- a/app/models/miq_schedule_worker/jobs.rb +++ b/app/models/miq_schedule_worker/jobs.rb @@ -149,6 +149,12 @@ def database_maintenance_reindex_timer end end + def database_maintenance_vacuum_timer + ::Settings.database.maintenance.vacuum_tables.each do |class_name| + queue_work(:class_name => class_name, :method_name => "vacuum", :role => "database_operations", :zone => nil) + end + end + def check_for_stuck_dispatch(threshold_seconds) class_n = "JobProxyDispatcher" method_n = "dispatch" diff --git a/app/models/miq_schedule_worker/runner.rb b/app/models/miq_schedule_worker/runner.rb index af898fe6c3e..4a26d6b8bef 100644 --- a/app/models/miq_schedule_worker/runner.rb +++ b/app/models/miq_schedule_worker/runner.rb @@ -287,6 +287,13 @@ def schedules_for_database_operations_role :tags => %i(database_operations database_maintenance_reindex_schedule), ) { enqueue(:database_maintenance_reindex_timer) } + sched = ::Settings.database.maintenance.vacuum_schedule + _log.info("database_maintenance_vacuum_schedule: #{sched}") + scheduler.schedule_cron( + sched, + :tags => %i(database_operations database_maintenance_vacuum_schedule), + ) { enqueue(:database_maintenance_vacuum_timer) } + @schedules[:database_operations] end diff --git a/config/settings.yml b/config/settings.yml index f34f14fa5fe..14cabe8b54a 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -89,6 +89,31 @@ - Metric - MiqQueue - MiqWorker + :vacuum_schedule: "0 2 * * 6" + :vacuum_tables: + - Vm + - BinaryBlobPart + - BinaryBlob + - CustomizationSpec + - FirewallRule + - Host + - Storage + - MiqSchedule + - EventLog + - PolicyEvent + - Snapshot + - Job + - Network + - MiqQueue + - MiqRequestTask + - MiqWorker + - MiqServer + - MiqSearch + - MiqScsiLun + - MiqScsiTarget + - StorageFile + - Tagging + - VimPerformanceState :metrics_collection: :collection_schedule: "1 * * * *" :daily_rollup_schedule: "23 0 * * *" diff --git a/lib/extensions/ar_base.rb b/lib/extensions/ar_base.rb index f9223e03ba5..358d82ecb69 100644 --- a/lib/extensions/ar_base.rb +++ b/lib/extensions/ar_base.rb @@ -20,5 +20,11 @@ def self.reindex def self.reindex_table_name table_name end + + def self.vacuum + _log.info("Vacuuming table #{table_name}") + result = connection.vacuum_full_analyze_table(table_name) + _log.info("Completed Vacuuming of table #{table_name} with result #{result.result_status}") + end end end diff --git a/spec/models/miq_schedule_worker/runner_spec.rb b/spec/models/miq_schedule_worker/runner_spec.rb index 2de53a172b4..b6ab87b10ea 100644 --- a/spec/models/miq_schedule_worker/runner_spec.rb +++ b/spec/models/miq_schedule_worker/runner_spec.rb @@ -276,8 +276,16 @@ @metrics_collection = {:collection_schedule => "1 * * * *", :daily_rollup_schedule => "23 0 * * *"} @metrics_history = {:purge_schedule => "50 * * * *"} - @database_maintenance = {:reindex_schedule => "1 * * * *", :reindex_tables => %w(Metric MiqQueue MiqWorker)} - database_config = {:metrics_collection => @metrics_collection, :metrics_history => @metrics_history} + @database_maintenance = { + :reindex_schedule => "1 * * * *", + :reindex_tables => %w(Metric MiqQueue MiqWorker), + :vacuum_schedule => "0 2 * * 6", + :vacuum_tables => %w(Vm BinaryBlobPart BinaryBlob CustomizationSpec FirewallRule Host Storage + MiqSchedule EventLog PolicyEvent Snapshot Job Network MiqQueue MiqRequestTask + MiqWorker MiqServer MiqSearch MiqScsiLun MiqScsiTarget StorageFile + Tagging VimPerformanceState) + } + database_config = {:metrics_collection => @metrics_collection, :metrics_history => @metrics_history} stub_server_configuration(:database => database_config) end @@ -288,7 +296,7 @@ it "queues the right items" do scheduled_jobs = @schedule_worker.schedules_for_database_operations_role - expect(scheduled_jobs.size).to be(4) + expect(scheduled_jobs.size).to be(5) scheduled_jobs.each do |job| expect(job).to be_a_kind_of(Rufus::Scheduler::CronJob) @@ -321,6 +329,13 @@ message = MiqQueue.where(:class_name => class_name, :method_name => "reindex").first expect(message).to have_attributes(:role => "database_operations", :zone => nil) end + when %w(database_operations database_maintenance_vacuum_schedule) + expect(job.original).to eq(@database_maintenance[:vacuum_schedule]) + expect(MiqQueue.count).to eq(@database_maintenance[:vacuum_tables].size) + @database_maintenance[:vacuum_tables].each do |class_name| + message = MiqQueue.where(:class_name => class_name, :method_name => "vacuum").first + expect(message).to have_attributes(:role => "database_operations", :zone => nil) + end else raise_unexpected_job_error(job) end @@ -336,7 +351,7 @@ it "queues the right items" do scheduled_jobs = @schedule_worker.schedules_for_database_operations_role - expect(scheduled_jobs.size).to be(4) + expect(scheduled_jobs.size).to be(5) scheduled_jobs.each do |job| expect(job).to be_kind_of(Rufus::Scheduler::CronJob) @@ -370,6 +385,13 @@ message = MiqQueue.where(:class_name => class_name, :method_name => "reindex").first expect(message).to have_attributes(:role => "database_operations", :zone => nil) end + when %w(database_operations database_maintenance_vacuum_schedule) + expect(job.original).to eq(@database_maintenance[:vacuum_schedule]) + expect(MiqQueue.count).to eq(@database_maintenance[:vacuum_tables].size) + @database_maintenance[:vacuum_tables].each do |class_name| + message = MiqQueue.where(:class_name => class_name, :method_name => "vacuum").first + expect(message).to have_attributes(:role => "database_operations", :zone => nil) + end else raise_unexpected_job_error(job) end From bf3d79ce335dc25a0e6c5df8d66cc08608162ae1 Mon Sep 17 00:00:00 2001 From: Jillian Tullo Date: Fri, 2 Feb 2018 13:33:33 -0500 Subject: [PATCH 2/2] Fix log message of reindexing to show the result --- config/settings.yml | 22 +++++++++++----------- lib/extensions/ar_base.rb | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config/settings.yml b/config/settings.yml index 14cabe8b54a..6be5faa3d9b 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -91,29 +91,29 @@ - MiqWorker :vacuum_schedule: "0 2 * * 6" :vacuum_tables: - - Vm - - BinaryBlobPart - BinaryBlob + - BinaryBlobPart - CustomizationSpec + - EventLog - FirewallRule - Host - - Storage - - MiqSchedule - - EventLog - - PolicyEvent - - Snapshot - Job - - Network - MiqQueue - MiqRequestTask - - MiqWorker - - MiqServer - - MiqSearch + - MiqSchedule - MiqScsiLun - MiqScsiTarget + - MiqSearch + - MiqServer + - MiqWorker + - Network + - PolicyEvent + - Snapshot + - Storage - StorageFile - Tagging - VimPerformanceState + - Vm :metrics_collection: :collection_schedule: "1 * * * *" :daily_rollup_schedule: "23 0 * * *" diff --git a/lib/extensions/ar_base.rb b/lib/extensions/ar_base.rb index 358d82ecb69..f02d9f71eee 100644 --- a/lib/extensions/ar_base.rb +++ b/lib/extensions/ar_base.rb @@ -13,8 +13,8 @@ def self.truncate def self.reindex _log.info("Reindexing table #{reindex_table_name}") - connection.reindex_table(reindex_table_name) - _log.info("Reindexing table #{reindex_table_name}") + result = connection.reindex_table(reindex_table_name) + _log.info("Completed Reindexing of table #{reindex_table_name} with result #{result.result_status}") end def self.reindex_table_name