Add purge timer for archived entities

app/models/container.rb

@@ -2,6 +2,7 @@ class Container < ApplicationRecord
   include SupportsFeatureMixin
   include NewWithTypeStiMixin
   include ArchivedMixin
+  include_concern 'Purging'
 
   has_one    :container_group, :through => :container_definition
   belongs_to :ext_management_system, :foreign_key => :ems_id

app/models/container/purging.rb

@@ -0,0 +1,20 @@
+class Container < ApplicationRecord
+  module Purging
+    extend ActiveSupport::Concern
+    include PurgingMixin
+
+    module ClassMethods
+      def purge_date
+        ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc
+      end
+
+      def purge_window_size
+        ::Settings.container_entities.history.purge_window_size
+      end
+
+      def purge_scope(older_than)
+        where(arel_table[:deleted_on].lteq(older_than))
+      end
+    end
+  end
+end

app/models/container_definition.rb

@@ -1,5 +1,6 @@
 class ContainerDefinition < ApplicationRecord
   include ArchivedMixin
+  include_concern 'Purging'
   # :name, :image, :image_pull_policy, :memory, :cpu
   belongs_to :container_group
   belongs_to :ext_management_system, :foreign_key => :ems_id

app/models/container_definition/purging.rb

@@ -0,0 +1,20 @@
+class ContainerDefinition < ApplicationRecord
+  module Purging
+    extend ActiveSupport::Concern
+    include PurgingMixin
+
+    module ClassMethods
+      def purge_date
+        ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc
+      end
+
+      def purge_window_size
+        ::Settings.container_entities.history.purge_window_size
+      end
+
+      def purge_scope(older_than)
+        where(arel_table[:deleted_on].lteq(older_than))
+      end
+    end
+  end
+end

app/models/container_group.rb

@@ -6,6 +6,7 @@ class ContainerGroup < ApplicationRecord
   include NewWithTypeStiMixin
   include TenantIdentityMixin
   include ArchivedMixin
+  include_concern 'Purging'
 
   # :name, :uid, :creation_timestamp, :resource_version, :namespace
   # :labels, :restart_policy, :dns_policy

app/models/container_group/purging.rb

@@ -0,0 +1,20 @@
+class ContainerGroup < ApplicationRecord
+  module Purging
+    extend ActiveSupport::Concern
+    include PurgingMixin
+
+    module ClassMethods
+      def purge_date
+        ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc
+      end
+
+      def purge_window_size
+        ::Settings.container_entities.history.purge_window_size
+      end
+
+      def purge_scope(older_than)
+        where(arel_table[:deleted_on].lteq(older_than))
+      end
+    end
+  end
+end

app/models/container_image.rb

@@ -4,7 +4,8 @@ class ContainerImage < ApplicationRecord
   include ScanningMixin
   include TenantIdentityMixin
   include CustomAttributeMixin
-
+  include ArchivedMixin
+  include_concern 'Purging'
 
   DOCKER_IMAGE_PREFIX = "docker://"
   DOCKER_PULLABLE_PREFIX = "docker-pullable://".freeze
@@ -27,9 +28,6 @@ class ContainerImage < ApplicationRecord
   serialize :exposed_ports, Hash
   serialize :environment_variables, Hash
 
-  # Needed for scanning & tagging action
-  delegate :my_zone, :to => :ext_management_system
-
   acts_as_miq_taggable
   virtual_column :display_registry, :type => :string
 

app/models/container_image/purging.rb

@@ -0,0 +1,20 @@
+class ContainerImage < ApplicationRecord
+  module Purging
+    extend ActiveSupport::Concern
+    include PurgingMixin
+
+    module ClassMethods
+      def purge_date
+        ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc
+      end
+
+      def purge_window_size
+        ::Settings.container_entities.history.purge_window_size
+      end
+
+      def purge_scope(older_than)
+        where(arel_table[:deleted_on].lteq(older_than))
+      end
+    end
+  end
+end

app/models/container_project.rb

@@ -2,6 +2,7 @@ class ContainerProject < ApplicationRecord
   include SupportsFeatureMixin
   include CustomAttributeMixin
   include ArchivedMixin
+  include_concern 'Purging'
   belongs_to :ext_management_system, :foreign_key => "ems_id"
   has_many :container_groups
   has_many :container_routes

app/models/container_project/purging.rb

@@ -0,0 +1,20 @@
+class ContainerProject < ApplicationRecord
+  module Purging
+    extend ActiveSupport::Concern
+    include PurgingMixin
+
+    module ClassMethods
+      def purge_date
+        ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc
+      end
+
+      def purge_window_size
+        ::Settings.container_entities.history.purge_window_size
+      end
+
+      def purge_scope(older_than)
+        where(arel_table[:deleted_on].lteq(older_than))
+      end
+    end
+  end
+end

app/models/miq_schedule_worker/jobs.rb

@@ -121,6 +121,14 @@ def miq_report_result_purge_timer
     queue_work(:class_name => "MiqReportResult", :method_name => "purge_timer", :zone => nil)
   end
 
+  def archived_entities_purge_timer
+    queue_work(:class_name => "Container", :method_name => "purge_timer", :zone => nil)
+    queue_work(:class_name => "ContainerGroup", :method_name => "purge_timer", :zone => nil)
+    queue_work(:class_name => "ContainerImage", :method_name => "purge_timer", :zone => nil)
+    queue_work(:class_name => "ContainerProject", :method_name => "purge_timer", :zone => nil)
+    queue_work(:class_name => "ContainerDefinition", :method_name => "purge_timer", :zone => nil)
+  end
+
   def storage_refresh_metrics
     queue_work(
       :class_name  => "StorageManager",

app/models/miq_schedule_worker/runner.rb

@@ -221,6 +221,11 @@ def schedules_for_scheduler_role
       enqueue :miq_report_result_purge_timer
     end
 
+    every = worker_settings[:container_entities_purge_interval]
+    scheduler.schedule_every(every, :first_in => every) do
+      enqueue :archived_entities_purge_timer
+    end
+
     # Schedule every 24 hours
     at = worker_settings[:storage_file_collection_time_utc]
     if Time.now.strftime("%Y-%m-%d #{at}").to_time(:utc) < Time.now.utc

config/settings.yml

@@ -1031,6 +1031,10 @@
   :history:
     :keep_policy_events: 6.months
     :purge_window_size: 1000
+:container_entities:
+  :history:
+    :keep_archived_entities: 6.months
+    :purge_window_size: 1000
 :product:
   :maindb: ExtManagementSystem
   :container_deployment_wizard: false
@@ -1363,6 +1367,7 @@
         :poll_method: :normal
         :queue_timeout: 120.minutes
     :schedule_worker:
+      :container_entities_purge_interval: 1.day
       :authentication_check_interval: 1.hour
       :chargeback_generation_interval: 1.day
       :chargeback_generation_time_utc: 01:00:00

spec/factories/container_definition.rb

@@ -0,0 +1,4 @@
+FactoryGirl.define do
+  factory :container_definition do
+  end
+end

spec/models/container/purging_spec.rb

@@ -0,0 +1,47 @@
+describe Container do
+  context "::Purging" do
+    context ".purge_queue" do
+      before do
+        EvmSpecHelper.create_guid_miq_server_zone
+      end
+      let(:purge_time) { (Time.zone.now + 10).round }
+
+      it "submits to the queue" do
+        expect(described_class).to receive(:purge_date).and_return(purge_time)
+        described_class.purge_timer
+
+        q = MiqQueue.all
+        expect(q.length).to eq(1)
+        expect(q.first).to have_attributes(
+          :class_name  => described_class.name,
+          :method_name => "purge_by_date",
+          :args        => [purge_time]
+        )
+      end
+    end
+
+    context ".purge" do
+      let(:deleted_date) { 6.months.ago }
+
+      before do
+        @old_container        = FactoryGirl.create(:container, :deleted_on => deleted_date - 1.day)
+        @purge_date_container = FactoryGirl.create(:container, :deleted_on => deleted_date)
+        @new_container        = FactoryGirl.create(:container, :deleted_on => deleted_date + 1.day)
+      end
+
+      def assert_unpurged_ids(unpurged_ids)
+        expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort)
+      end
+
+      it "purge_date and older" do
+        described_class.purge(deleted_date)
+        assert_unpurged_ids(@new_container.id)
+      end
+
+      it "with a window" do
+        described_class.purge(deleted_date, 1)
+        assert_unpurged_ids(@new_container.id)
+      end
+    end
+  end
+end

spec/models/container_definition/purging_spec.rb

@@ -0,0 +1,47 @@
+describe ContainerDefinition do
+  context "::Purging" do
+    context ".purge_queue" do
+      before do
+        EvmSpecHelper.create_guid_miq_server_zone
+      end
+      let(:purge_time) { (Time.zone.now + 10).round }
+
+      it "submits to the queue" do
+        expect(described_class).to receive(:purge_date).and_return(purge_time)
+        described_class.purge_timer
+
+        q = MiqQueue.all
+        expect(q.length).to eq(1)
+        expect(q.first).to have_attributes(
+          :class_name  => described_class.name,
+          :method_name => "purge_by_date",
+          :args        => [purge_time]
+        )
+      end
+    end
+
+    context ".purge" do
+      let(:deleted_date) { 6.months.ago }
+
+      before do
+        @old_container_def        = FactoryGirl.create(:container_definition, :deleted_on => deleted_date - 1.day)
+        @purge_date_container_def = FactoryGirl.create(:container_definition, :deleted_on => deleted_date)
+        @new_container_def        = FactoryGirl.create(:container_definition, :deleted_on => deleted_date + 1.day)
+      end
+
+      def assert_unpurged_ids(unpurged_ids)
+        expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort)
+      end
+
+      it "purge_date and older" do
+        described_class.purge(deleted_date)
+        assert_unpurged_ids(@new_container_def.id)
+      end
+
+      it "with a window" do
+        described_class.purge(deleted_date, 1)
+        assert_unpurged_ids(@new_container_def.id)
+      end
+    end
+  end
+end

spec/models/container_group/purging_spec.rb

@@ -0,0 +1,47 @@
+describe ContainerGroup do
+  context "::Purging" do
+    context ".purge_queue" do
+      before do
+        EvmSpecHelper.create_guid_miq_server_zone
+      end
+      let(:purge_time) { (Time.zone.now + 10).round }
+
+      it "submits to the queue" do
+        expect(described_class).to receive(:purge_date).and_return(purge_time)
+        described_class.purge_timer
+
+        q = MiqQueue.all
+        expect(q.length).to eq(1)
+        expect(q.first).to have_attributes(
+          :class_name  => described_class.name,
+          :method_name => "purge_by_date",
+          :args        => [purge_time]
+        )
+      end
+    end
+
+    context ".purge" do
+      let(:deleted_date) { 6.months.ago }
+
+      before do
+        @old_container_group        = FactoryGirl.create(:container_group, :deleted_on => deleted_date - 1.day)
+        @purge_date_container_group = FactoryGirl.create(:container_group, :deleted_on => deleted_date)
+        @new_container_group        = FactoryGirl.create(:container_group, :deleted_on => deleted_date + 1.day)
+      end
+
+      def assert_unpurged_ids(unpurged_ids)
+        expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort)
+      end
+
+      it "purge_date and older" do
+        described_class.purge(deleted_date)
+        assert_unpurged_ids(@new_container_group.id)
+      end
+
+      it "with a window" do
+        described_class.purge(deleted_date, 1)
+        assert_unpurged_ids(@new_container_group.id)
+      end
+    end
+  end
+end

spec/models/container_image/purging_spec.rb

@@ -0,0 +1,47 @@
+describe ContainerImage do
+  context "::Purging" do
+    context ".purge_queue" do
+      before do
+        EvmSpecHelper.create_guid_miq_server_zone
+      end
+      let(:purge_time) { (Time.zone.now + 10).round }
+
+      it "submits to the queue" do
+        expect(described_class).to receive(:purge_date).and_return(purge_time)
+        described_class.purge_timer
+
+        q = MiqQueue.all
+        expect(q.length).to eq(1)
+        expect(q.first).to have_attributes(
+          :class_name  => described_class.name,
+          :method_name => "purge_by_date",
+          :args        => [purge_time]
+        )
+      end
+    end
+
+    context ".purge" do
+      let(:deleted_date) { 6.months.ago }
+
+      before do
+        @old_container_image        = FactoryGirl.create(:container_image, :deleted_on => deleted_date - 1.day)
+        @purge_date_container_image = FactoryGirl.create(:container_image, :deleted_on => deleted_date)
+        @new_container_image        = FactoryGirl.create(:container_image, :deleted_on => deleted_date + 1.day)
+      end
+
+      def assert_unpurged_ids(unpurged_ids)
+        expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort)
+      end
+
+      it "purge_date and older" do
+        described_class.purge(deleted_date)
+        assert_unpurged_ids(@new_container_image.id)
+      end
+
+      it "with a window" do
+        described_class.purge(deleted_date, 1)
+        assert_unpurged_ids(@new_container_image.id)
+      end
+    end
+  end
+end