Targeted refresh with batch saver_strategy support

Also added hello_workflow configuration_workflow to populate_tower rake task

app/models/manageiq/providers/ansible_tower/inventory/collector.rb

@@ -1,5 +1,4 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Collector < ManageIQ::Providers::Inventory::Collector
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
   require_nested :TargetCollection
 end

app/models/manageiq/providers/ansible_tower/inventory/parser.rb

@@ -1,4 +1,3 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Parser < ManageIQ::Providers::Inventory::Parser
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
 end

app/models/manageiq/providers/ansible_tower/inventory/persister.rb

@@ -1,5 +1,4 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Persister < ManageIQ::Providers::Inventory::Persister
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
   require_nested :TargetCollection
 end

app/models/manageiq/providers/ansible_tower/inventory/persister/definitions/automation_collections.rb

@@ -2,7 +2,8 @@ module ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::Aut
   extend ActiveSupport::Concern
 
   def initialize_automation_inventory_collections
-    %i(credentials
+    %i(authentication_configuration_script_bases
+       credentials
        configuration_script_sources
        configured_systems).each do |name|
 

app/models/manageiq/providers/ansible_tower/shared/inventory/collector/target_collection.rb

@@ -5,26 +5,31 @@ def connection
 
   def initialize(_manager, _target)
     super
+
+    infer_related_manager_refs!
   end
 
   def inventories
     find_records(:inventory_root_groups, connection.api.inventories)
   end
 
   def hosts
-    find_records(:configured_systems, connection.api.hosts)
+    return @hosts if @hosts.present?
+    @hosts = find_records(:configured_systems, connection.api.hosts)
   end
 
   def job_templates
-    find_records(:configuration_scripts, connection.api.job_templates)
+    return @job_templates if @job_templates.present?
+    @job_templates = find_records(:configuration_scripts, connection.api.job_templates)
   end
 
   def configuration_workflows
     find_records(:configuration_scripts, connection.api.workflow_job_templates)
   end
 
   def projects
-    find_records(:configuration_script_sources, connection.api.projects)
+    return @projects if @projects.present?
+    @projects = find_records(:configuration_script_sources, connection.api.projects)
   end
 
   def credentials
@@ -33,6 +38,37 @@ def credentials
 
   protected
 
+  def infer_related_manager_refs!
+    if references(:configured_systems).present?
+      hosts.each do |host|
+        add_simple_target!(:inventory_root_groups, host.inventory_id.to_s)
+      end
+    end
+
+    if references(:configuration_scripts).present?
+      job_templates.each do |job_template|
+        add_simple_target!(:inventory_root_groups, job_template.inventory_id.to_s)
+        add_simple_target!(:configuration_script_sources, job_template.try(:project_id)) if job_template.try(:project_id).present?
+        %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
+          next unless job_template.respond_to?(credential_attr)
+          credential_id = job_template.public_send(credential_attr).to_s
+          next if credential_id.blank?
+          add_simple_target!(:credentials, credential_id)
+        end
+      end
+    end
+
+    # target was added in previous block, compute references again
+    target.manager_refs_by_association_reset
+    if references(:configuration_script_sources)
+      projects.each do |project|
+        add_simple_target!(:credentials, project.credential_id.to_s)
+      end
+    end
+
+    target.manager_refs_by_association_reset
+  end
+
   # Calls API and finds existing records.
   # @param inventory_collection_name [Symbol] IC name (as identified in persister's definitions)
   # @param endpoint - endpoint for AnsibleTowerClient api call
@@ -44,7 +80,7 @@ def find_records(inventory_collection_name, endpoint)
       begin
         [endpoint.find(refs[0])]
       rescue AnsibleTowerClient::ResourceNotFoundError
-        [nil]
+        []
       end
     else
       endpoint.all(:id__in => refs.join(','))
@@ -63,4 +99,10 @@ def manager_ref_by_collection(inventory_collection_name)
     @tmp_persister ||= self.class.to_s.gsub('::Collector', '::Persister').constantize.new(@manager, @target)
     @tmp_persister.collections[inventory_collection_name].manager_ref[0]
   end
+
+  def add_simple_target!(association, ems_ref)
+    return if ems_ref.blank?
+
+    target.add_target(:association => association, :manager_ref => { manager_ref_by_collection(association) => ems_ref })
+  end
 end

app/models/manageiq/providers/ansible_tower/shared/inventory/parser/automation_manager.rb

@@ -44,13 +44,19 @@ def configuration_scripts
         :manager_ref                 => job_template.playbook
       )
 
-      inventory_object.authentications = []
-      %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
-        next unless job_template.respond_to?(credential_attr)
-        credential_id = job_template.public_send(credential_attr).to_s
-        next if credential_id.blank?
-        inventory_object.authentications << persister.credentials.lazy_find(credential_id)
-      end
+      configuration_script_authentications(inventory_object, job_template)
+    end
+  end
+
+  def configuration_script_authentications(persister_configuration_script, job_template)
+    %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
+      next unless job_template.respond_to?(credential_attr)
+      credential_id = job_template.public_send(credential_attr).to_s
+      next if credential_id.blank?
+      persister.authentication_configuration_script_bases.build(
+        :configuration_script_base => persister_configuration_script,
+        :authentication            => persister.credentials.lazy_find(credential_id)
+      )
     end
   end
 

config/settings.yml

@@ -15,6 +15,6 @@
     :inventory_object_refresh: true
     :allow_targeted_refresh: true
     :inventory_collections:
-      :saver_strategy: default
+      :saver_strategy: batch
 :log:
   :level_ansible_tower: info

lib/tasks_private/spec_helper.rake

@@ -1,7 +1,7 @@
 namespace :spec do
   desc "Populate expected Tower objects for casssettes and spec tests"
   task :populate_tower do
-    tower_host = ENV['TOWER_URL'] || "https://dev-ansible-tower3.example.com/api/v1/"
+    tower_host = ENV['TOWER_URL'] ||"https://dev-ansible-tower3.example.com/api/v1/"
     id = ENV['TOWER_USER'] || 'testuser'
     password = ENV['TOWER_PASSWORD'] || 'secret'
     populate = PopulateTower.new(tower_host, id, password)
@@ -409,6 +409,17 @@ class PopulateTower
     }
     @conn.post(uri, data)
 
+    # create workflow job template
+    uri = '/api/v1/workflow_job_templates/'
+    data = {
+      :name         => 'hello_workflow',
+      :description  => 'test workflow',
+      :inventory    => inventory['id'],
+      :organization => organization['id']
+    }
+    workflow_template = create_obj(uri, data)
+    @tower_data[:items][data[:name]] = { :id => workflow_template['id'] }
+
     # Create a project with failed update.
     uri = '/api/v1/projects/'
     data = {

spec/models/manageiq/providers/ansible_tower/automation_manager/refresher_targeted_spec.rb

@@ -0,0 +1,7 @@
+describe ManageIQ::Providers::AnsibleTower::AutomationManager::Refresher do
+  it_behaves_like 'refresh targeted',
+                  :provider_ansible_tower,
+                  described_class.parent,
+                  :ansible,
+                  described_class.name.underscore + '_targeted'
+end

spec/support/ansible_shared/automation_manager/refresh_configuartion_script_source.rb

@@ -1,7 +1,7 @@
 shared_examples_for "refresh configuration_script_source" do |ansible_provider, manager_class, ems_type, cassette_path|
   include_context "uses tower_data.yml"
 
-  let(:tower_url) { ENV['TOWER_URL'] || "https://example.com/api/v1/" }
+  let(:tower_url) { ENV['TOWER_URL'] || "https://dev-ansible-tower3.example.com/api/v1/" }
   let(:auth_userid) { ENV['TOWER_USER'] || 'testuser' }
   let(:auth_password) { ENV['TOWER_PASSWORD'] || 'secret' }
 
@@ -10,9 +10,9 @@
   let(:provider) do
     _guid, _server, zone = EvmSpecHelper.create_guid_miq_server_zone
     FactoryBot.create(ansible_provider,
-                       :zone       => zone,
-                       :url        => tower_url,
-                       :verify_ssl => false,).tap { |provider| provider.authentications << auth }
+                      :zone       => zone,
+                      :url        => tower_url,
+                      :verify_ssl => false,).tap { |provider| provider.authentications << auth }
   end
   let(:manager_class) { manager_class }
 
@@ -26,15 +26,23 @@
   it "will perform a targeted refresh" do
     credential = FactoryBot.create(:"#{ems_type}_scm_credential", :name => '2keep')
     automation_manager.credentials << credential
-    configuration_script_source = FactoryBot.create(:"#{ems_type}_configuration_script_source",
-                                                     :authentication => credential,
-                                                     :manager        => automation_manager,
-                                                     :manager_ref    => targeted_refresh_id)
-    configuration_script_source.configuration_script_payloads.create!(:manager_ref => '2b_rm', :name => '2b_rm')
-    configuration_script_source_other = FactoryBot.create(:"#{ems_type}_configuration_script_source",
-                                                           :manager_ref => nonexistent_repo_id,
-                                                           :manager     => automation_manager,
-                                                           :name        => 'Dont touch this')
+    configuration_script_source = FactoryBot.create(
+      :"#{ems_type}_configuration_script_source",
+      :authentication => credential,
+      :manager        => automation_manager,
+      :manager_ref    => targeted_refresh_id
+    )
+    configuration_script_source.configuration_script_payloads.create!(
+      :manager_ref => '2b_rm',
+      :name        => '2b_rm',
+      :type        => manager_class::ConfigurationScriptPayload
+    )
+    configuration_script_source_other = FactoryBot.create(
+      :"#{ems_type}_configuration_script_source",
+      :manager_ref => nonexistent_repo_id,
+      :manager     => automation_manager,
+      :name        => 'Dont touch this'
+    )
 
     # When re-recording the cassetes, comment this to default to normal poll sleep time
     stub_const("ManageIQ::Providers::AnsibleTower::Shared::AutomationManager::ConfigurationScriptSource::REFRESH_ON_TOWER_SLEEP", 0.seconds)
@@ -47,7 +55,10 @@
         configuration_script_payloads = configuration_script_source.configuration_script_payloads
 
         VCR.use_cassette(cassette_path) do
-          EmsRefresh.refresh([[configuration_script_source.class.to_s, configuration_script_source.id]])
+          EmsRefresh.refresh([InventoryRefresh::Target.new(
+            :association => :configuration_script_sources,
+            :manager_id  => configuration_script_source.manager_id,
+            :manager_ref => { :manager_ref => configuration_script_source.manager_ref })])
 
           expect(automation_manager.reload.last_refresh_error).to be_nil
           expect(automation_manager.configuration_script_sources.count).to eq(2)