Merge pull request ManageIQ#153 from slemrmartin/targeted-refresh

Targeted refresh

app/models/manageiq/providers/ansible_tower/automation_manager/refresher.rb

@@ -1,3 +1,3 @@
-class ManageIQ::Providers::AnsibleTower::AutomationManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
+class ManageIQ::Providers::AnsibleTower::AutomationManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
   include ManageIQ::Providers::AnsibleTower::Shared::AutomationManager::Refresher
 end

app/models/manageiq/providers/ansible_tower/inventory.rb

@@ -6,4 +6,13 @@ class ManageIQ::Providers::AnsibleTower::Inventory < ManageIQ::Providers::Invent
   def self.default_manager_name
     "AutomationManager"
   end
+
+  def self.parser_classes_for(ems, target)
+    case target
+    when InventoryRefresh::TargetCollection
+      [ManageIQ::Providers::AnsibleTower::Inventory::Parser::AutomationManager]
+    else
+      super
+    end
+  end
 end

app/models/manageiq/providers/ansible_tower/inventory/collector.rb

@@ -1,4 +1,4 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Collector < ManageIQ::Providers::Inventory::Collector
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
+  require_nested :TargetCollection
 end

app/models/manageiq/providers/ansible_tower/inventory/collector/target_collection.rb

@@ -0,0 +1,3 @@
+class ManageIQ::Providers::AnsibleTower::Inventory::Collector::TargetCollection < ManageIQ::Providers::Inventory::Collector
+  include ManageIQ::Providers::AnsibleTower::Shared::Inventory::Collector::TargetCollection
+end

app/models/manageiq/providers/ansible_tower/inventory/parser.rb

@@ -1,4 +1,3 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Parser < ManageIQ::Providers::Inventory::Parser
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
 end

app/models/manageiq/providers/ansible_tower/inventory/persister.rb

@@ -1,4 +1,4 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Persister < ManageIQ::Providers::Inventory::Persister
   require_nested :AutomationManager
-  require_nested :ConfigurationScriptSource
+  require_nested :TargetCollection
 end

app/models/manageiq/providers/ansible_tower/inventory/persister/automation_manager.rb

@@ -1,22 +1,7 @@
 class ManageIQ::Providers::AnsibleTower::Inventory::Persister::AutomationManager < ManageIQ::Providers::AnsibleTower::Inventory::Persister
-  include ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::Collections
+  include ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::AutomationCollections
 
   def initialize_inventory_collections
-    %i(credentials
-       configuration_script_sources
-       configured_systems).each do |name|
-
-      add_collection(automation, name)
-    end
-
-    add_collection(automation, :configuration_scripts) do |builder|
-      builder.add_properties(:model_class => ManageIQ::Providers::AutomationManager::ConfigurationScript)
-    end
-
-    add_configuration_script_payloads
-
-    add_inventory_root_groups
-
-    add_vms
+    initialize_automation_inventory_collections
   end
 end

app/models/manageiq/providers/ansible_tower/inventory/persister/definitions/collections.rb → app/models/manageiq/providers/ansible_tower/inventory/persister/definitions/automation_collections.rb

@@ -1,13 +1,37 @@
-module ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::Collections
+module ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::AutomationCollections
   extend ActiveSupport::Concern
 
+  def initialize_automation_inventory_collections
+    %i(authentication_configuration_script_bases
+       credentials
+       configuration_script_sources
+       configured_systems).each do |name|
+
+      add_collection(automation, name)
+    end
+
+    add_configuration_scripts
+
+    add_configuration_script_payloads
+
+    add_inventory_root_groups
+
+    add_vms
+  end
+
   # ------ IC provider specific definitions -------------------------
   def add_inventory_root_groups
     add_collection(automation, :inventory_root_groups) do |builder|
       builder.add_properties(:model_class => ManageIQ::Providers::AutomationManager::InventoryRootGroup)
     end
   end
 
+  def add_configuration_scripts
+    add_collection(automation, :configuration_scripts) do |builder|
+      builder.add_properties(:model_class => ManageIQ::Providers::AutomationManager::ConfigurationScript)
+    end
+  end
+
   def add_configuration_script_payloads(extra_properties = {})
     add_collection(automation, :configuration_script_payloads, extra_properties) do |builder|
       builder.add_properties(

app/models/manageiq/providers/ansible_tower/inventory/persister/target_collection.rb

@@ -0,0 +1,15 @@
+class ManageIQ::Providers::AnsibleTower::Inventory::Persister::TargetCollection < ManageIQ::Providers::AnsibleTower::Inventory::Persister
+  include ManageIQ::Providers::AnsibleTower::Inventory::Persister::Definitions::AutomationCollections
+
+  def targeted?
+    true
+  end
+
+  def strategy
+    :local_db_find_missing_references
+  end
+
+  def initialize_inventory_collections
+    initialize_automation_inventory_collections
+  end
+end

app/models/manageiq/providers/ansible_tower/shared/inventory/collector/target_collection.rb

@@ -0,0 +1,117 @@
+module ManageIQ::Providers::AnsibleTower::Shared::Inventory::Collector::TargetCollection
+  MAX_FILTER_SIZE = 200
+
+  def connection
+    @connection ||= manager.connect
+  end
+
+  def initialize(_manager, _target)
+    super
+
+    infer_related_manager_refs!
+  end
+
+  def inventories
+    find_records(:inventory_root_groups, connection.api.inventories)
+  end
+
+  def hosts
+    return @hosts if @hosts.present?
+    @hosts = find_records(:configured_systems, connection.api.hosts)
+  end
+
+  def job_templates
+    return @job_templates if @job_templates.present?
+    @job_templates = find_records(:configuration_scripts, connection.api.job_templates)
+  end
+
+  def configuration_workflows
+    find_records(:configuration_scripts, connection.api.workflow_job_templates)
+  end
+
+  def projects
+    return @projects if @projects.present?
+    @projects = find_records(:configuration_script_sources, connection.api.projects)
+  end
+
+  def credentials
+    find_records(:credentials, connection.api.credentials)
+  end
+
+  protected
+
+  def infer_related_manager_refs!
+    if references(:configured_systems).present?
+      hosts.each do |host|
+        add_simple_target!(:inventory_root_groups, host.inventory_id.to_s)
+      end
+    end
+
+    if references(:configuration_scripts).present?
+      job_templates.each do |job_template|
+        add_simple_target!(:inventory_root_groups, job_template.inventory_id.to_s)
+        add_simple_target!(:configuration_script_sources, job_template.try(:project_id)) if job_template.try(:project_id).present?
+        %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
+          next unless job_template.respond_to?(credential_attr)
+          credential_id = job_template.public_send(credential_attr).to_s
+          next if credential_id.blank?
+          add_simple_target!(:credentials, credential_id)
+        end
+      end
+    end
+
+    # target was added in previous block, compute references again
+    target.manager_refs_by_association_reset
+    if references(:configuration_script_sources)
+      projects.each do |project|
+        add_simple_target!(:credentials, project.credential_id.to_s)
+      end
+    end
+
+    target.manager_refs_by_association_reset
+  end
+
+  # Calls API and finds existing records.
+  # @param inventory_collection_name [Symbol] IC name (as identified in persister's definitions)
+  # @param endpoint - endpoint for AnsibleTowerClient api call
+  def find_records(inventory_collection_name, endpoint)
+    refs = references(inventory_collection_name)
+    return [] if refs.blank?
+
+    if refs.size == 1
+      begin
+        [endpoint.find(refs[0])]
+      rescue AnsibleTowerClient::ResourceNotFoundError
+        []
+      end
+    else
+      multi_query(refs) do |refs_batch|
+        # returns Enumeration
+        endpoint.all(:id__in => refs_batch.join(','))
+      end
+    end
+  end
+
+  def multi_query(refs)
+    refs.each_slice(MAX_FILTER_SIZE).map { |refs_batch| yield(refs_batch) }.lazy.flat_map(&:lazy)
+  end
+
+  # @param collection [Symbol] inventory collection name (as identified in persister's definitions)
+  def references(collection)
+    manager_ref = manager_ref_by_collection(collection)
+    target.manager_refs_by_association.try(:[], collection).try(:[], manager_ref.to_sym).try(:to_a).try(:compact) || []
+  end
+
+  # Using first manager_ref from persister's IC definitions
+  # All calls are identified by one unique ID
+  def manager_ref_by_collection(inventory_collection_name)
+    @tmp_persister ||= self.class.to_s.gsub('::Collector', '::Persister').constantize.new(@manager, @target)
+    @tmp_persister.collections[inventory_collection_name].manager_ref[0]
+  end
+
+  def add_simple_target!(association, ems_ref)
+    return if ems_ref.blank?
+
+    target.add_target(:association => association, :manager_ref => { manager_ref_by_collection(association) => ems_ref })
+  end
+end

app/models/manageiq/providers/ansible_tower/shared/inventory/parser/automation_manager.rb

@@ -44,13 +44,19 @@ def configuration_scripts
         :manager_ref                 => job_template.playbook
       )
 
-      inventory_object.authentications = []
-      %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
-        next unless job_template.respond_to?(credential_attr)
-        credential_id = job_template.public_send(credential_attr).to_s
-        next if credential_id.blank?
-        inventory_object.authentications << persister.credentials.lazy_find(credential_id)
-      end
+      configuration_script_authentications(inventory_object, job_template)
+    end
+  end
+
+  def configuration_script_authentications(persister_configuration_script, job_template)
+    %w(credential_id cloud_credential_id network_credential_id).each do |credential_attr|
+      next unless job_template.respond_to?(credential_attr)
+      credential_id = job_template.public_send(credential_attr).to_s
+      next if credential_id.blank?
+      persister.authentication_configuration_script_bases.build(
+        :configuration_script_base => persister_configuration_script,
+        :authentication            => persister.credentials.lazy_find(credential_id)
+      )
     end
   end
 

config/settings.yml

@@ -10,5 +10,11 @@
     :password:
     :port:
     :user:
+:ems_refresh:
+  :ansible_tower_automation:
+    :inventory_object_refresh: true
+    :allow_targeted_refresh: true
+    :inventory_collections:
+      :saver_strategy: batch
 :log:
   :level_ansible_tower: info

lib/tasks_private/spec_helper.rake

@@ -1,7 +1,7 @@
 namespace :spec do
   desc "Populate expected Tower objects for casssettes and spec tests"
   task :populate_tower do
-    tower_host = ENV['TOWER_URL'] || "https://dev-ansible-tower3.example.com/api/v1/"
+    tower_host = ENV['TOWER_URL'] ||"https://dev-ansible-tower3.example.com/api/v1/"
     id = ENV['TOWER_USER'] || 'testuser'
     password = ENV['TOWER_PASSWORD'] || 'secret'
     populate = PopulateTower.new(tower_host, id, password)
@@ -409,6 +409,17 @@ class PopulateTower
     }
     @conn.post(uri, data)
 
+    # create workflow job template
+    uri = '/api/v1/workflow_job_templates/'
+    data = {
+      :name         => 'hello_workflow',
+      :description  => 'test workflow',
+      :inventory    => inventory['id'],
+      :organization => organization['id']
+    }
+    workflow_template = create_obj(uri, data)
+    @tower_data[:items][data[:name]] = { :id => workflow_template['id'] }
+
     # Create a project with failed update.
     uri = '/api/v1/projects/'
     data = {

spec/models/manageiq/providers/ansible_tower/automation_manager/refresher_targeted_spec.rb

@@ -0,0 +1,7 @@
+describe ManageIQ::Providers::AnsibleTower::AutomationManager::Refresher do
+  it_behaves_like 'refresh targeted',
+                  :provider_ansible_tower,
+                  described_class.parent,
+                  :ansible,
+                  described_class.name.underscore + '_targeted'
+end