Skip to content

Commit

Permalink
Merge pull request #10148 from Icinga/enhanced-sort-types-by-load-dep…
Browse files Browse the repository at this point in the history 
…endencies

Sort config types by their load dependencies once
  • Loading branch information
@yhabteab
yhabteab authored Sep 26, 2024
2 parents 0fff415 + eb97676 commit 92df9ef
Show file tree
Hide file tree
Showing 6 changed files with 224 additions and 131 deletions.
1 change: 1 addition & 0 deletions lib/base/initialize.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ enum class InitializePriority {
RegisterBuiltinTypes,
RegisterFunctions,
RegisterTypes,
SortTypes,
EvaluateConfigFragments,
Default,
FreezeNamespaces,
Expand Down
47 changes: 47 additions & 0 deletions lib/base/type.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
/* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */

#include "base/type.hpp"
#include "base/atomic.hpp"
#include "base/configobject.hpp"
#include "base/debug.hpp"
#include "base/scriptglobal.hpp"
#include "base/namespace.hpp"
#include "base/objectlock.hpp"
#include <functional>

using namespace icinga;

Expand Down Expand Up @@ -32,6 +36,43 @@ INITIALIZE_ONCE_WITH_PRIORITY([]() {
Type::Register(type);
}, InitializePriority::RegisterTypeType);

static std::vector<Type::Ptr> l_SortedByLoadDependencies;
static Atomic l_SortingByLoadDependenciesDone (false);

INITIALIZE_ONCE_WITH_PRIORITY([] {
std::unordered_set<Type*> visited;

std::function<void(Type*)> visit;
// Please note that this callback does not detect any cyclic load dependencies,
// instead, it relies on the "sort_by_load_after" unit test to fail.
visit = ([&visit, &visited](Type* type) {
if (visited.find(type) != visited.end()) {
return;
}
visited.emplace(type);

for (auto dependency : type->GetLoadDependencies()) {
visit(dependency);
}

// We have managed to reach the final/top node in this dependency graph,
// so let's place them in reverse order to their final place.
l_SortedByLoadDependencies.emplace_back(type);
});

// Sort the types by their load_after dependencies in a Depth-First search manner.
for (const Type::Ptr& type : Type::GetAllTypes()) {
// Note that only those types that are assignable to the dynamic ConfigObject type can have "load_after"
// dependencies, otherwise they are just some Icinga 2 primitive types such as Number, String, etc. and
// we need to ignore them.
if (ConfigObject::TypeInstance->IsAssignableFrom(type)) {
visit(type.get());
}
}

l_SortingByLoadDependenciesDone.store(true);
}, InitializePriority::SortTypes);

String Type::ToString() const
{
return "type '" + GetName() + "'";
Expand Down Expand Up @@ -72,6 +113,12 @@ std::vector<Type::Ptr> Type::GetAllTypes()
return types;
}

const std::vector<Type::Ptr>& Type::GetConfigTypesSortedByLoadDependencies()
{
VERIFY(l_SortingByLoadDependenciesDone.load());
return l_SortedByLoadDependencies;
}

String Type::GetPluralName() const
{
String name = GetName();
Expand Down
15 changes: 15 additions & 0 deletions lib/base/type.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,21 @@ class Type : public Object
static Type::Ptr GetByName(const String& name);
static std::vector<Type::Ptr> GetAllTypes();

/**
* Returns a list of config types sorted by their "load_after" dependencies.
*
* All dependencies of a given type are listed at a lower index than that of the type itself. In other words,
* if a `Service` type load depends on the `Host` and `ApiListener` types, the Host and ApiListener types are
* guaranteed to appear first on the list. Nevertheless, the order of the Host and ApiListener types themselves
* is arbitrary if the two types are not dependent.
*
* It should be noted that this method will fail fatally when used prior to the completion
* of namespace initialization.
*
* @return std::vector<Type::Ptr>
*/
static const std::vector<Ptr>& GetConfigTypesSortedByLoadDependencies();

void SetField(int id, const Value& value, bool suppress_events = false, const Value& cookie = Empty) override;
Value GetField(int id) const override;

Expand Down
201 changes: 76 additions & 125 deletions lib/config/configitem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,180 +444,131 @@ bool ConfigItem::CommitNewItems(const ActivationContext::Ptr& context, WorkQueue
<< "Committing " << total << " new items.";
#endif /* I2_DEBUG */

std::set<Type::Ptr> types;
std::set<Type::Ptr> completed_types;
int itemsCount {0};

for (const Type::Ptr& type : Type::GetAllTypes()) {
if (ConfigObject::TypeInstance->IsAssignableFrom(type))
types.insert(type);
}

while (types.size() != completed_types.size()) {
for (const Type::Ptr& type : types) {
if (completed_types.find(type) != completed_types.end())
continue;
for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
std::atomic<int> committed_items(0);

bool unresolved_dep = false;
{
auto items (itemsByType.find(type.get()));

/* skip this type (for now) if there are unresolved load dependencies */
for (auto pLoadDep : type->GetLoadDependencies()) {
if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
unresolved_dep = true;
break;
if (items != itemsByType.end()) {
for (const ItemPair& pair: items->second) {
newItems.emplace_back(pair.first);
}
}

if (unresolved_dep)
continue;

std::atomic<int> committed_items(0);
upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;

{
auto items (itemsByType.find(type.get()));
if (!item->Commit(ip.second)) {
if (item->IsIgnoreOnError()) {
item->Unregister();
}

if (items != itemsByType.end()) {
for (const ItemPair& pair: items->second) {
newItems.emplace_back(pair.first);
return;
}

upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
committed_items++;
});

if (!item->Commit(ip.second)) {
if (item->IsIgnoreOnError()) {
item->Unregister();
}

return;
}

committed_items++;
});

upq.Join();
}
upq.Join();
}
}

itemsCount += committed_items;

completed_types.insert(type);
itemsCount += committed_items;

#ifdef I2_DEBUG
if (committed_items > 0)
Log(LogDebug, "configitem")
<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
if (committed_items > 0)
Log(LogDebug, "configitem")
<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */

if (upq.HasExceptions())
return false;
}
if (upq.HasExceptions())
return false;
}

#ifdef I2_DEBUG
Log(LogDebug, "configitem")
<< "Committed " << itemsCount << " items.";
#endif /* I2_DEBUG */

completed_types.clear();
for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
std::atomic<int> notified_items(0);

while (types.size() != completed_types.size()) {
for (const Type::Ptr& type : types) {
if (completed_types.find(type) != completed_types.end())
continue;
{
auto items (itemsByType.find(type.get()));

bool unresolved_dep = false;
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;

/* skip this type (for now) if there are unresolved load dependencies */
for (auto pLoadDep : type->GetLoadDependencies()) {
if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
unresolved_dep = true;
break;
}
}
if (!item->m_Object)
return;

if (unresolved_dep)
continue;

std::atomic<int> notified_items(0);

{
auto items (itemsByType.find(type.get()));

if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;

if (!item->m_Object)
return;
try {
item->m_Object->OnAllConfigLoaded();
notified_items++;
} catch (const std::exception& ex) {
if (!item->m_IgnoreOnError)
throw;

try {
item->m_Object->OnAllConfigLoaded();
notified_items++;
} catch (const std::exception& ex) {
if (!item->m_IgnoreOnError)
throw;
Log(LogNotice, "ConfigObject")
<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);

Log(LogNotice, "ConfigObject")
<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);
item->Unregister();

item->Unregister();

{
std::unique_lock<std::mutex> lock(item->m_Mutex);
item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
}
{
std::unique_lock<std::mutex> lock(item->m_Mutex);
item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
}
});
}
});

upq.Join();
}
upq.Join();
}

completed_types.insert(type);
}

#ifdef I2_DEBUG
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */

if (upq.HasExceptions())
return false;
if (upq.HasExceptions())
return false;

notified_items = 0;
for (auto loadDep : type->GetLoadDependencies()) {
auto items (itemsByType.find(loadDep));
notified_items = 0;
for (auto loadDep : type->GetLoadDependencies()) {
auto items (itemsByType.find(loadDep));

if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;
if (items != itemsByType.end()) {
upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
const ConfigItem::Ptr& item = ip.first;

if (!item->m_Object)
return;
if (!item->m_Object)
return;

ActivationScope ascope(item->m_ActivationContext);
item->m_Object->CreateChildObjects(type);
notified_items++;
});
}
ActivationScope ascope(item->m_ActivationContext);
item->m_Object->CreateChildObjects(type);
notified_items++;
});
}
}

upq.Join();
upq.Join();

#ifdef I2_DEBUG
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
if (notified_items > 0)
Log(LogDebug, "configitem")
<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
#endif /* I2_DEBUG */

if (upq.HasExceptions())
return false;
if (upq.HasExceptions())
return false;

// Make sure to activate any additionally generated items
if (!CommitNewItems(context, upq, newItems))
return false;
}
// Make sure to activate any additionally generated items
if (!CommitNewItems(context, upq, newItems))
return false;
}

return true;
Expand Down
Loading

0 comments on commit 92df9ef

Please sign in to comment.