Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[P4Testgen] Fix behavior of coverage on edge cases with no nodes to cover #4275

Merged
merged 11 commits into from
Dec 17, 2023
4 changes: 4 additions & 0 deletions backends/p4tools/common/options.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ std::optional<ICompileContext *> AbstractP4cToolOptions::process(
}
P4CContext::get().options().file = remainingArgs->at(0);

if (!validateOptions()) {
return std::nullopt;
}

return compilerContext;
}

Expand Down
4 changes: 4 additions & 0 deletions backends/p4tools/common/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ class AbstractP4cToolOptions : protected Util::Options {
/// Hook for customizing options processing.
std::vector<const char *> *process(int argc, char *const argv[]) override;

/// Checks if parsed options make sense with respect to each-other.
/// @returns true if the validation was successfull and false otherwise.
virtual bool validateOptions() const { return true; }

/// Converts a vector of command-line arguments into the traditional (argc, argv) format.
static std::tuple<int, char **> convertArgs(const std::vector<const char *> &args);

Expand Down
7 changes: 6 additions & 1 deletion backends/p4tools/modules/testgen/lib/test_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,13 @@ bool TestBackEnd::run(const FinalState &state) {
}

const P4::Coverage::CoverageSet &visitedNodes = symbex.getVisitedNodes();
if (coverableNodes.empty()) {
if (!testgenOptions.hasCoverageTracking) {
printFeature("test_info", 4, "============ Test %1% ============", testCount);
} else if (coverableNodes.empty()) {
printFeature("test_info", 4,
"============ Test %1%: No coverable nodes ============", testCount);
// All 0 nodes covered.
coverage = 1.0;
} else {
coverage =
static_cast<float>(visitedNodes.size()) / static_cast<float>(coverableNodes.size());
Expand Down
13 changes: 13 additions & 0 deletions backends/p4tools/modules/testgen/options.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ TestgenOptions::TestgenOptions()
"STATEMENTS",
"TABLE_ENTRIES",
};
hasCoverageTracking = true;
auto selectionString = cstring(arg).toUpper();
auto it = COVERAGE_OPTIONS.find(selectionString);
if (it != COVERAGE_OPTIONS.end()) {
Expand Down Expand Up @@ -382,4 +383,16 @@ TestgenOptions::TestgenOptions()
"produce no tests or only tests that contain counter examples.");
}

bool TestgenOptions::validateOptions() const {
if (minCoverage > 0 && !hasCoverageTracking) {
::error(
ErrorType::ERR_INVALID,
"It is not allowed to have --assert-min-coverage set to non-zero without a coverage "
"tracking enabled with --track-coverage option. Without coverage tracking, the "
"--assert-min-coverage is meaningless.");
return false;
}
return true;
}

} // namespace P4Tools::P4Testgen
7 changes: 7 additions & 0 deletions backends/p4tools/modules/testgen/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,18 @@ class TestgenOptions : public AbstractP4cToolOptions {
/// Multiple options are possible. Currently supported: STATEMENTS, TABLE_ENTRIES.
P4::Coverage::CoverageOptions coverageOptions;

/// Indicates that coverage tracking is enabled for some coverage criteria. This is used for
/// sanity checking and it also affects information printed to output.
bool hasCoverageTracking = false;

/// Specifies minimum coverage that needs to be achieved for P4Testgen to exit successfully.
float minCoverage = 0;

const char *getIncludePath() override;

protected:
bool validateOptions() const override;

private:
TestgenOptions();
};
Expand Down
Loading