[P4Testgen] Add a test case generation API to P4Testgen.

BUILD.bazel

@@ -241,6 +241,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "test_helpers",
+    srcs = ["test/gtest/helpers.cpp"],
+    hdrs = ["test/gtest/helpers.h"],
+    data = ["test"],
+    deps = [
+        ":ir_frontend_midend_control_plane",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "p4c_bmv2_common_lib",
     srcs = glob(["backends/bmv2/common/*.cpp"]),

backends/p4tools/BUILD.bazel

@@ -125,27 +125,36 @@ filegroup(
 cc_library(
     name = "testgen_lib",
     srcs = glob([
-        "modules/testgen/core/**/*.h",
-        "modules/testgen/lib/*.h",
-        "modules/testgen/*.h",
-    ]) + glob([
         "modules/testgen/core/*.cpp",
         "modules/testgen/core/small_step/*.cpp",
         "modules/testgen/core/symbolic_executor/*.cpp",
         "modules/testgen/lib/*.cpp",
+    ]) + glob([
+        "modules/testgen/core/*.h",
+        "modules/testgen/core/small_step/*.h",
+        "modules/testgen/core/symbolic_executor/*.h",
+        "modules/testgen/lib/*.h",
     ]) + [
-        "common/options.h",
         "modules/testgen/options.cpp",
         "modules/testgen/testgen.cpp",
         ":register_testgen_targets",
         ":target_register_headers",
         ":testgen_targets_src",
     ],
+    # We onlu make top-level, library, and test backend header files visible to other targets.
+    hdrs = ["common/options.h"] + glob([
+        "modules/testgen/lib/*.h",
+        "modules/testgen/*.h",
+    ]) + glob(
+        ["modules/testgen/targets/%s/test_backend/*.h" % target for target in TESTGEN_TARGETS],
+    ),
     copts = [
         "-fexceptions",
         "-w",
     ],
     features = ["-use_header_modules"],
+    # Needs to be accessible by other targets.
+    visibility = ["//visibility:public"],
     deps = [
         ":common",
         ":register_testgen_targets",
@@ -182,3 +191,19 @@ build_test(
         ":p4testgen",
     ],
 )
+
+cc_test(
+    name = "p4testgen_api_test",
+    srcs = [
+        "modules/testgen/targets/bmv2/test/testgen_api/api_test.cpp",
+    ],
+    copts = [
+        "-fexceptions",
+        "-w",
+    ],
+    deps = [
+        ":testgen_lib",
+        "//:test_helpers",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

backends/p4tools/modules/testgen/lib/test_backend.cpp

@@ -140,7 +140,16 @@ bool TestBackEnd::run(const FinalState &state) {
 
         // Output the test.
         Util::withTimer("backend", [this, &testSpec, &selectedBranches] {
-            testWriter->outputTest(testSpec, selectedBranches, testCount, coverage);
+            if (testWriter->isInFileMode()) {
+                testWriter->writeTestToFile(testSpec, selectedBranches, testCount, coverage);
+            } else {
+                auto testOpt =
+                    testWriter->produceTest(testSpec, selectedBranches, testCount, coverage);
+                if (!testOpt.has_value()) {
+                    BUG("Failed to produce test.");
+                }
+                tests.push_back(testOpt.value());
+            }
         });
 
         printTraces("============ End Test %1% ============\n", testCount);

backends/p4tools/modules/testgen/lib/test_backend.h

@@ -48,6 +48,9 @@ class TestBackEnd {
     /// The accumulated coverage of all finished test cases. Number in range [0, 1].
     float coverage = 0;
 
+    /// The list of tests accumulated in the test back end.
+    AbstractTestList tests;
+
     explicit TestBackEnd(const ProgramInfo &programInfo,
                          const TestBackendConfiguration &testBackendConfiguration,
                          SymbolicExecutor &symbex)
@@ -133,6 +136,10 @@ class TestBackEnd {
 
     /// Returns the configuration options for the test back end.
     [[nodiscard]] const TestBackendConfiguration &getTestBackendConfiguration() const;
+
+    /// Returns the list of tests accumulated in the test back end.
+    /// If the test write is in file mode this list will be empty.
+    [[nodiscard]] const AbstractTestList &getTests() const { return tests; }
 };
 
 }  // namespace P4Tools::P4Testgen

backends/p4tools/modules/testgen/lib/test_framework.cpp

@@ -1,5 +1,7 @@
 #include "backends/p4tools/modules/testgen/lib/test_framework.h"
 
+#include "backends/p4tools/modules/testgen/lib/exceptions.h"
+
 namespace P4Tools::P4Testgen {
 
 TestFramework::TestFramework(const TestBackendConfiguration &testBackendConfiguration)
@@ -8,4 +10,16 @@ TestFramework::TestFramework(const TestBackendConfiguration &testBackendConfigur
 const TestBackendConfiguration &TestFramework::getTestBackendConfiguration() const {
     return testBackendConfiguration.get();
 }
+
+bool TestFramework::isInFileMode() const {
+    return getTestBackendConfiguration().fileBasePath.has_value();
+}
+
+AbstractTestReferenceOrError TestFramework::produceTest(const TestSpec * /*spec*/,
+                                                        cstring /*selectedBranches*/,
+                                                        size_t /*testIdx*/,
+                                                        float /*currentCoverage*/) {
+    TESTGEN_UNIMPLEMENTED("produceTest() not implemented for this test framework.");
+}
+
 }  // namespace P4Tools::P4Testgen

backends/p4tools/modules/testgen/lib/test_framework.h

@@ -2,15 +2,19 @@
 #define BACKENDS_P4TOOLS_MODULES_TESTGEN_LIB_TEST_FRAMEWORK_H_
 
 #include <cstddef>
+#include <filesystem>
+#include <functional>
 #include <iosfwd>
 #include <map>
+#include <optional>
 #include <string>
 #include <utility>
 
 #include <inja/inja.hpp>
 
 #include "backends/p4tools/common/lib/format_int.h"
 #include "backends/p4tools/common/lib/trace_event.h"
+#include "lib/castable.h"
 #include "lib/cstring.h"
 
 #include "backends/p4tools/modules/testgen/lib/test_backend_configuration.h"
@@ -19,6 +23,18 @@
 
 namespace P4Tools::P4Testgen {
 
+/// Type definitions for abstract tests.
+struct AbstractTest : ICastable {};
+/// TODO: It would be nice if this were a reference to signal non-nullness.
+/// Consider using an optional_ref implementation.
+using AbstractTestReference = const AbstractTest *;
+using AbstractTestReferenceOrError = std::optional<AbstractTestReference>;
+using AbstractTestList = std::vector<AbstractTestReference>;
+
+/// An file path which may or may not be set. Can influence the execution behavior the test
+/// framework.
+using OptionalFilePath = std::optional<std::filesystem::path>;
+
 /// THe default base class for the various test frameworks. Every test framework has a test
 /// name and a seed associated with it. Also contains a variety of common utility functions.
 class TestFramework {
@@ -136,13 +152,25 @@ class TestFramework {
 
     /// The method used to output the test case to be implemented by
     /// all the test frameworks (eg. STF, PTF, etc.).
-    /// @param spec the testcase specification to be outputted
-    /// @param selectedBranches string describing branches selected for this testcase
-    /// @param testIdx testcase unique number identifier
+    /// @param spec the testcase specification to be outputted.
+    /// @param selectedBranches string describing branches selected for this testcase.
+    /// @param testIdx testcase unique number identifier. TODO: Make this a member?
     /// @param currentCoverage current coverage ratio (between 0.0 and 1.0)
-    // attaches arbitrary string data to the test preamble.
-    virtual void outputTest(const TestSpec *spec, cstring selectedBranches, size_t testIdx,
-                            float currentCoverage) = 0;
+    /// TODO (https://github.com/p4lang/p4c/issues/4403): This should not return void but instead a
+    /// status.
+    virtual void writeTestToFile(const TestSpec *spec, cstring selectedBranches, size_t testIdx,
+                                 float currentCoverage) = 0;
+
+    /// The method used to return the test case. This method is optional to each test framework.
+    /// @param spec the testcase specification to be outputted.
+    /// @param selectedBranches string describing branches selected for this testcase.
+    /// @param testIdx testcase unique number identifier. TODO: Make this a member?
+    /// @param currentCoverage current coverage ratio (between 0.0 and 1.0).
+    virtual AbstractTestReferenceOrError produceTest(const TestSpec *spec, cstring selectedBranches,
+                                                     size_t testIdx, float currentCoverage);
+
+    /// @Returns true if the test framework is configured to write to a file.
+    [[nodiscard]] bool isInFileMode() const;
 };
 
 }  // namespace P4Tools::P4Testgen

backends/p4tools/modules/testgen/options.cpp

@@ -2,7 +2,6 @@
 
 #include <algorithm>
 #include <cstdlib>
-#include <iostream>
 #include <iterator>
 #include <map>
 #include <stdexcept>
@@ -12,7 +11,6 @@
 #include "backends/p4tools/common/lib/util.h"
 #include "backends/p4tools/common/options.h"
 #include "lib/error.h"
-#include "lib/exceptions.h"
 
 #include "backends/p4tools/modules/testgen/lib/logging.h"
 

backends/p4tools/modules/testgen/targets/bmv2/CMakeLists.txt

@@ -32,6 +32,7 @@ set(TESTGEN_SOURCES
 
 set(TESTGEN_GTEST_SOURCES
   ${TESTGEN_GTEST_SOURCES}
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/testgen_api/api_test.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/test_backend/ptf.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/test_backend/stf.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/small-step/binary.cpp

backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/ptf.cpp

@@ -83,7 +83,7 @@ TEST_F(PTFTest, Ptf01) {
 
     TestBackendConfiguration testBackendConfiguration{"test01", 1, "test01", 1};
     auto testWriter = PTF(testBackendConfiguration);
-    testWriter.outputTest(&testSpec, "", 1, 0);
+    testWriter.writeTestToFile(&testSpec, "", 1, 0);
 }
 
 /// Create a test spec with two Exact matches and print an ptf test.
@@ -114,7 +114,7 @@ TEST_F(PTFTest, Ptf02) {
 
     TestBackendConfiguration testBackendConfiguration{"test02", 1, "test02", 2};
     auto testWriter = PTF(testBackendConfiguration);
-    testWriter.outputTest(&testSpec, "", 2, 0);
+    testWriter.writeTestToFile(&testSpec, "", 2, 0);
 }
 
 TableConfig PTFTest::gettest1TableConfig() {
@@ -160,7 +160,7 @@ TEST_F(PTFTest, Ptf03) {
     TestBackendConfiguration testBackendConfiguration{"test03", 1, "test03", 3};
     auto testWriter = PTF(testBackendConfiguration);
     try {
-        testWriter.outputTest(&testSpec, "", 3, 0);
+        testWriter.writeTestToFile(&testSpec, "", 3, 0);
     } catch (const Util::CompilerBug &e) {
         EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch"));
     }
@@ -228,7 +228,7 @@ TEST_F(PTFTest, Ptf04) {
     TestBackendConfiguration testBackendConfiguration{"test04", 1, "test04", 4};
     auto testWriter = PTF(testBackendConfiguration);
     try {
-        testWriter.outputTest(&testSpec, "", 4, 0);
+        testWriter.writeTestToFile(&testSpec, "", 4, 0);
     } catch (const Util::CompilerBug &e) {
         EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch"));
     }

backends/p4tools/modules/testgen/targets/bmv2/test/test_backend/stf.cpp

@@ -83,7 +83,7 @@ TEST_F(STFTest, Stf01) {
 
     TestBackendConfiguration testBackendConfiguration{"test01", 1, "test01", 1};
     auto testWriter = STF(testBackendConfiguration);
-    testWriter.outputTest(&testSpec, "", 1, 0);
+    testWriter.writeTestToFile(&testSpec, "", 1, 0);
 }
 
 /// Create a test spec with two Exact matches and print an stf test.
@@ -114,7 +114,7 @@ TEST_F(STFTest, Stf02) {
 
     TestBackendConfiguration testBackendConfiguration{"test02", 1, "test02", 2};
     auto testWriter = STF(testBackendConfiguration);
-    testWriter.outputTest(&testSpec, "", 2, 0);
+    testWriter.writeTestToFile(&testSpec, "", 2, 0);
 }
 
 TableConfig STFTest::gettest1TableConfig() {
@@ -160,7 +160,7 @@ TEST_F(STFTest, Stf03) {
     TestBackendConfiguration testBackendConfiguration{"test03", 1, "test03", 3};
     auto testWriter = STF(testBackendConfiguration);
     try {
-        testWriter.outputTest(&testSpec, "", 3, 0);
+        testWriter.writeTestToFile(&testSpec, "", 3, 0);
     } catch (const Util::CompilerBug &e) {
         EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch"));
     }
@@ -228,7 +228,7 @@ TEST_F(STFTest, Stf04) {
     TestBackendConfiguration testBackendConfiguration{"test04", 1, "test04", 4};
     auto testWriter = STF(testBackendConfiguration);
     try {
-        testWriter.outputTest(&testSpec, "", 4, 0);
+        testWriter.writeTestToFile(&testSpec, "", 4, 0);
     } catch (const Util::CompilerBug &e) {
         EXPECT_THAT(e.what(), HasSubstr("Unimplemented for Ternary FieldMatch"));
     }