infra: refine fuzztest work (google#8992)

Enable Fuzztest fuzzers for Tensorflow.

This depends on google/fuzztest#79 and
eventually a PR on the tensorflow repo with the (to be refined) diff in
this PR.

Signed-off-by: David Korczynski <david@adalogics.com>

Signed-off-by: David Korczynski <david@adalogics.com>
Co-authored-by: Oliver Chang <oliverchang@users.noreply.github.com>

infra/base-images/base-builder/compile_fuzztests.sh

@@ -15,24 +15,42 @@
 #
 ################################################################################
 
+set -x
+
+# In order to identify fuzztest test case "bazel query" is used to search
+# the project. A search of the entire project is done with a default "...",
+# however, some projects may fail to, or have very long processing time, if
+# searching the entire project. Additionally, it may include fuzzers in
+# dependencies, which should not be build as part of a given project.
+# Tensorflow is an example project that will fail when the entire project is
+# queried. FUZZTEST_TARGET_FOLDER makes it posible to specify the folder
+# where fuzztest fuzzers should be search for. FUZZTEST_TARGET_FOLDER is passed
+# to "bazel query" below.
+if [[ ${FUZZTEST_TARGET_FOLDER:-"unset"} == "unset" ]];
+then
+  export TARGET_FOLDER="..."
+else
+  TARGET_FOLDER=${FUZZTEST_TARGET_FOLDER}
+fi
+
+BUILD_ARGS="--config=oss-fuzz --subcommands"
+if [[ ${FUZZTEST_EXTRA_ARGS:-"unset"} != "unset" ]];
+then
+  BUILD_ARGS="$BUILD_ARGS ${FUZZTEST_EXTRA_ARGS}"
+fi
+
 # Trigger setup_configs rule of fuzztest as it generates the necessary
 # configuration file based on OSS-Fuzz environment variables.
-bazel run @com_google_fuzztest//bazel:setup_configs > fuzztest.bazelrc
+bazel run @com_google_fuzztest//bazel:setup_configs >> /etc/bazel.bazelrc
 
 # Bazel target names of the fuzz binaries.
-FUZZ_TEST_BINARIES=$(bazel query 'kind("cc_test", rdeps(..., @com_google_fuzztest//fuzztest:fuzztest_gtest_main))')
+FUZZ_TEST_BINARIES=$(bazel query "kind(\"cc_test\", rdeps(${TARGET_FOLDER}, @com_google_fuzztest//fuzztest:fuzztest_gtest_main))")
 
 # Bazel output paths of the fuzz binaries.
-FUZZ_TEST_BINARIES_OUT_PATHS=$(bazel cquery 'kind("cc_test", rdeps(..., @com_google_fuzztest//fuzztest:fuzztest_gtest_main))' --output=files)
-
-#echo "Fuzz test binaries:"
-#echo ${FUZZ_TEST_BINARIES}
-#echo "-----------------------------"
-#echo ${FUZZ_TEST_BINARIES3}
-#echo "-----------------------------"
+FUZZ_TEST_BINARIES_OUT_PATHS=$(bazel cquery "kind(\"cc_test\", rdeps(${TARGET_FOLDER}, @com_google_fuzztest//fuzztest:fuzztest_gtest_main))" --output=files)
 
 # Build the project and fuzz binaries
-bazel build --subcommands --config=oss-fuzz ${FUZZ_TEST_BINARIES[*]}
+bazel build $BUILD_ARGS -- ${FUZZ_TEST_BINARIES[*]}
 
 # Iterate the fuzz binaries and list each fuzz entrypoint in the binary. For
 # each entrypoint create a wrapper script that calls into the binaries the
@@ -52,13 +70,22 @@ for fuzz_main_file in $FUZZ_TEST_BINARIES_OUT_PATHS; do
 # LLVMFuzzerTestOneInput for fuzzer detection.
 this_dir=\$(dirname \"\$0\")
 chmod +x \$this_dir/$fuzz_basename
-$fuzz_basename --fuzz=$fuzz_entrypoint -- \$@" > $OUT/$TARGET_FUZZER
+\$this_dir/$fuzz_basename --fuzz=$fuzz_entrypoint -- \$@" > $OUT/$TARGET_FUZZER
     chmod +x $OUT/$TARGET_FUZZER
   done
 done
 
-# synchronise coverage directory to bazel generated code.
-if [ "$SANITIZER" = "coverage" ]
+# Synchronise coverage directory to bazel output artifacts. This is a
+# best-effort basis in that it will include source code in common
+# bazel output folders.
+# For projects that store results in non-standard folders or want to
+# manage what code to include in the coverage report more specifically,
+# the FUZZTEST_DO_SYNC environment variable is made available. Projects
+# can then implement a custom way of synchronising source code with the
+# coverage build. Set FUZZTEST_DO_SYNC to something other than "yes" and
+# no effort will be made to automatically synchronise the source code with
+# the code coverage visualisation utility.
+if [[ "$SANITIZER" = "coverage" && ${FUZZTEST_DO_SYNC:-"yes"} == "yes" ]]
 then 
   # Synchronize bazel source files to coverage collection.
   declare -r REMAP_PATH="${OUT}/proc/self/cwd"
@@ -68,16 +95,25 @@ then
   declare -r RSYNC_FILTER_ARGS=("--include" "*.h" "--include" "*.cc" "--include" \
     "*.hpp" "--include" "*.cpp" "--include" "*.c" "--include" "*/" "--include" "*.inc" \
     "--exclude" "*")
+
   project_folders="$(find . -name 'bazel-*' -type l -printf '%P\n' | \
                     grep -v -x -F \
                     -e 'bazel-bin' \
-                    -e 'bazel-out' \
                     -e 'bazel-testlogs')"
   for link in $project_folders; do
    if [[ -d "${PWD}"/$link/external  ]]
    then
      rsync -avLk "${RSYNC_FILTER_ARGS[@]}" "${PWD}"/$link/external "${REMAP_PATH}"
    fi
+   # k8-opt is a common path for storing bazel output artifacts, e.g. bazel-out/k8-opt.
+   # It's the output folder for default amd-64 builds, but projects may specify custom
+   # platform output directories, see: https://github.com/bazelbuild/bazel/issues/13818
+   # We support the default at the moment, and if a project needs custom synchronizing of
+   # output artifacts and code coverage we currently recommend using FUZZTEST_DO_SYNC.
+   if [[ -d "${PWD}"/$link/k8-opt  ]]
+   then
+     rsync -avLk "${RSYNC_FILTER_ARGS[@]}" "${PWD}"/$link/k8-opt "${REMAP_PATH}"/$link
+   fi
   done
 
   # Delete symlinks and sync the current folder.