Hide useless logs and add useful ones (#31)

Hide numerous download progress logs from `wget2` and `gsutil`.
Add logs for cloud experiments to look for anomalies.

experiment/builder_runner.py

@@ -354,7 +354,11 @@ def build_and_run(self, generated_project: str, target_path: str,
              check=True,
              cwd=oss_fuzz_checkout.OSS_FUZZ_DIR)
     except sp.CalledProcessError as e:
-      print('Failed to evaluate target on cloud:', e.stdout, e.stderr)
+      print(f'Failed to evaluate {os.path.realpath(target_path)} on cloud:',
+            e.stdout, e.stderr)
+      return build_result, None
+
+    print(f'Evaluated {os.path.realpath(target_path)} on cloud.')
 
     storage_client = storage.Client()
     bucket = storage_client.bucket(self.experiment_bucket)
@@ -367,19 +371,33 @@ def build_and_run(self, generated_project: str, target_path: str,
         'wb') as f:
       blob = bucket.blob(build_log_name)
       if blob.exists():
+        print(f'Downloading cloud build log of {os.path.realpath(target_path)}:'
+              f' {build_log_name} to {f}')
         blob.download_to_file(f)
+      else:
+        print(f'Cannot find cloud build log of {os.path.realpath(target_path)} '
+              f':{build_log_name}')
 
     with open(self.work_dirs.run_logs_target(generated_target_name), 'wb') as f:
       blob = bucket.blob(run_log_name)
       if blob.exists():
         build_result.succeeded = True
+        print(f'Downloading cloud run log of {os.path.realpath(target_path)}:'
+              f' {run_log_name} to {f}')
         blob.download_to_file(f)
+      else:
+        print(f'Cannot find cloud run log of {os.path.realpath(target_path)} '
+              f':{run_log_name}')
 
     if not build_result.succeeded:
       errors = code_fixer.extract_error_message(
           self.work_dirs.build_logs_target(generated_target_name, iteration))
       build_result.errors = errors
+      print(f'Cloud evaluation of {os.path.realpath(target_path)} indicates a '
+            f'failure: {errors}')
       return build_result, None
+    print(f'Cloud evaluation of {os.path.realpath(target_path)} indicates a '
+          'success.')
 
     corpus_dir = self.work_dirs.corpus(generated_target_name)
     with open(os.path.join(corpus_dir, 'corpus.zip'), 'wb') as f:

report/docker_run.sh

@@ -35,7 +35,7 @@ RUN_TIMEOUT=$3
 # When running on GCP this step is unnecessary.
 if [[ $GOOGLE_APPLICATION_CREDENTIALS != '' ]]
 then
-  gcloud auth activate-service-account LLM-EVAL@oss-fuzz.iam.gserviceaccount.com --key-file=${GOOGLE_APPLICATION_CREDENTIALS:?}
+  gcloud auth activate-service-account LLM-EVAL@oss-fuzz.iam.gserviceaccount.com --key-file="${GOOGLE_APPLICATION_CREDENTIALS:?}"
 fi
 
 if [[ $BENCHMARK_SET = '' ]]
@@ -69,17 +69,26 @@ EXPERIMENT_NAME="${DATE:?}-${FREQUENCY_LABEL:?}-${BENCHMARK_SET:?}"
 GCS_REPORT_DIR=${EXPERIMENT_NAME:?}
 
 # Generate a report and upload it to GCS
-bash report/upload_report.sh ${LOCAL_RESULTS_DIR:?} ${GCS_REPORT_DIR:?} &
+bash report/upload_report.sh "${LOCAL_RESULTS_DIR:?}" "${GCS_REPORT_DIR:?}" &
+pid_report=$!
 
 # Run the experiment
 /venv/bin/python3 run_all_experiments.py \
   --benchmarks-directory "benchmark-sets/${BENCHMARK_SET:?}" \
-  --run-timeout ${RUN_TIMEOUT:?} \
-  --cloud-experiment-name ${EXPERIMENT_NAME:?} \
+  --run-timeout "${RUN_TIMEOUT:?}" \
+  --cloud-experiment-name "${EXPERIMENT_NAME:?}" \
   --cloud-experiment-bucket 'oss-fuzz-gcb-experiment-run-logs' \
   --template-directory 'prompts/template_xml' \
   --work-dir ${LOCAL_RESULTS_DIR:?} \
   --num-samples 10 \
   --mode 'vertex_ai_code-bison-32k'
 
-echo 1 > /experiment_status
+export ret_val=$?
+
+touch /experiment_ended
+
+# Wait for the report process to finish uploading.
+wait $pid_report
+
+# Exit with the return value of `./run_all_experiments`.
+exit $ret_val

report/upload_report.sh

@@ -41,36 +41,45 @@ then
   echo "GCS directory was not specified as the second argument. Defaulting to ${GCS_DIR:?}."
 fi
 
-# Spin up the web server generating the report (and bg the process)
-PYTHONPATH=. /venv/bin/python3 report/web.py ${RESULTS_DIR:?} ${WEB_PORT:?} &
+mkdir results-report
 
 while true; do
-  mkdir results-report
-  cd results-report
+  # Spin up the web server generating the report (and bg the process).
+  PYTHONPATH=. /venv/bin/python3 report/web.py "${RESULTS_DIR:?}" "${WEB_PORT:?}" &
+  pid_web=$!
 
-  # Recursively get all the experiment results
-  wget2 --inet4-only --no-host-directories --http2-request-window 10 --recursive localhost:${WEB_PORT:?}/
+  cd results-report || exit 1
 
-  # Also fetch the sorted line cov diff report
-  wget2 --inet4-only localhost:${WEB_PORT:?}/sort -O sort.html
+  # Recursively get all the experiment results.
+  echo "Download results from localhost."
+  wget2 --quiet --inet4-only --no-host-directories --http2-request-window 10 --recursive localhost:${WEB_PORT:?}/ 2>&1
 
-  # Upload the report to GCS
-  gsutil -m -h "Content-Type:text/html" \
+  # Also fetch the sorted line cov diff report.
+  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort -O sort.html 2>&1
+
+  # Stop the server.
+  kill -9 "$pid_web"
+
+  # Upload the report to GCS.
+  echo "Uploading the report."
+  gsutil -q -m -h "Content-Type:text/html" \
          -h "Cache-Control:public, max-age=3600" \
-         cp -r . gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}
+         cp -r . "gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}"
 
   cd ..
 
   # Upload the raw results into the same GCS directory
-  gsutil -m cp -r ${RESULTS_DIR:?} \
-         gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}
+  echo "Uploading the raw results."
+  gsutil -q -m cp -r "${RESULTS_DIR:?}" \
+         "gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}"
 
   echo "See the published report at https://llm-exp.oss-fuzz.com/Result-reports/${GCS_DIR:?}/"
-  if [[ -f /experiment_status ]] && [[ "$(cat /experiment_status)" == "1" ]]; then
+
+  if [[ -f /experiment_ended ]]; then
     echo "Experiment finished."
-    break
-  else
-    echo "Experiment is running..."
-    sleep 600
+    exit
   fi
+
+  echo "Experiment is running..."
+  sleep 600
 done

report/web.py

@@ -15,6 +15,7 @@
 
 import dataclasses
 import json
+import logging
 import os
 import re
 import sys
@@ -26,6 +27,9 @@
 from experiment import evaluator
 
 app = Flask(__name__)
+# Disable Flask request logs
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)
 
 RESULTS_DIR = ''