Merge pull request #89 from JohT/fix/visualization-timeout

Fix JavaScript Graph Visualization by running it after all CSV reports that write data into the Graph

Author: JohT

.github/workflows/code-structure-analysis.yml

@@ -141,6 +141,7 @@ jobs:
         name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
         path: |
           ./temp/**/runtime/*
+          ./temp/**/reports/*
         retention-days: 5
 
     # Upload successful results in case they are needed for troubleshooting

graph-visualization/renderVisualizations.js

@@ -13,7 +13,7 @@ console.log(`renderVisualizations.js: dirname=${__dirname}`);
 
 /**
  * Crops the image in the buffer so that there is no empty frame around it.
- * @param {Buffer} buffer 
+ * @param {Buffer} buffer
  * @returns Buffer
  */
 const autoCropImageBuffer = async (buffer) => {
@@ -31,41 +31,72 @@ const autoCropImageBuffer = async (buffer) => {
  */
 const camelToKebabCase = (str) => str.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`);
 
+/**
+ * Take a screenshot after an error happened.
+ *
+ * @param {string} htmlFilename
+ * @param {string} reason
+ */
+const makeScreenshotOfError = async (page, htmlFilename, reason) => {
+  const reportName = basename(htmlFilename, ".html");
+  const directoryName = camelToKebabCase(reportName);
+  console.log(`Taking an error screenshot of report ${reportName}. Reason: ${reason}`);
+  await page.screenshot({ path: `./${directoryName}/error-${reportName}-${reason}.png`, omitBackground: false });
+};
+
+/**
+ * Handle a catched error by taking a screenshot.
+ *
+ * @param {string} htmlFilename
+ * @param {string} reason
+ * @return
+ */
+const handleErrorWithScreenshot = (page, htmlFilename, reason) => async (error) => {
+  await makeScreenshotOfError(page, htmlFilename, reason);
+  throw new Error(error);
+};
 /**
  * Creates a new page and takes a screenshot of all canvas tags that are contained within a div.
  * Note: Don't run this in parallel. See https://github.com/puppeteer/puppeteer/issues/1479
  * @param {Browser} browser
  * @param {string} htmlFilename
  */
 const takeCanvasScreenshots = async (browser, htmlFilename) => {
+  const reportName = basename(htmlFilename, ".html");
+  const directoryName = camelToKebabCase(reportName);
+  if (!existsSync(directoryName)) {
+    mkdirSync(directoryName);
+  }
+
   const page = await browser.newPage();
+  page.on("requestfailed", (request) => console.log(`${request.failure().errorText} ${request.url()}`));
   await page.setViewport({ width: 1500, height: 1000, isMobile: false, isLandscape: true, hasTouch: false, deviceScaleFactor: 1 });
 
   console.log(`Loading ${htmlFilename}`);
   await page.goto(`file://${htmlFilename}`);
 
+  if (!process.env.NEO4J_INITIAL_PASSWORD) {
+    throw new Error("Missing environment variable NEO4J_INITIAL_PASSWORD");
+  }
   // Login with Neo4j server password from the environment variable NEO4J_INITIAL_PASSWORD
   const loginButton = await page.waitForSelector("#neo4j-server-login");
   await page.type("#neo4j-server-password", process.env.NEO4J_INITIAL_PASSWORD);
   await loginButton.click();
 
   // Wait for the graph visualization to be rendered onto a HTML5 canvas
   console.log(`Waiting for visualizations to be finished`);
-  await page.waitForSelector(".visualization-finished", { timeout: 90_000 });
+  await page
+    .waitForSelector(".visualization-finished", { timeout: 90_000 })
+    .catch(handleErrorWithScreenshot(page, htmlFilename, "visualization-did-not-finish"));
 
   // Get all HTML canvas tag elements
   const canvasElements = await page.$$("canvas");
   if (canvasElements.length <= 0) {
-    console.error(`No elements with CSS selector 'canvas' found in ${htmlFilename}`);
+    await makeScreenshotOfError(page, htmlFilename, "no-canvas-found");
   }
   console.log(`Found ${canvasElements.length} visualizations`);
 
   // Take a png screenshot of every canvas element and save them with increasing indices
-  const reportName = basename(htmlFilename, ".html");
-  const directoryName = camelToKebabCase(reportName);
-  if (!existsSync(directoryName)) {
-    mkdirSync(directoryName);
-  }
   await Promise.all(
     Array.from(canvasElements).map(async (canvasElement, index) => {
       console.log(`Exporting image ${reportName}-${index}.png...`);
@@ -74,10 +105,8 @@ const takeCanvasScreenshots = async (browser, htmlFilename) => {
       }, canvasElement);
       let data = Buffer.from(dataUrl.split(",").pop(), "base64");
       console.log(`Cropping image ${reportName}-${index}.png...`);
-      data = await autoCropImageBuffer(data);
+      data = await autoCropImageBuffer(data).catch(handleErrorWithScreenshot(page, htmlFilename, `failed-to-crop-image-${index}`));
       writeFileSync(`./${directoryName}/${reportName}-${index}.png`, data);
-      // console.log(`Taking screenshot ${reportName} of canvas ${index} in ${htmlFilename} of element...`);
-      // await canvasElement.screenshot({ path: `./${directoryName}/${reportName}-${index}.png`, omitBackground: true });
     })
   );
 };
@@ -89,17 +118,34 @@ let browser;
  * and takes a screenshot of the canvas elements using {@link takeCanvasScreenshots}.
  */
 (async () => {
-  console.log('renderVisualizations.js: Starting headless browser...');
-  browser = await puppeteer.launch({ headless: "new" }); // { headless: false } for testing
+  console.log("renderVisualizations.js: Starting headless browser...");
+  browser = await puppeteer.launch({
+    headless: "new",
+    dumpio: true,
+    args: [
+      "--enable-logging", 
+      "--disable-search-engine-choice-screen",
+      "--ash-no-nudges",
+      "--no-first-run",
+      "--no-default-browser-check",
+      "--hide-scrollbars",
+      "--disable-features=Translate",
+      "--disable-features=InterestFeedContentSuggestions",
+      "--disable-extensions",
+      "--disable-default-apps",
+      "--disable-component-extensions-with-background-pages",
+      "--disable-client-side-phishing-detection",
+      "--use-gl=disabled",
+      "--disable-features=Vulkan", 
+    ],
+  }); // { headless: false } for testing
 
   // Get all *.html files in this (script) directory and its subdirectories
-  // The separate filter is needed to ignore the "node_modules" directory. 
+  // The separate filter is needed to ignore the "node_modules" directory.
   // Glob's build-in filter doesn't seem to work on Windows.
-  const htmlFiles = globSync(`${__dirname}/**/*.html`, { absolute: true }).filter(file => !file.includes('node_modules'));
+  const htmlFiles = globSync(`${__dirname}/**/*.html`, { absolute: true }).filter((file) => !file.includes("node_modules"));
   for (const htmlFile of htmlFiles) {
     await takeCanvasScreenshots(browser, htmlFile);
   }
   console.log(`renderVisualizations.js: Successfully rendered ${htmlFiles.length} html file(s)`);
-})()
-  .catch((err) => console.error(err))
-  .finally(() => browser?.close());
+})().finally(() => browser?.close());

graph-visualization/visualization-pagination.js

@@ -26,11 +26,14 @@ function paginatedGraphVisualization({
   /**
    * Marks the given element as finished when the visualization is completed.
    * @param {Element} indexedVisualizationElement
+   * @param {string} logDescription
    */
-  function markVisualizationAsFinished(indexedVisualizationElement) {
+  function markVisualizationAsFinished(indexedVisualizationElement, logDescription) {
     indexedVisualizationElement.classList.add(classOfFinishedVisualization);
     const unfinishedVisualizations = document.querySelectorAll(`.${classOfIndexedVisualizationElement}:not(.${classOfFinishedVisualization})`);
     if (unfinishedVisualizations.length === 0) {
+      console.debug(`${logDescription}: Last visualization finished on element ${JSON.stringify(indexedVisualizationElement)}.`);
+      console.debug(`${logDescription}: Mark whole visualization as finished on parent element ${JSON.stringify( indexedVisualizationElement.parentElement)}`);
       indexedVisualizationElement.parentElement.classList.add(classOfFinishedVisualization);
     }
   }
@@ -48,19 +51,23 @@ function paginatedGraphVisualization({
 
     neoViz.registerOnEvent(NeoVis.NeoVisEvents.CompletionEvent, (event) => {
       if (event.recordCount == 0) {
+        if (index=0) {
+          log.error('No query results. Nothing to visualize. Check the query and if the nodes and properties have been written.')
+        }
         indexedVisualizationContainer.remove(); // remove an empty canvas
-        markVisualizationAsFinished(indexedVisualizationContainer);
+        markVisualizationAsFinished(indexedVisualizationContainer, 'No query results (anymore)');
       } else {
         setTimeout(() => {
           neoViz.stabilize();
-          markVisualizationAsFinished(indexedVisualizationContainer);
+          markVisualizationAsFinished(indexedVisualizationContainer, 'Visualization stabilized');
         }, 5000);
       }
     });
     neoViz.registerOnEvent(NeoVis.NeoVisEvents.ErrorEvent, (event) => {
       indexedVisualizationContainer.classList.add(classOfFailedVisualization);
       indexedVisualizationContainer.textContent = event.error.message;
-      markVisualizationAsFinished(indexedVisualizationContainer);
+      console.error(`Visualization Error: ${JSON.stringify(event.error)}`)
+      markVisualizationAsFinished(indexedVisualizationContainer, 'Error event');
     });
     const parameters = {
       blockSize: recordsPerVisualization,

scripts/reports/GraphVisualization.sh

@@ -33,4 +33,6 @@ if [ ! -d "${GRAPH_VISUALIZATION_DIRECTORY}/node_modules" ] ; then
 fi
 
 # Execute the node.js script to render the graph visualizations as image files
-(cd "${REPORTS_DIRECTORY}" && exec node "${GRAPH_VISUALIZATION_DIRECTORY}/renderVisualizations.js")
+(cd "${REPORTS_DIRECTORY}" && exec node "${GRAPH_VISUALIZATION_DIRECTORY}/renderVisualizations.js") 
+
+echo "GraphVisualization: Successfully finished"

scripts/reports/compilations/AllReports.sh

@@ -15,11 +15,10 @@ set -o errexit -o pipefail
 REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )}
 echo "AllReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}"
 
-REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")}
-echo "AllReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
-
-# Run all report scripts
-for report_script_file in "${REPORTS_SCRIPT_DIR}"/*.sh; do 
-    echo "AllReports: Starting ${report_script_file}..."; 
-    source "${report_script_file}"
-done
+# The reports will not be generically searched as files anymore
+# but will be processed in order. Especially the visualization
+# needs to be done as a last step to be able to use properties
+# and data written to the Graph in the CsvReports.
+source "${REPORT_COMPILATIONS_SCRIPT_DIR}/CsvReports.sh"
+source "${REPORT_COMPILATIONS_SCRIPT_DIR}/JupyterReports.sh"
+source "${REPORT_COMPILATIONS_SCRIPT_DIR}/VisualizationReports.sh"

scripts/reports/compilations/VisualizationReports.sh

@@ -6,6 +6,7 @@
 # Therefore these reports will take longer and require more ressources than just plain database queries/procedures.
 
 # Requires reports/*.sh
+# Needs to run after reports/TopologySortCsv.sh that provides the property "topologicalSortIndex" to be queried.
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail