From d72605436dbe28fbcff7a9d72c294a754807c420 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Tue, 6 Aug 2019 15:32:55 -0700 Subject: [PATCH] chore(flakiness): update flakiness format (#4808) This patch: - updates Flakiness Dashboard format to define version per-build and to pass COMMIT information - drops the README.md generation - we'll move on to a designated flakiness dashboard viewer --- .appveyor.yml | 1 - .cirrus.yml | 1 - test/test.js | 6 +- test/utils.js | 12 +- .../flakiness-dashboard/FlakinessDashboard.js | 159 +++++------------- 5 files changed, 56 insertions(+), 123 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 551462e60ac41..d093148397596 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,7 +10,6 @@ environment: build: off install: - - ps: $env:FLAKINESS_DASHBOARD_BUILD_SHA="$env:APPVEYOR_REPO_COMMIT" - ps: $env:FLAKINESS_DASHBOARD_BUILD_URL="https://ci.appveyor.com/project/aslushnikov/puppeteer/branch/master/job/$env:APPVEYOR_JOB_ID" - ps: Install-Product node $env:nodejs_version - npm install diff --git a/.cirrus.yml b/.cirrus.yml index 353933cb03760..d0d00b6167f08 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,7 +2,6 @@ env: DISPLAY: :99.0 FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992] FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME} - FLAKINESS_DASHBOARD_BUILD_SHA: ${CIRRUS_CHANGE_IN_REPO} FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID} task: diff --git a/test/test.js b/test/test.js index c9ee901dee59e..8e70d98886fb8 100644 --- a/test/test.js +++ b/test/test.js @@ -111,6 +111,8 @@ new Reporter(testRunner, { showSlowTests: process.env.CI ? 5 : 0, }); -utils.initializeFlakinessDashboardIfNeeded(testRunner); -testRunner.run(); +(async() => { + await utils.initializeFlakinessDashboardIfNeeded(testRunner); + testRunner.run(); +})(); diff --git a/test/utils.js b/test/utils.js index f407f59b5bb44..fbc5528ccd169 100644 --- a/test/utils.js +++ b/test/utils.js @@ -161,7 +161,7 @@ const utils = module.exports = { }); }, - initializeFlakinessDashboardIfNeeded: function(testRunner) { + initializeFlakinessDashboardIfNeeded: async function(testRunner) { // Generate testIDs for all tests and verify they don't clash. // This will add |test.testId| for every test. // @@ -181,18 +181,22 @@ const utils = module.exports = { // CIRRUS_BASE_SHA env variables. if (!process.env.FLAKINESS_DASHBOARD_PASSWORD || process.env.CIRRUS_BASE_SHA) return; - const sha = process.env.FLAKINESS_DASHBOARD_BUILD_SHA; + const {sha, timestamp} = await FlakinessDashboard.getCommitDetails(__dirname, 'HEAD'); const dashboard = new FlakinessDashboard({ - dashboardName: process.env.FLAKINESS_DASHBOARD_NAME, + commit: { + sha, + timestamp, + url: `https://github.com/GoogleChrome/puppeteer/commit/${sha}`, + }, build: { url: process.env.FLAKINESS_DASHBOARD_BUILD_URL, - name: sha.substring(0, 8), }, dashboardRepo: { url: 'https://github.com/aslushnikov/puppeteer-flakiness-dashboard.git', username: 'puppeteer-flakiness', email: 'aslushnikov+puppeteerflakiness@gmail.com', password: process.env.FLAKINESS_DASHBOARD_PASSWORD, + branch: process.env.FLAKINESS_DASHBOARD_NAME, }, }); diff --git a/utils/flakiness-dashboard/FlakinessDashboard.js b/utils/flakiness-dashboard/FlakinessDashboard.js index 908cc4b5d8b5d..0f7646f5a97b3 100644 --- a/utils/flakiness-dashboard/FlakinessDashboard.js +++ b/utils/flakiness-dashboard/FlakinessDashboard.js @@ -16,43 +16,51 @@ const GREEN_COLOR = '\x1b[32m'; const YELLOW_COLOR = '\x1b[33m'; const RESET_COLOR = '\x1b[0m'; +const DASHBOARD_VERSION = 1; +const DASHBOARD_FILENAME = 'dashboard.json'; + class FlakinessDashboard { - constructor({dashboardName, build, dashboardRepo, options}) { - if (!dashboardName) - throw new Error('"options.dashboardName" must be specified!'); + static async getCommitDetails(repoPath, ref = 'HEAD') { + const {stdout: timestamp} = await spawnAsyncOrDie('git', 'show', '-s', '--format=%ct', ref, {cwd: repoPath}); + const {stdout: sha} = await spawnAsyncOrDie('git', 'rev-parse', ref, {cwd: repoPath}); + return {timestamp: timestamp * 1000, sha: sha.trim()}; + } + + constructor({build, commit, dashboardRepo}) { + if (!commit) + throw new Error('"options.commit" must be specified!'); + if (!commit.sha) + throw new Error('"options.commit.sha" must be specified!'); + if (!commit.timestamp) + throw new Error('"options.commit.timestamp" must be specified!'); if (!build) throw new Error('"options.build" must be specified!'); if (!build.url) throw new Error('"options.build.url" must be specified!'); - if (!build.name) - throw new Error('"options.build.name" must be specified!'); - this._dashboardName = dashboardName; + if (!dashboardRepo.branch) + throw new Error('"options.dashboardRepo.branch" must be specified!'); this._dashboardRepo = dashboardRepo; - this._options = options; - this._build = new Build(Date.now(), build.name, build.url, []); + this._build = new Build(Date.now(), build.url, commit, []); } reportTestResult(test) { - this._build.reportTestResult(test); + this._build._tests.push(test); } async uploadAndCleanup() { console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`); const startTimestamp = Date.now(); - const branch = this._dashboardRepo.branch || this._dashboardName.trim().toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, ''); + const branch = this._dashboardRepo.branch.toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, ''); console.log(` > Dashboard URL: ${this._dashboardRepo.url}`); console.log(` > Dashboard Branch: ${branch}`); const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password); console.log(` > Dashboard Checkout: ${git.path()}`); - // Do at max 5 attempts to upload changes to github. + // Do at max 7 attempts to upload changes to github. let success = false; const MAX_ATTEMPTS = 7; for (let i = 0; !success && i < MAX_ATTEMPTS; ++i) { - const dashboard = await Dashboard.create(this._dashboardName, git.path(), this._options); - dashboard.addBuild(this._build); - await dashboard.saveJSON(); - await dashboard.generateReadme(); + await saveBuildToDashboard(git.path(), this._build); // if push went through - great! We're done! if (await git.commitAllAndPush(`update dashboard\n\nbuild: ${this._build._url}`)) { success = true; @@ -77,112 +85,33 @@ class FlakinessDashboard { } } -const DASHBOARD_VERSION = 1; -const DASHBOARD_FILENAME = 'dashboard.json'; - -class Dashboard { - static async create(name, dashboardPath, options = {}) { - const filePath = path.join(dashboardPath, DASHBOARD_FILENAME); - let data = null; - try { - data = JSON.parse(await readFileAsync(filePath)); - } catch (e) { - // Looks like there's no dashboard yet - create one. - return new Dashboard(name, dashboardPath, [], options); - } - if (!data.version) - throw new Error('cannot parse dashboard data: missing "version" field!'); - if (data.version > DASHBOARD_VERSION) - throw new Error('cannot manage dashboards that are newer then this'); - const builds = data.builds.map(build => new Build(build.timestamp, build.name, build.url, build.tests)); - return new Dashboard(name, dashboardPath, builds, options); - } - - async saveJSON() { - const data = { version: DASHBOARD_VERSION }; - data.builds = this._builds.map(build => ({ - timestamp: build._timestamp, - name: build._name, - url: build._url, - tests: build._tests, - })); - await writeFileAsync(path.join(this._dashboardPath, DASHBOARD_FILENAME), JSON.stringify(data)); - } - - async generateReadme() { - const flakyTests = new Map(); - for (const build of this._builds) { - for (const test of build._tests) { - if (test.result !== 'ok') - flakyTests.set(test.testId, test); - } - } - - const text = []; - text.push(`# ${this._name}`); - text.push(``); - - for (const [testId, test] of flakyTests) { - text.push(`#### [${test.name}](${test.url}) - ${test.description}`); - text.push(''); - - let headers = '|'; - let splitters = '|'; - let dataColumns = '|'; - for (let i = this._builds.length - 1; i >= 0; --i) { - const build = this._builds[i]; - headers += ` [${build._name}](${build._url}) |`; - splitters += ' :---: |'; - const test = build._testsMap.get(testId); - if (test) { - const r = test.result.toLowerCase(); - let text = r; - if (r === 'ok') - text = '✅'; - else if (r.includes('fail')) - text = '🛑'; - dataColumns += ` [${text}](${test.url}) |`; - } else { - dataColumns += ` missing |`; - } - } - text.push(headers); - text.push(splitters); - text.push(dataColumns); - text.push(''); - } - - await writeFileAsync(path.join(this._dashboardPath, 'README.md'), text.join('\n')); - } - - constructor(name, dashboardPath, builds, options) { - const { - maxBuilds = 100, - } = options; - this._name = name; - this._dashboardPath = dashboardPath; - this._builds = builds.slice(builds.length - maxBuilds); - } - - addBuild(build) { - this._builds.push(build); - } +async function saveBuildToDashboard(dashboardPath, build) { + const filePath = path.join(dashboardPath, DASHBOARD_FILENAME); + let data = null; + try { + data = JSON.parse(await readFileAsync(filePath)); + } catch (e) { + // Looks like there's no dashboard yet - create one. + data = {builds: []}; + } + if (!data.builds) + throw new Error('Unrecognized dashboard format!'); + data.builds.push({ + version: DASHBOARD_VERSION, + timestamp: build._timestamp, + url: build._url, + commit: build._commit, + tests: build._tests, + }); + await writeFileAsync(filePath, JSON.stringify(data)); } class Build { - constructor(timestamp, name, url, tests) { + constructor(timestamp, url, commit, tests) { this._timestamp = timestamp; - this._name = name; this._url = url; + this._commit = commit; this._tests = tests; - this._testsMap = new Map(); - for (const test of tests) - this._testsMap.set(test.testId, test); - } - - reportTestResult(test) { - this._tests.push(test); - this._testsMap.set(test.testId, test); } }