From 9deca789af1ab0305fcceed5e9c91fd2ca9aed7f Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 14:57:35 +0800 Subject: [PATCH 1/9] update --- .github/workflows/_pytorch_pr_comment.yml | 70 +++++++++++++++++++++++ .github/workflows/ascend_npu_test.yml | 15 +++++ 2 files changed, 85 insertions(+) create mode 100644 .github/workflows/_pytorch_pr_comment.yml diff --git a/.github/workflows/_pytorch_pr_comment.yml b/.github/workflows/_pytorch_pr_comment.yml new file mode 100644 index 0000000..5c25562 --- /dev/null +++ b/.github/workflows/_pytorch_pr_comment.yml @@ -0,0 +1,70 @@ +name: '_pytorch_pr_comment' + +on: + workflow_call: + inputs: + repository: + required: false + type: string + default: 'pytorch/pytorch' + description: 'The full name of the repository containing the pull request' + pr-number: + required: true + type: number + description: 'The number of the pull request' + pr-comment-author: + required: true + type: string + description: 'The author of the pull request comment' + secrets: + pr-token: + required: true + description: 'A token used to create or update a pull request comment' + +jobs: + comment: + name: Create or update the pull request comment + runs-on: ubuntu-latest + steps: + # Find the first comment containing the specified string + # and by the specified author. + # See: https://github.com/peter-evans/find-comment + - name: Find comment + uses: peter-evans/find-comment@v3 + id: fc + with: + token: ${{ secrets.pr-token }} + repository: ${{ inputs.repository }} + issue-number: ${{ inputs.pr-number }} + comment-author: ${{ inputs.pr-comment-author }} + body-includes: '' + direction: first + + - name: Show comment info + run: | + echo "comment id: ${{ steps.fc.outputs.comment-id }}" + echo "comment node id: ${{ steps.fc.outputs.comment-node-id }}" + echo "comment body: ${{ steps.fc.outputs.comment-body }}" + echo "comment author: ${{ steps.fc.outputs.comment-author }}" + echo "comment created at: ${{ steps.fc.outputs.comment-created-at }}" + + # Create or update the comment just found. + # See: https://github.com/peter-evans/create-or-update-comment + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + token: ${{ secrets.pr-token }} + repository: ${{ inputs.repository }} + issue-number: ${{ inputs.pr-number }} + comment-id: ${{ steps.fc.outputs.comment-id }} + edit-mode: replace + # TODO: use body-path instead + body: | + + + ## Report + + - something + - something else + + diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index da6c8d4..02b8487 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -105,6 +105,7 @@ jobs: needs: - prepare - build + if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_ut.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -118,6 +119,7 @@ jobs: - prepare - build - test + if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_benchmark.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -126,3 +128,16 @@ jobs: artifact_name: ${{ needs.build.outputs.artifact_name }} secrets: pr-token: ${{ secrets.COSDT_BOT_TOKEN }} + + pr-comment: + name: Run benchmarks + needs: + - prepare + # if: ${{ github.event_name == 'repository_dispatch' }} + uses: ./.github/workflows/_pytorch_pr_comment.yml + with: + repository: cosdt/pytorch-upstream + pr-number: 1 + pr-comment-author: cosdt + secrets: + pr-token: ${{ secrets.COSDT_BOT_TOKEN }} From 583df520672d394847b4db4d108fce6f307f94fd Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 15:03:15 +0800 Subject: [PATCH 2/9] update --- .github/workflows/_pytorch_pr_comment.yml | 7 ++++--- .github/workflows/ascend_npu_test.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_pytorch_pr_comment.yml b/.github/workflows/_pytorch_pr_comment.yml index 5c25562..597e008 100644 --- a/.github/workflows/_pytorch_pr_comment.yml +++ b/.github/workflows/_pytorch_pr_comment.yml @@ -61,10 +61,11 @@ jobs: # TODO: use body-path instead body: | - + ## Report - + - something - something else + - other things - + diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index 02b8487..e9d9bde 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -95,6 +95,7 @@ jobs: name: Build torch_npu needs: - prepare + if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_build.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -130,7 +131,7 @@ jobs: pr-token: ${{ secrets.COSDT_BOT_TOKEN }} pr-comment: - name: Run benchmarks + name: Comment report on upstream PRs needs: - prepare # if: ${{ github.event_name == 'repository_dispatch' }} From ba4e13bec4fe9f9ebda6b530965158e9af4cc03d Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 15:06:03 +0800 Subject: [PATCH 3/9] update --- .github/workflows/ascend_npu_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index e9d9bde..344813b 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -139,6 +139,6 @@ jobs: with: repository: cosdt/pytorch-upstream pr-number: 1 - pr-comment-author: cosdt + pr-comment-author: cosdt-bot secrets: pr-token: ${{ secrets.COSDT_BOT_TOKEN }} From a31bb57ec44883b647de19a2d3bd4ce8017f8769 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 15:55:04 +0800 Subject: [PATCH 4/9] update --- .github/workflows/ascend_npu_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index 344813b..efa5a45 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -72,7 +72,7 @@ on: # Only cancel the previous runs when triggered by a pull request concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'repository_dispatch' }} jobs: prepare: From 2dd6e7256387bf314f07855951f3e2c9cf756cae Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 17:08:08 +0800 Subject: [PATCH 5/9] update --- .ci/benchmark.py | 28 ++++++++++++++++++--- .github/workflows/_ascend_npu_benchmark.yml | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.ci/benchmark.py b/.ci/benchmark.py index f6c9400..7694348 100644 --- a/.ci/benchmark.py +++ b/.ci/benchmark.py @@ -1,14 +1,23 @@ import argparse import os -import sys + from src.benchmark.utils import read_metrics, to_markdown_table def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--path", type=str, required=True, help="Report path.") - parser.add_argument("--write-gh-job-summary", action="store_true", help="Write to GitHub job summary.") - parser.add_argument("--update-readme", action="store_true", help="Update statistics report in README.md.") + parser.add_argument( + "--path", type=str, required=True, help="The path of benchmark report." + ) + parser.add_argument( + "--output", type=str, required=False, help="The output path of the Markdown report." + ) + parser.add_argument( + "--write-gh-job-summary", action="store_true", help="Write to GitHub job summary." + ) + parser.add_argument( + "--update-readme", action="store_true", help="Update statistics report in README.md." + ) return parser.parse_args() @@ -18,6 +27,11 @@ def generate_report(path: str): return html_table +def save_output_report(path: str, report): + with open(path, "w") as f: + f.write(report) + + def write_job_summary(report): summary_path = os.environ["GITHUB_STEP_SUMMARY"] with open(summary_path, "a") as f: @@ -55,6 +69,12 @@ def update_readme(report): # Generate statistics report report = generate_report(args.path) + # Output to markdown report + if args.output: + save_output_report(args.output, report) + else: + print(report) + # Write to workflow job summary if args.write_gh_job_summary: write_job_summary(report) diff --git a/.github/workflows/_ascend_npu_benchmark.yml b/.github/workflows/_ascend_npu_benchmark.yml index 67c46b0..ac97458 100644 --- a/.github/workflows/_ascend_npu_benchmark.yml +++ b/.github/workflows/_ascend_npu_benchmark.yml @@ -32,7 +32,7 @@ defaults: shell: bash -el {0} jobs: - test: + benchmark: name: run benchmarks for torch_npu runs-on: ${{ inputs.runner }} env: From 796112fe12d4e8f0375dd32f53eb8845f73835e8 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 19:39:15 +0800 Subject: [PATCH 6/9] update --- .github/workflows/ascend_npu_test.yml | 4 +--- requirements.txt | 1 + src/benchmark/utils.py | 16 ++-------------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index efa5a45..2b0df4d 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -95,7 +95,6 @@ jobs: name: Build torch_npu needs: - prepare - if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_build.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -106,7 +105,6 @@ jobs: needs: - prepare - build - if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_ut.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -120,7 +118,6 @@ jobs: - prepare - build - test - if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_benchmark.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -130,6 +127,7 @@ jobs: secrets: pr-token: ${{ secrets.COSDT_BOT_TOKEN }} + # TODO pr-comment: name: Comment report on upstream PRs needs: diff --git a/requirements.txt b/requirements.txt index a5d5159..60a4d67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +git+https://github.com/pytorch/benchmark.git tabulate diff --git a/src/benchmark/utils.py b/src/benchmark/utils.py index 983823f..cff89ff 100644 --- a/src/benchmark/utils.py +++ b/src/benchmark/utils.py @@ -1,21 +1,9 @@ import ast import dataclasses import json -import pathlib -from typing import Dict, List, Optional - -# Same as torchbenchmark.util.experiment.instantiator.TorchBenchModelConfig -# https://github.com/pytorch/benchmark/blob/main/torchbenchmark/util/experiment/instantiator.py#L26 -@dataclasses.dataclass -class TorchBenchModelConfig: - name: str - test: str - device: str - batch_size: Optional[int] - extra_args: List[str] - extra_env: Optional[Dict[str, str]] = None - output_dir: Optional[pathlib.Path] = None +from typing import List +from torchbenchmark.util.experiment.instantiator import TorchBenchModelConfig @dataclasses.dataclass From d44a0921283e10c77c7bd8f7f1ea89ebf2de0b8e Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 9 Dec 2024 20:28:01 +0800 Subject: [PATCH 7/9] update --- .github/workflows/ascend_npu_test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index 2b0df4d..e20b676 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -48,9 +48,10 @@ on: required: true type: choice options: - - ascendai/cann:7.1-openeuler2203sp2 - - ascendai/cann:8.0.rc2.alpha003-910b-ubuntu22.04-py3.9 - - ascendai/cann:8.0.rc3.alpha002-910b-ubuntu22.04-py3.9 + - 7.0.1.beta1-910b-ubuntu22.04-py3.8 + - 8.0.rc1.beta1-910b-ubuntu22.04-py3.8 + - 8.0.rc2.beta1-910b-ubuntu22.04-py3.9 + - 8.0.rc3.beta1-910b-ubuntu22.04-py3.10 - ascendai/cann:latest default: 'ascendai/cann:latest' description: 'The docker image which will be loaded' From 337deac9b9223cb11baaeceb1aa406197d4ebe77 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Tue, 10 Dec 2024 09:18:11 +0800 Subject: [PATCH 8/9] update --- .github/workflows/_pytorch_pr_comment.yml | 1 - .github/workflows/ascend_npu_test.yml | 1 + requirements.txt | 1 - src/benchmark/utils.py | 16 ++++++++++++++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_pytorch_pr_comment.yml b/.github/workflows/_pytorch_pr_comment.yml index 597e008..0a53d46 100644 --- a/.github/workflows/_pytorch_pr_comment.yml +++ b/.github/workflows/_pytorch_pr_comment.yml @@ -38,7 +38,6 @@ jobs: issue-number: ${{ inputs.pr-number }} comment-author: ${{ inputs.pr-comment-author }} body-includes: '' - direction: first - name: Show comment info run: | diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index e20b676..5dd897b 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -119,6 +119,7 @@ jobs: - prepare - build - test + if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_benchmark.yml with: runner: ${{ needs.prepare.outputs.runner }} diff --git a/requirements.txt b/requirements.txt index 60a4d67..a5d5159 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -git+https://github.com/pytorch/benchmark.git tabulate diff --git a/src/benchmark/utils.py b/src/benchmark/utils.py index cff89ff..983823f 100644 --- a/src/benchmark/utils.py +++ b/src/benchmark/utils.py @@ -1,9 +1,21 @@ import ast import dataclasses import json +import pathlib +from typing import Dict, List, Optional -from typing import List -from torchbenchmark.util.experiment.instantiator import TorchBenchModelConfig + +# Same as torchbenchmark.util.experiment.instantiator.TorchBenchModelConfig +# https://github.com/pytorch/benchmark/blob/main/torchbenchmark/util/experiment/instantiator.py#L26 +@dataclasses.dataclass +class TorchBenchModelConfig: + name: str + test: str + device: str + batch_size: Optional[int] + extra_args: List[str] + extra_env: Optional[Dict[str, str]] = None + output_dir: Optional[pathlib.Path] = None @dataclasses.dataclass From 7c47198bc2ab44e32e309aa0ff87aa62bff67e28 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Tue, 10 Dec 2024 09:19:37 +0800 Subject: [PATCH 9/9] update --- .github/workflows/ascend_npu_test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ascend_npu_test.yml b/.github/workflows/ascend_npu_test.yml index 5dd897b..151c268 100644 --- a/.github/workflows/ascend_npu_test.yml +++ b/.github/workflows/ascend_npu_test.yml @@ -119,7 +119,6 @@ jobs: - prepare - build - test - if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_ascend_npu_benchmark.yml with: runner: ${{ needs.prepare.outputs.runner }} @@ -133,8 +132,8 @@ jobs: pr-comment: name: Comment report on upstream PRs needs: - - prepare - # if: ${{ github.event_name == 'repository_dispatch' }} + - benchmark + if: ${{ github.event_name == 'repository_dispatch' }} uses: ./.github/workflows/_pytorch_pr_comment.yml with: repository: cosdt/pytorch-upstream