From 879257ca62c1c81c14cd97d4adca521ef0a46405 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 14:52:01 -0700 Subject: [PATCH 01/10] Update lint.yml --- .github/workflows/lint.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bd981991edf7..38c24f1b9def 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,3 +46,43 @@ jobs: regex: '^(?.*?):(?\d+):(?\d+): (?\w\d+) (?.*)' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + clang-tidy: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.7.4 + architecture: x64 + - name: Checkout PyTorch + uses: actions/checkout@master + - name: Checkout PR tip + run: | + set -eux + git checkout ${GITHUB_HEAD_REF} + echo ::set-output name=commit_sha::$(git rev-parse ${GITHUB_HEAD_REF}) + id: get_pr_tip + - name: Install dependencies + run: | + echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} + pip install pyyaml + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" + sudo apt-get update + sudo apt-get install -y clang-tidy-8 + sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 + ./tools/run-clang-tidy-in-ci.sh > ${GITHUB_WORKSPACE}/clang-tidy-output.txt + echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} + cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt + env: + SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} + - name: Add annotations + uses: suo/add-annotations-github-action@releases/v0.0.1 + with: + check_name: 'clang-tidy' + linter_output_path: 'clang-tidy-output.txt' + commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }} + regex: '^(?.*?):(?\d+):(?\d+): (?\w\d+) (?[\s|\w]*)' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d47b2d0537f64389bbb3256a3227f88517cfbb32 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 14:57:45 -0700 Subject: [PATCH 02/10] introduce error --- .clang-tidy | 1 - .github/workflows/lint.yml | 1 + torch/csrc/jit/script/init.cpp | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index fae404cddb40..5a289a1e1444 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,7 +28,6 @@ Checks: ' ,performance-* ,-performance-noexcept-move-constructor ' -WarningsAsErrors: '*' HeaderFilterRegex: 'torch/csrc/.*' AnalyzeTemporaryDtors: false CheckOptions: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 38c24f1b9def..228311bc22ed 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -65,6 +65,7 @@ jobs: id: get_pr_tip - name: Install dependencies run: | + set -eux echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} pip install pyyaml wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - diff --git a/torch/csrc/jit/script/init.cpp b/torch/csrc/jit/script/init.cpp index dac7007178ab..9da37ba5de15 100644 --- a/torch/csrc/jit/script/init.cpp +++ b/torch/csrc/jit/script/init.cpp @@ -1017,8 +1017,10 @@ void initJitScriptBindings(PyObject* module) { return pythonResolver(rcb)->resolveType(name, range); }); - py::class_>( + + py::class_>( m, "LoggerBase"); + py::enum_(m, "AggregationType") .value("SUM", logging::LockingLogger::AggregationType::SUM) .value("AVG", logging::LockingLogger::AggregationType::AVG) From a14fbe488555c69a762916aee1a44991fe90fc44 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 15:22:09 -0700 Subject: [PATCH 03/10] set +e --- .clang-tidy | 1 + tools/run-clang-tidy-in-ci.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 5a289a1e1444..fae404cddb40 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,6 +28,7 @@ Checks: ' ,performance-* ,-performance-noexcept-move-constructor ' +WarningsAsErrors: '*' HeaderFilterRegex: 'torch/csrc/.*' AnalyzeTemporaryDtors: false CheckOptions: diff --git a/tools/run-clang-tidy-in-ci.sh b/tools/run-clang-tidy-in-ci.sh index e8f090acf79c..48335f56756f 100755 --- a/tools/run-clang-tidy-in-ci.sh +++ b/tools/run-clang-tidy-in-ci.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -x BASE_BRANCH=master if [[ $SYSTEM_PULLREQUEST_TARGETBRANCH ]]; then From d02a8135ffa155296e22e1a9d31dd0624312accd Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 15:51:54 -0700 Subject: [PATCH 04/10] set +e --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 228311bc22ed..d12c3b765ab8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -65,7 +65,7 @@ jobs: id: get_pr_tip - name: Install dependencies run: | - set -eux + set -ux echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} pip install pyyaml wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - From f51d45b39995b6c34a2cd709481842774f72ed7e Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 16:25:43 -0700 Subject: [PATCH 05/10] set +e --- .clang-tidy | 1 - .github/workflows/lint.yml | 2 +- tools/clang_tidy.py | 10 ++++++++-- tools/run-clang-tidy-in-ci.sh | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index fae404cddb40..5a289a1e1444 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,7 +28,6 @@ Checks: ' ,performance-* ,-performance-noexcept-move-constructor ' -WarningsAsErrors: '*' HeaderFilterRegex: 'torch/csrc/.*' AnalyzeTemporaryDtors: false CheckOptions: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d12c3b765ab8..228311bc22ed 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -65,7 +65,7 @@ jobs: id: get_pr_tip - name: Install dependencies run: | - set -ux + set -eux echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} pip install pyyaml wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - diff --git a/tools/clang_tidy.py b/tools/clang_tidy.py index 3bd9c095cd8f..0baac9e36f1b 100755 --- a/tools/clang_tidy.py +++ b/tools/clang_tidy.py @@ -18,6 +18,7 @@ import collections import fnmatch import json +import os import os.path import re import shlex @@ -172,7 +173,7 @@ def run_clang_tidy(options, line_filters, files): with open(options.config_file) as config: # Here we convert the YAML config file to a JSON blob. - command += ["-config", json.dumps(yaml.load(config))] + command += ["-config", json.dumps(yaml.load(config, Loader=yaml.FullLoader))] command += options.extra_args if line_filters: @@ -293,8 +294,13 @@ def main(): if options.diff: line_filters = [get_changed_lines(options.diff, f) for f in files] - print(run_clang_tidy(options, line_filters, files), file=sys.stderr) + pwd = os.getcwd() + "/" + clang_tidy_output = run_clang_tidy(options, line_filters, files) + formatted_output = [] + for line in clang_tidy_output.splitlines(): + if line.startswith(pwd): + print(line[len(pwd):]) if __name__ == "__main__": main() diff --git a/tools/run-clang-tidy-in-ci.sh b/tools/run-clang-tidy-in-ci.sh index 48335f56756f..e8f090acf79c 100755 --- a/tools/run-clang-tidy-in-ci.sh +++ b/tools/run-clang-tidy-in-ci.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -x +set -ex BASE_BRANCH=master if [[ $SYSTEM_PULLREQUEST_TARGETBRANCH ]]; then From 5ffea2ae6b8edc55a4fa94a2677c22abf4adf34c Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 16:56:29 -0700 Subject: [PATCH 06/10] set +e --- .github/workflows/lint.yml | 48 +++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 228311bc22ed..74a21928cdb1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -66,15 +66,57 @@ jobs: - name: Install dependencies run: | set -eux - echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} pip install pyyaml wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" sudo apt-get update sudo apt-get install -y clang-tidy-8 sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 - ./tools/run-clang-tidy-in-ci.sh > ${GITHUB_WORKSPACE}/clang-tidy-output.txt - echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} + + BASE_BRANCH=master + if [[ $SYSTEM_PULLREQUEST_TARGETBRANCH ]]; then + git remote add upstream https://github.com/pytorch/pytorch + git fetch upstream "$SYSTEM_PULLREQUEST_TARGETBRANCH" + BASE_BRANCH="upstream/$SYSTEM_PULLREQUEST_TARGETBRANCH" + fi + + if [[ ! -d build ]]; then + git submodule update --init --recursive + + mkdir build + pushd build + # We really only need compile_commands.json, so no need to build! + time cmake .. + popd + + # Generate ATen files. + time python aten/src/ATen/gen.py \ + -s aten/src/ATen \ + -d build/aten/src/ATen \ + aten/src/ATen/Declarations.cwrap \ + aten/src/THNN/generic/THNN.h \ + aten/src/THCUNN/generic/THCUNN.h \ + aten/src/ATen/nn.yaml \ + aten/src/ATen/native/native_functions.yaml + + # Generate PyTorch files. + time python tools/setup_helpers/generate_code.py \ + --declarations-path build/aten/src/ATen/Declarations.yaml \ + --nn-path aten/src + fi + + # Run Clang-Tidy + # The negative filters below are to exclude files that include onnx_pb.h or + # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. + python tools/clang_tidy.py \ + --verbose \ + --paths torch/csrc/ \ + --diff "$BASE_BRANCH" \ + -g"-torch/csrc/jit/export.cpp" \ + -g"-torch/csrc/jit/import.cpp" \ + -g"-torch/csrc/jit/netdef_converter.cpp" \ + "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt + cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt env: SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} From 6b3b00ee507ac361ab396e91aa0451ddcabd92e9 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 17:01:29 -0700 Subject: [PATCH 07/10] set +e --- .github/workflows/lint.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 74a21928cdb1..998ceca6a652 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -109,7 +109,6 @@ jobs: # The negative filters below are to exclude files that include onnx_pb.h or # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. python tools/clang_tidy.py \ - --verbose \ --paths torch/csrc/ \ --diff "$BASE_BRANCH" \ -g"-torch/csrc/jit/export.cpp" \ @@ -121,7 +120,7 @@ jobs: env: SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} - name: Add annotations - uses: suo/add-annotations-github-action@releases/v0.0.1 + uses: suo/add-annotations-github-action@master with: check_name: 'clang-tidy' linter_output_path: 'clang-tidy-output.txt' From f1901fac49bbf7c02ef4cb5cd4892da386e5f14f Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Thu, 10 Oct 2019 17:07:13 -0700 Subject: [PATCH 08/10] set +e --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 998ceca6a652..216e9f0145a5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -125,6 +125,6 @@ jobs: check_name: 'clang-tidy' linter_output_path: 'clang-tidy-output.txt' commit_sha: ${{ steps.get_pr_tip.outputs.commit_sha }} - regex: '^(?.*?):(?\d+):(?\d+): (?\w\d+) (?[\s|\w]*)' + regex: '^(?.*?):(?\d+):(?\d+): (?.*?) (?\[.*\])' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d6918929c524f7384e6ac5911fd598d3de9296c6 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Fri, 11 Oct 2019 09:40:42 -0700 Subject: [PATCH 09/10] set +e --- .github/workflows/lint.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 216e9f0145a5..29f66bf69b68 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -66,13 +66,16 @@ jobs: - name: Install dependencies run: | set -eux + # Install dependencies pip install pyyaml wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" sudo apt-get update sudo apt-get install -y clang-tidy-8 sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 - + - name: Generate PyTorch files + run: | + set -eux BASE_BRANCH=master if [[ $SYSTEM_PULLREQUEST_TARGETBRANCH ]]; then git remote add upstream https://github.com/pytorch/pytorch @@ -104,11 +107,15 @@ jobs: --declarations-path build/aten/src/ATen/Declarations.yaml \ --nn-path aten/src fi - + env: + SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} + - name: Run clang-tidy + run: | + set -eux # Run Clang-Tidy # The negative filters below are to exclude files that include onnx_pb.h or # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. - python tools/clang_tidy.py \ + python tools/clang_tidy.py \ --paths torch/csrc/ \ --diff "$BASE_BRANCH" \ -g"-torch/csrc/jit/export.cpp" \ @@ -117,8 +124,6 @@ jobs: "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt - env: - SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} - name: Add annotations uses: suo/add-annotations-github-action@master with: From abbaaa87d70bfdac7dd36b08627b6b041a3c4407 Mon Sep 17 00:00:00 2001 From: Michael Suo Date: Fri, 11 Oct 2019 09:47:14 -0700 Subject: [PATCH 10/10] set +e --- .github/workflows/lint.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 29f66bf69b68..54f16600f014 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -73,7 +73,7 @@ jobs: sudo apt-get update sudo apt-get install -y clang-tidy-8 sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-8 1000 - - name: Generate PyTorch files + - name: Run clang-tidy run: | set -eux BASE_BRANCH=master @@ -107,11 +107,7 @@ jobs: --declarations-path build/aten/src/ATen/Declarations.yaml \ --nn-path aten/src fi - env: - SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} - - name: Run clang-tidy - run: | - set -eux + # Run Clang-Tidy # The negative filters below are to exclude files that include onnx_pb.h or # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. @@ -124,6 +120,8 @@ jobs: "$@" > ${GITHUB_WORKSPACE}/clang-tidy-output.txt cat ${GITHUB_WORKSPACE}/clang-tidy-output.txt + env: + SYSTEM_PULLREQUEST_TARGETBRANCH: ${{ github.base_ref }} - name: Add annotations uses: suo/add-annotations-github-action@master with: