From 7e619b95880cdec72d1353c9c99dedbc2575ad36 Mon Sep 17 00:00:00 2001 From: "Rong Rong (AI Infra)" Date: Thu, 24 Jun 2021 10:12:37 -0700 Subject: [PATCH] First step to rearrange files in tools folder (#60473) Summary: Changes including: - introduced `linter/`, `testing/`, `stats/` folders in `tools/` - move appropriate scripts into these folders - change grepped references in the pytorch/pytorch repo Next step - introduce `build/` folder for build scripts Pull Request resolved: https://github.com/pytorch/pytorch/pull/60473 Test Plan: - CI (this is important b/c pytorch/test-infra also rely on some script reference. - tools/tests/ Reviewed By: albanD Differential Revision: D29352716 Pulled By: walterddr fbshipit-source-id: bad40b5ce130b35dfd9e59b8af34f9025f3285fd --- .circleci/config.yml | 6 +- .../job-specs/job-specs-custom.yml | 2 +- .../job-specs/pytorch-job-specs.yml | 4 +- .github/templates/linux_ci_workflow.yml.j2 | 8 +- .github/templates/windows_ci_workflow.yml.j2 | 6 +- .github/workflows/build_linux_conda.yml | 2 +- .github/workflows/build_linux_libtorch.yml | 2 +- .github/workflows/build_linux_wheels.yml | 2 +- .github/workflows/clang_format.yml | 4 +- .github/workflows/lint.yml | 14 +-- ...inux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml | 8 +- ...inux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml | 8 +- .../pytorch-linux-xenial-py3.6-gcc5.4.yml | 8 +- .../workflows/pytorch-win-vs2019-cpu-py3.yml | 6 +- .../pytorch-win-vs2019-cuda10-cudnn7-py3.yml | 6 +- .../pytorch-win-vs2019-cuda11-cudnn8-py3.yml | 6 +- CONTRIBUTING.md | 6 +- Makefile | 2 +- mypy-strict.ini | 1 - mypy.ini | 8 +- test/run_test.py | 4 +- test/test_import_time.py | 2 +- test/test_testing.py | 93 --------------- tools/README.md | 14 +-- tools/actions_local_runner.py | 4 +- tools/git-pre-commit | 6 +- tools/{stats_utils => linter}/__init__.py | 0 tools/{ => linter}/clang_format_all.py | 0 tools/{ => linter}/clang_format_ci.sh | 2 +- tools/{ => linter}/clang_format_utils.py | 0 tools/{ => linter}/clang_tidy.py | 0 tools/{ => linter}/flake8_hook.py | 0 tools/{ => linter}/git-clang-format | 0 tools/{ => linter}/mypy_wrapper.py | 2 +- tools/{ => linter}/run_shellcheck.sh | 0 tools/{ => linter}/trailing_newlines.py | 0 tools/{ => linter}/translate_annotations.py | 2 +- tools/stats/__init__.py | 0 tools/{ => stats}/export_slow_tests.py | 2 +- tools/{ => stats}/print_test_stats.py | 8 +- .../{stats_utils => stats}/s3_stat_parser.py | 0 tools/{ => stats}/test_history.py | 10 +- tools/test/test_mypy_wrapper.py | 2 +- tools/test/test_stats.py | 10 +- tools/test/test_test_history.py | 2 +- tools/test/test_test_selection.py | 112 ++++++++++++++++++ tools/test/test_trailing_newlines.py | 2 +- tools/test/test_translate_annotations.py | 2 +- tools/testing/__init__.py | 0 tools/{ => testing}/explicit_ci_jobs.py | 0 .../testing/test_selections.py | 0 51 files changed, 203 insertions(+), 185 deletions(-) rename tools/{stats_utils => linter}/__init__.py (100%) rename tools/{ => linter}/clang_format_all.py (100%) rename tools/{ => linter}/clang_format_ci.sh (86%) rename tools/{ => linter}/clang_format_utils.py (100%) rename tools/{ => linter}/clang_tidy.py (100%) rename tools/{ => linter}/flake8_hook.py (100%) rename tools/{ => linter}/git-clang-format (100%) rename tools/{ => linter}/mypy_wrapper.py (99%) rename tools/{ => linter}/run_shellcheck.sh (100%) rename tools/{ => linter}/trailing_newlines.py (100%) rename tools/{ => linter}/translate_annotations.py (98%) create mode 100644 tools/stats/__init__.py rename tools/{ => stats}/export_slow_tests.py (97%) rename tools/{ => stats}/print_test_stats.py (98%) rename tools/{stats_utils => stats}/s3_stat_parser.py (100%) rename tools/{ => stats}/test_history.py (96%) create mode 100644 tools/test/test_test_selection.py create mode 100644 tools/testing/__init__.py rename tools/{ => testing}/explicit_ci_jobs.py (100%) rename torch/testing/_internal/framework_utils.py => tools/testing/test_selections.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 069b3847e583b..985713c35a210 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -686,7 +686,7 @@ jobs: export CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" cd workspace export PYTHONPATH="\${PWD}" - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test EOL echo "(cat docker_commands.sh | docker exec -u jenkins -e LANG=C.UTF-8 -i "$id" bash) 2>&1" > command.sh unbuffer bash command.sh | ts @@ -840,7 +840,7 @@ jobs: export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1} export PYTHONPATH="$PWD" pip install typing_extensions boto3 - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test when: always - store_test_results: path: test/test-reports @@ -1455,7 +1455,7 @@ jobs: # Using the same IAM user to write stats to our OSS bucket export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4} export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4} - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test when: always - store_test_results: path: test/test-reports diff --git a/.circleci/verbatim-sources/job-specs/job-specs-custom.yml b/.circleci/verbatim-sources/job-specs/job-specs-custom.yml index 4c7f991bb74a3..aa7b6e508f3a2 100644 --- a/.circleci/verbatim-sources/job-specs/job-specs-custom.yml +++ b/.circleci/verbatim-sources/job-specs/job-specs-custom.yml @@ -213,7 +213,7 @@ # Using the same IAM user to write stats to our OSS bucket export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4} export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4} - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test when: always - store_test_results: path: test/test-reports diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml index 4c2ee9b386316..940abb77bdfbf 100644 --- a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml +++ b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml @@ -224,7 +224,7 @@ jobs: export CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" cd workspace export PYTHONPATH="\${PWD}" - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test EOL echo "(cat docker_commands.sh | docker exec -u jenkins -e LANG=C.UTF-8 -i "$id" bash) 2>&1" > command.sh unbuffer bash command.sh | ts @@ -378,7 +378,7 @@ jobs: export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1} export PYTHONPATH="$PWD" pip install typing_extensions boto3 - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test when: always - store_test_results: path: test/test-reports diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index 4ca89cae2e0fb..a61a2795cabe4 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -145,7 +145,7 @@ jobs: sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} @@ -337,7 +337,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -360,7 +360,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -373,7 +373,7 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test {%- if enable_doc_jobs %} pytorch_python_doc_build: diff --git a/.github/templates/windows_ci_workflow.yml.j2 b/.github/templates/windows_ci_workflow.yml.j2 index 16230ac609674..735fcbd2b7fa1 100644 --- a/.github/templates/windows_ci_workflow.yml.j2 +++ b/.github/templates/windows_ci_workflow.yml.j2 @@ -203,7 +203,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -226,7 +226,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -239,4 +239,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/.github/workflows/build_linux_conda.yml b/.github/workflows/build_linux_conda.yml index b5bec78fb6d61..1ca2e82cb7b39 100644 --- a/.github/workflows/build_linux_conda.yml +++ b/.github/workflows/build_linux_conda.yml @@ -93,7 +93,7 @@ jobs: path: /remote/**/*.bz2 - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} diff --git a/.github/workflows/build_linux_libtorch.yml b/.github/workflows/build_linux_libtorch.yml index bf020667b8237..ddfca4be17605 100644 --- a/.github/workflows/build_linux_libtorch.yml +++ b/.github/workflows/build_linux_libtorch.yml @@ -92,7 +92,7 @@ jobs: path: /remote/**/*.zip - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} diff --git a/.github/workflows/build_linux_wheels.yml b/.github/workflows/build_linux_wheels.yml index a3aa14c43cc8f..7a26c3e6e1f40 100644 --- a/.github/workflows/build_linux_wheels.yml +++ b/.github/workflows/build_linux_wheels.yml @@ -91,7 +91,7 @@ jobs: path: /remote/**/*.whl - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index de5f1211121ba..a12197b84abb5 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -29,12 +29,12 @@ jobs: # only run clang-format on allowlisted files echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "| clang-format failures found! Run: " - echo "| tools/clang_format_ci.sh ${BASE_SHA} " + echo "| tools/linter/clang_format_ci.sh ${BASE_SHA} " echo "| to fix this error. " echo "| For more info, see: https://github.com/pytorch/pytorch/wiki/clang-format " echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - tools/clang_format_ci.sh "${BASE_SHA}" + tools/linter/clang_format_ci.sh "${BASE_SHA}" GIT_DIFF=$(git diff) if [[ -z $GIT_DIFF ]]; then diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 74afc3aa708ad..ec988819ef923 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,7 +31,7 @@ jobs: - name: Ensure correct trailing newlines if: always() && steps.requirements.outcome == 'success' run: | - (! git --no-pager grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)tools/clang_format_hash' | tools/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false)) + (! git --no-pager grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)tools/clang_format_hash' | tools/linter/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false)) - name: Ensure no trailing spaces if: always() run: | @@ -160,7 +160,7 @@ jobs: - name: Run ShellCheck if: always() && steps.install_shellcheck.outcome == 'success' run: | - if ! tools/run_shellcheck.sh .extracted_scripts .jenkins/pytorch; then + if ! tools/linter/run_shellcheck.sh .extracted_scripts .jenkins/pytorch; then echo echo 'ShellCheck gave a nonzero exit code. Please fix the warnings' echo 'listed above. Note that if a path in one of the above warning' @@ -233,7 +233,7 @@ jobs: - name: Install dependencies run: | set -eux - pip install typing-extensions # for tools/translate_annotations.py + pip install typing-extensions # for tools/linter/translate_annotations.py pip install -r requirements-flake8.txt flake8 --version - name: Run flake8 @@ -245,7 +245,7 @@ jobs: env: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - tools/translate_annotations.py \ + tools/linter/translate_annotations.py \ --file="${GITHUB_WORKSPACE}"/flake8-output.txt \ --regex='^(?P.*?):(?P\d+):(?P\d+): (?P\w+\d+) (?P.*)' \ --commit="$HEAD_SHA" \ @@ -275,7 +275,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - fetch-depth: 0 # to allow tools/clang_tidy.py to do its thing + fetch-depth: 0 # to allow tools/linter/clang_tidy.py to do its thing - name: Prepare output dir with HEAD commit SHA env: HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -328,7 +328,7 @@ jobs: # /torch/csrc/generic/*.cpp is excluded because those files aren't actually built. # deploy/interpreter files are excluded due to using macros and other techniquies # that are not easily converted to accepted c++ - python3 tools/clang_tidy.py \ + python3 tools/linter/clang_tidy.py \ --verbose \ --paths torch/csrc/ \ --diff-file pr.diff \ @@ -353,7 +353,7 @@ jobs: cat "${GITHUB_WORKSPACE}"/clang-tidy-output.txt - tools/translate_annotations.py \ + tools/linter/translate_annotations.py \ --file=clang-tidy-output.txt \ --regex='^(?P.*?):(?P\d+):(?P\d+): (?P.*?) \[(?P.*)\]' \ --commit="$HEAD_SHA" \ diff --git a/.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml b/.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml index 9b6a290c16080..236bb47b59b5d 100644 --- a/.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml +++ b/.github/workflows/pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7.yml @@ -143,7 +143,7 @@ jobs: sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} @@ -335,7 +335,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -358,7 +358,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -371,4 +371,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/.github/workflows/pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml b/.github/workflows/pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml index 03195f5c7f3ee..f449f445edf85 100644 --- a/.github/workflows/pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml +++ b/.github/workflows/pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7.yml @@ -143,7 +143,7 @@ jobs: sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} @@ -335,7 +335,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -358,7 +358,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -371,4 +371,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml index a161da4760630..c100c0fbdf8f6 100644 --- a/.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/pytorch-linux-xenial-py3.6-gcc5.4.yml @@ -144,7 +144,7 @@ jobs: sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} @@ -336,7 +336,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -359,7 +359,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -372,7 +372,7 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test pytorch_python_doc_build: runs-on: linux.2xlarge diff --git a/.github/workflows/pytorch-win-vs2019-cpu-py3.yml b/.github/workflows/pytorch-win-vs2019-cpu-py3.yml index 55496a0551bcb..3f9bcea8e74d2 100644 --- a/.github/workflows/pytorch-win-vs2019-cpu-py3.yml +++ b/.github/workflows/pytorch-win-vs2019-cpu-py3.yml @@ -167,7 +167,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -190,7 +190,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -203,4 +203,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/.github/workflows/pytorch-win-vs2019-cuda10-cudnn7-py3.yml b/.github/workflows/pytorch-win-vs2019-cuda10-cudnn7-py3.yml index 918469cf30fc8..160a2dec1eff3 100644 --- a/.github/workflows/pytorch-win-vs2019-cuda10-cudnn7-py3.yml +++ b/.github/workflows/pytorch-win-vs2019-cuda10-cudnn7-py3.yml @@ -185,7 +185,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -208,7 +208,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -221,4 +221,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/.github/workflows/pytorch-win-vs2019-cuda11-cudnn8-py3.yml b/.github/workflows/pytorch-win-vs2019-cuda11-cudnn8-py3.yml index 38ea8bd16d929..f8837fcc3c826 100644 --- a/.github/workflows/pytorch-win-vs2019-cuda11-cudnn8-py3.yml +++ b/.github/workflows/pytorch-win-vs2019-cuda11-cudnn8-py3.yml @@ -184,7 +184,7 @@ jobs: - name: Checkout PyTorch uses: actions/checkout@v2 with: - # deep clone, to allow tools/print_test_stats.py to use Git commands + # deep clone, to allow tools/stats/print_test_stats.py to use Git commands fetch-depth: 0 - uses: actions/download-artifact@v2 name: Download PyTorch Test Reports @@ -207,7 +207,7 @@ jobs: run: .github/scripts/parse_ref.py - name: Display and upload test statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update - # tools/print_test_stats.py to natively support GitHub Actions + # tools/stats/print_test_stats.py to natively support GitHub Actions env: SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }} @@ -220,4 +220,4 @@ jobs: CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' run: | export PYTHONPATH=$PWD - python tools/print_test_stats.py --upload-to-s3 --compare-with-s3 test + python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02d4f63a35471..1b22c755455d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -439,12 +439,12 @@ is part of the test suite `ContainerAliasingTest` in the file ### Run Specific CI Jobs You can generate a commit that limits the CI to only run a specific job by using -`tools/explicit_ci_jobs.py` like so: +`tools/testing/explicit_ci_jobs.py` like so: ```bash # --job: specify one or more times to filter to a specific job + its dependencies # --make-commit: commit CI changes to git with a message explaining the change -python tools/explicit_ci_jobs.py --job binary_linux_manywheel_3_6m_cpu_devtoolset7_nightly_test --make-commit +python tools/testing/explicit_ci_jobs.py --job binary_linux_manywheel_3_6m_cpu_devtoolset7_nightly_test --make-commit # Make your changes @@ -1128,7 +1128,7 @@ have more checks than older versions. In our CI, we run clang-tidy-6.0. uncommitted changes). Changes are picked up based on a `git diff` with the given revision: ```bash - python tools/clang_tidy.py -d build -p torch/csrc --diff 'HEAD~1' + python tools/linter/clang_tidy.py -d build -p torch/csrc --diff 'HEAD~1' ``` Above, it is assumed you are in the PyTorch root folder. `path/to/build` should diff --git a/Makefile b/Makefile index 8d61fd0c543ec..4c59f8e66ac90 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ SHELLCHECK_GHA_GENERATED_FOLDER=.shellcheck_generated_gha shellcheck-gha: @$(RM) -r $(SHELLCHECK_GHA_GENERATED_FOLDER) tools/extract_scripts.py --out=$(SHELLCHECK_GHA_GENERATED_FOLDER) - tools/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER) + tools/linter/run_shellcheck.sh $(SHELLCHECK_GHA_GENERATED_FOLDER) generate-gha-workflows: .github/scripts/generate_ci_workflows.py diff --git a/mypy-strict.ini b/mypy-strict.ini index 4c988883fdc26..857ef2b0486d5 100644 --- a/mypy-strict.ini +++ b/mypy-strict.ini @@ -40,7 +40,6 @@ files = .github, benchmarks/instruction_counts, tools, - torch/testing/_internal/framework_utils.py, torch/utils/_pytree.py, torch/utils/benchmark/utils/common.py, torch/utils/benchmark/utils/timer.py, diff --git a/mypy.ini b/mypy.ini index 9ad9600d81404..fe11fe88bc38d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -38,11 +38,11 @@ files = test/test_type_hints.py, test/test_type_info.py, test/test_utils.py, - tools/clang_format_utils.py, - tools/clang_tidy.py, + tools/linter/clang_format_utils.py, + tools/linter/clang_tidy.py, tools/generate_torch_version.py, tools/render_junit.py, - tools/stats_utils + tools/stats # # `exclude` is a regex, not a list of paths like `files` (sigh) @@ -109,7 +109,7 @@ warn_unused_ignores = False [mypy-tools.generate_torch_version] warn_unused_ignores = False -[mypy-tools.stats_utils.s3_stat_parser] +[mypy-tools.stats.s3_stat_parser] warn_unused_ignores = False # diff --git a/test/run_test.py b/test/run_test.py index 7f9f23aef0bd6..4a16f4340c0a4 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -16,18 +16,18 @@ import torch from torch.utils import cpp_extension from torch.testing._internal.common_utils import FILE_SCHEMA, IS_IN_CI, TEST_WITH_ROCM, shell, set_cwd -from torch.testing._internal.framework_utils import calculate_shards import torch.distributed as dist from typing import Dict, Optional, Tuple, List, Any from typing_extensions import TypedDict try: sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) - from tools.stats_utils.s3_stat_parser import ( + from tools.stats.s3_stat_parser import ( get_previous_reports_for_branch, get_previous_reports_for_pr, Report, HAVE_BOTO3) + from tools.testing.test_selections import calculate_shards except ImportError: print("Unable to import s3_stat_parser from tools. Running without S3 stats...") HAVE_BOTO3 = False diff --git a/test/test_import_time.py b/test/test_import_time.py index 88c01ffef933c..38ce685aa4294 100644 --- a/test/test_import_time.py +++ b/test/test_import_time.py @@ -4,7 +4,7 @@ # these tests could eventually be changed to fail if the import/init # time is greater than a certain threshold, but for now we just use them # as a way to track the duration of `import torch` in our ossci-metrics -# S3 bucket (see tools/print_test_stats.py) +# S3 bucket (see tools/stats/print_test_stats.py) class TestImportTime(TestCase): def test_time_import_torch(self): TestCase.runWithPytorchAPIUsageStderr('import torch') diff --git a/test/test_testing.py b/test/test_testing.py index a96c7fdb603df..8e47eafaa4772 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -3,7 +3,6 @@ import itertools import math import os -import random import re import unittest from typing import Any, Callable, Iterator, List, Tuple @@ -12,7 +11,6 @@ from torch.testing._internal.common_utils import \ (IS_FBCODE, IS_SANDCASTLE, IS_WINDOWS, TestCase, make_tensor, run_tests, skipIfRocm, slowTest) -from torch.testing._internal.framework_utils import calculate_shards from torch.testing._internal.common_device_type import \ (PYTORCH_TESTING_DEVICE_EXCEPT_FOR_KEY, PYTORCH_TESTING_DEVICE_ONLY_FOR_KEY, dtypes, get_device_type_test_bases, instantiate_device_type_tests, onlyCUDA, onlyOnCPUAndCUDA, @@ -619,97 +617,6 @@ def test_get_supported_dtypes(self, device): class TestFrameworkUtils(TestCase): - tests = [ - 'super_long_test', - 'long_test1', - 'long_test2', - 'normal_test1', - 'normal_test2', - 'normal_test3', - 'short_test1', - 'short_test2', - 'short_test3', - 'short_test4', - 'short_test5', - ] - - test_times = { - 'super_long_test': 55, - 'long_test1': 22, - 'long_test2': 18, - 'normal_test1': 9, - 'normal_test2': 7, - 'normal_test3': 5, - 'short_test1': 1, - 'short_test2': 0.6, - 'short_test3': 0.4, - 'short_test4': 0.3, - 'short_test5': 0.01, - } - - def test_calculate_2_shards_with_complete_test_times(self): - expected_shards = [ - (60, ['super_long_test', 'normal_test3']), - (58.31, ['long_test1', 'long_test2', 'normal_test1', 'normal_test2', 'short_test1', 'short_test2', - 'short_test3', 'short_test4', 'short_test5']) - ] - self.assertEqual(expected_shards, calculate_shards(2, self.tests, self.test_times)) - - - def test_calculate_5_shards_with_complete_test_times(self): - expected_shards = [ - (55, ['super_long_test']), - (22, ['long_test1', ]), - (18, ['long_test2', ]), - (11.31, ['normal_test1', 'short_test1', 'short_test2', 'short_test3', 'short_test4', 'short_test5']), - (12, ['normal_test2', 'normal_test3']), - ] - self.assertEqual(expected_shards, calculate_shards(5, self.tests, self.test_times)) - - - def test_calculate_2_shards_with_incomplete_test_times(self): - incomplete_test_times = {k: v for k, v in self.test_times.items() if 'test1' in k} - expected_shards = [ - (22, ['long_test1', 'long_test2', 'normal_test3', 'short_test3', 'short_test5']), - (10, ['normal_test1', 'short_test1', 'super_long_test', 'normal_test2', 'short_test2', 'short_test4']), - ] - self.assertEqual(expected_shards, calculate_shards(2, self.tests, incomplete_test_times)) - - - def test_calculate_5_shards_with_incomplete_test_times(self): - incomplete_test_times = {k: v for k, v in self.test_times.items() if 'test1' in k} - expected_shards = [ - (22, ['long_test1', 'normal_test2', 'short_test5']), - (9, ['normal_test1', 'normal_test3']), - (1, ['short_test1', 'short_test2']), - (0, ['super_long_test', 'short_test3']), - (0, ['long_test2', 'short_test4']), - ] - self.assertEqual(expected_shards, calculate_shards(5, self.tests, incomplete_test_times)) - - def test_calculate_2_shards_against_optimal_shards(self): - for _ in range(100): - random.seed(120) - random_times = {k: random.random() * 10 for k in self.tests} - # all test times except first two - rest_of_tests = [i for k, i in random_times.items() if k != 'super_long_test' and k != 'long_test1'] - sum_of_rest = sum(rest_of_tests) - random_times['super_long_test'] = max(sum_of_rest / 2, max(rest_of_tests)) - random_times['long_test1'] = sum_of_rest - random_times['super_long_test'] - # An optimal sharding would look like the below, but we don't need to compute this for the test: - # optimal_shards = [ - # (sum_of_rest, ['super_long_test', 'long_test1']), - # (sum_of_rest, [i for i in self.tests if i != 'super_long_test' and i != 'long_test1']), - # ] - calculated_shards = calculate_shards(2, self.tests, random_times) - max_shard_time = max(calculated_shards[0][0], calculated_shards[1][0]) - if sum_of_rest != 0: - # The calculated shard should not have a ratio worse than 7/6 for num_shards = 2 - self.assertGreaterEqual(7.0 / 6.0, max_shard_time / sum_of_rest) - sorted_tests = sorted(self.tests) - sorted_shard_tests = sorted(calculated_shards[0][1] + calculated_shards[1][1]) - # All the tests should be represented by some shard - self.assertEqual(sorted_tests, sorted_shard_tests) @skipIfRocm @unittest.skipIf(IS_WINDOWS, "Skipping because doesn't work for windows") diff --git a/tools/README.md b/tools/README.md index 880532ab4f7b5..795f37ef82bb9 100644 --- a/tools/README.md +++ b/tools/README.md @@ -37,11 +37,11 @@ Build system pieces: Developer tools which you might find useful: -* [clang_tidy.py](clang_tidy.py) - Script for running clang-tidy +* [linter/clang_tidy.py](linter/clang_tidy.py) - Script for running clang-tidy on lines of your script which you changed. * [extract_scripts.py](extract_scripts.py) - Extract scripts from `.github/workflows/*.yml` into a specified dir, on which linters such as - [run_shellcheck.sh](run_shellcheck.sh) can be run. Assumes that every `run` + [linter/run_shellcheck.sh](linter/run_shellcheck.sh) can be run. Assumes that every `run` script has `shell: bash` unless a different shell is explicitly listed on that specific step (so `defaults` doesn't currently work), but also has some rules for other situations such as [actions/github-script][]. Exits with nonzero @@ -53,17 +53,17 @@ Developer tools which you might find useful: can conveniently run diffs on them when working on code-generation. (See also [generated_dirs.txt](generated_dirs.txt) which specifies the list of directories with generated files.) -* [mypy_wrapper.py](mypy_wrapper.py) - Run `mypy` on a single file using the +* [linter/mypy_wrapper.py](linter/mypy_wrapper.py) - Run `mypy` on a single file using the appropriate subset of our `mypy*.ini` configs. -* [run_shellcheck.sh](run_shellcheck.sh) - Find `*.sh` files (recursively) in +* [linter/run_shellcheck.sh](linter/run_shellcheck.sh) - Find `*.sh` files (recursively) in the directories specified as arguments, and run [ShellCheck][] on all of them. -* [test_history.py](test_history.py) - Query S3 to display history of a single +* [stats/test_history.py](stats/test_history.py) - Query S3 to display history of a single test across multiple jobs over time. -* [trailing_newlines.py](trailing_newlines.py) - Take names of UTF-8 files from +* [linter/trailing_newlines.py](linter/trailing_newlines.py) - Take names of UTF-8 files from stdin, print names of nonempty files whose contents don't end in exactly one trailing newline, exit with status 1 if no output printed or 0 if some filenames were printed. -* [translate_annotations.py](translate_annotations.py) - Read [Flake8][] or +* [linter/translate_annotations.py](linter/translate_annotations.py) - Read [Flake8][] or [clang-tidy][] warnings (according to a `--regex`) from a `--file`, convert to the JSON format accepted by [pytorch/add-annotations-github-action], and translate line numbers from `HEAD` back in time to the given `--commit` by diff --git a/tools/actions_local_runner.py b/tools/actions_local_runner.py index 369f7138d59a6..3968057107b30 100755 --- a/tools/actions_local_runner.py +++ b/tools/actions_local_runner.py @@ -221,7 +221,7 @@ def env(self) -> Dict[str, Any]: async def quick(self, files: List[str]) -> CommandResult: return await shell_cmd( - [sys.executable, "tools/mypy_wrapper.py"] + [sys.executable, "tools/linter/mypy_wrapper.py"] + [os.path.join(REPO_ROOT, f) for f in files], env=self.env(), ) @@ -270,7 +270,7 @@ def filter_files(self, files: List[str]) -> List[str]: async def quick(self, files: List[str]) -> CommandResult: return await shell_cmd( - ["tools/run_shellcheck.sh"] + [os.path.join(REPO_ROOT, f) for f in files], + ["tools/linter/run_shellcheck.sh"] + [os.path.join(REPO_ROOT, f) for f in files], ) async def full(self) -> None: diff --git a/tools/git-pre-commit b/tools/git-pre-commit index 7e7c718f86074..a829fe9444b33 100755 --- a/tools/git-pre-commit +++ b/tools/git-pre-commit @@ -2,12 +2,12 @@ set -e echo "Running pre-commit flake8" -python tools/flake8_hook.py +python tools/linter/flake8_hook.py if [ $(which clang-tidy) ] then echo "Running pre-commit clang-tidy" - python tools/clang_tidy.py \ + python tools/linter/clang_tidy.py \ --paths torch/csrc \ --diff HEAD \ -g"-torch/csrc/jit/passes/onnx/helper.cpp" \ @@ -22,4 +22,4 @@ else fi echo "Running pre-commit clang-format" -tools/git-clang-format HEAD~ --force +tools/linter/git-clang-format HEAD~ --force diff --git a/tools/stats_utils/__init__.py b/tools/linter/__init__.py similarity index 100% rename from tools/stats_utils/__init__.py rename to tools/linter/__init__.py diff --git a/tools/clang_format_all.py b/tools/linter/clang_format_all.py similarity index 100% rename from tools/clang_format_all.py rename to tools/linter/clang_format_all.py diff --git a/tools/clang_format_ci.sh b/tools/linter/clang_format_ci.sh similarity index 86% rename from tools/clang_format_ci.sh rename to tools/linter/clang_format_ci.sh index 05400dad265b6..6f5220e516d19 100755 --- a/tools/clang_format_ci.sh +++ b/tools/linter/clang_format_ci.sh @@ -10,4 +10,4 @@ find . -type f \ -path './torch/csrc/jit/*' -or \ -path './test/cpp/jit/*' -or \ -path './test/cpp/tensorexpr/*' \ - | xargs tools/git-clang-format --verbose "$1" -- + | xargs tools/linter/git-clang-format --verbose "$1" -- diff --git a/tools/clang_format_utils.py b/tools/linter/clang_format_utils.py similarity index 100% rename from tools/clang_format_utils.py rename to tools/linter/clang_format_utils.py diff --git a/tools/clang_tidy.py b/tools/linter/clang_tidy.py similarity index 100% rename from tools/clang_tidy.py rename to tools/linter/clang_tidy.py diff --git a/tools/flake8_hook.py b/tools/linter/flake8_hook.py similarity index 100% rename from tools/flake8_hook.py rename to tools/linter/flake8_hook.py diff --git a/tools/git-clang-format b/tools/linter/git-clang-format similarity index 100% rename from tools/git-clang-format rename to tools/linter/git-clang-format diff --git a/tools/mypy_wrapper.py b/tools/linter/mypy_wrapper.py similarity index 99% rename from tools/mypy_wrapper.py rename to tools/linter/mypy_wrapper.py index 24b5098be61ef..fb1dbcbc65dd4 100755 --- a/tools/mypy_wrapper.py +++ b/tools/linter/mypy_wrapper.py @@ -196,7 +196,7 @@ def main(args: List[str]) -> None: "python.linting.mypyPath": "${env:HOME}/miniconda3/envs/pytorch/bin/python", "python.linting.mypyArgs": [ - "${workspaceFolder}/tools/mypy_wrapper.py" + "${workspaceFolder}/tools/linter/mypy_wrapper.py" ] } diff --git a/tools/run_shellcheck.sh b/tools/linter/run_shellcheck.sh similarity index 100% rename from tools/run_shellcheck.sh rename to tools/linter/run_shellcheck.sh diff --git a/tools/trailing_newlines.py b/tools/linter/trailing_newlines.py similarity index 100% rename from tools/trailing_newlines.py rename to tools/linter/trailing_newlines.py diff --git a/tools/translate_annotations.py b/tools/linter/translate_annotations.py similarity index 98% rename from tools/translate_annotations.py rename to tools/linter/translate_annotations.py index 33fcdc3a36050..4bcfc78486d49 100755 --- a/tools/translate_annotations.py +++ b/tools/linter/translate_annotations.py @@ -24,7 +24,7 @@ class Diff(TypedDict): hunks: List[Hunk] -# adapted from the similar regex in tools/clang_tidy.py +# adapted from the similar regex in tools/linter/clang_tidy.py # @@ -start,count +start,count @@ hunk_pattern = r'^@@\s+-(\d+)(?:,(\d+))?\s+\+(\d+)(?:,(\d+))?\s+@@' diff --git a/tools/stats/__init__.py b/tools/stats/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/export_slow_tests.py b/tools/stats/export_slow_tests.py similarity index 97% rename from tools/export_slow_tests.py rename to tools/stats/export_slow_tests.py index c0cb43a4a6144..4d880fd778a88 100644 --- a/tools/export_slow_tests.py +++ b/tools/stats/export_slow_tests.py @@ -5,7 +5,7 @@ import os import statistics from collections import defaultdict -from tools.stats_utils.s3_stat_parser import get_previous_reports_for_branch, Report, Version2Report +from tools.stats.s3_stat_parser import get_previous_reports_for_branch, Report, Version2Report from typing import cast, DefaultDict, Dict, List, Any from urllib.request import urlopen diff --git a/tools/print_test_stats.py b/tools/stats/print_test_stats.py similarity index 98% rename from tools/print_test_stats.py rename to tools/stats/print_test_stats.py index f92ecdc4d86be..caf2f08da3e0a 100755 --- a/tools/print_test_stats.py +++ b/tools/stats/print_test_stats.py @@ -18,9 +18,9 @@ import requests from typing_extensions import TypedDict -from tools.stats_utils.s3_stat_parser import (newify_case, get_S3_object_from_bucket, get_test_stats_summaries_for_job, - Report, Status, Commit, HAVE_BOTO3, Version2Case, VersionedReport, - Version1Report, Version2Report, ReportMetaMeta) +from tools.stats.s3_stat_parser import (newify_case, get_S3_object_from_bucket, get_test_stats_summaries_for_job, + Report, Status, Commit, HAVE_BOTO3, Version2Case, VersionedReport, + Version1Report, Version2Report, ReportMetaMeta) @@ -50,7 +50,7 @@ class SuiteDiff(TypedDict): # TODO: consolidate this with the get_cases function from -# tools/test_history.py +# tools/stats/test_history.py # Here we translate to a three-layer format (file -> suite -> case) # rather than a two-layer format (suite -> case) because as mentioned in diff --git a/tools/stats_utils/s3_stat_parser.py b/tools/stats/s3_stat_parser.py similarity index 100% rename from tools/stats_utils/s3_stat_parser.py rename to tools/stats/s3_stat_parser.py diff --git a/tools/test_history.py b/tools/stats/test_history.py similarity index 96% rename from tools/test_history.py rename to tools/stats/test_history.py index 44f3f4162c57c..24678aabba938 100755 --- a/tools/test_history.py +++ b/tools/stats/test_history.py @@ -7,8 +7,8 @@ from signal import SIG_DFL, SIGPIPE, signal from typing import Dict, Iterator, List, Optional, Set, Tuple -from tools.stats_utils.s3_stat_parser import (Report, get_cases, - get_test_stats_summaries) +from tools.stats.s3_stat_parser import (Report, get_cases, + get_test_stats_summaries) def get_git_commit_history( @@ -193,7 +193,7 @@ def description() -> str: followed by the time of the specified test in that job at that commit. Example: - $ tools/test_history.py --mode=multiline --ref=594a66 --sha-length=8 --test=test_set_dir \ + $ tools/stats/test_history.py --mode=multiline --ref=594a66 --sha-length=8 --test=test_set_dir \ --job pytorch_linux_xenial_py3_6_gcc5_4_test --job pytorch_linux_xenial_py3_6_gcc7_test 2021-02-10 11:13:34Z 594a66d7 pytorch_linux_xenial_py3_6_gcc5_4_test 0.36s 2021-02-10 11:13:34Z 594a66d7 pytorch_linux_xenial_py3_6_gcc7_test 0.573s errored @@ -211,7 +211,7 @@ def description() -> str: Another multiline example, this time with the --all flag: - $ tools/test_history.py --mode=multiline --all --ref=321b9 --delta=12 --sha-length=8 \ + $ tools/stats/test_history.py --mode=multiline --all --ref=321b9 --delta=12 --sha-length=8 \ --test=test_qr_square_many_batched_complex_cuda 2021-01-07 10:04:56Z 321b9883 pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test2 424.284s 2021-01-07 10:04:56Z 321b9883 pytorch_linux_xenial_cuda10_2_cudnn7_py3_slow_test 0.006s skipped @@ -226,7 +226,7 @@ def description() -> str: columns is guaranteed to match the order of the jobs passed on the command line. Example: - $ tools/test_history.py --mode=columns --ref=3cf783 --sha-length=8 --test=test_set_dir \ + $ tools/stats/test_history.py --mode=columns --ref=3cf783 --sha-length=8 --test=test_set_dir \ --job pytorch_linux_xenial_py3_6_gcc5_4_test --job pytorch_linux_xenial_py3_6_gcc7_test 2021-02-10 12:18:50Z 3cf78395 0.644s 0.312s 2021-02-10 11:13:34Z 594a66d7 0.360s errored diff --git a/tools/test/test_mypy_wrapper.py b/tools/test/test_mypy_wrapper.py index 3d8fab42a2972..df7b0ab9e27d9 100644 --- a/tools/test/test_mypy_wrapper.py +++ b/tools/test/test_mypy_wrapper.py @@ -1,6 +1,6 @@ import unittest -from tools import mypy_wrapper +from tools.linter import mypy_wrapper class TestMypyWrapper(unittest.TestCase): diff --git a/tools/test/test_stats.py b/tools/test/test_stats.py index af916ae4fd980..f2fa1b0a99a54 100644 --- a/tools/test/test_stats.py +++ b/tools/test/test_stats.py @@ -2,11 +2,11 @@ import unittest from typing import Dict, List -from tools import print_test_stats -from tools.stats_utils.s3_stat_parser import (Commit, Report, ReportMetaMeta, - Status, Version1Case, - Version1Report, Version2Case, - Version2Report) +from tools.stats import print_test_stats +from tools.stats.s3_stat_parser import (Commit, Report, ReportMetaMeta, + Status, Version1Case, + Version1Report, Version2Case, + Version2Report) def fakehash(char: str) -> str: diff --git a/tools/test/test_test_history.py b/tools/test/test_test_history.py index 2526855ec6533..8863c24a5d55c 100644 --- a/tools/test/test_test_history.py +++ b/tools/test/test_test_history.py @@ -4,7 +4,7 @@ import unittest from typing import List, Optional -from tools import test_history +from tools.stats import test_history from typing_extensions import TypedDict diff --git a/tools/test/test_test_selection.py b/tools/test/test_test_selection.py new file mode 100644 index 0000000000000..5ea6fa8b3c622 --- /dev/null +++ b/tools/test/test_test_selection.py @@ -0,0 +1,112 @@ +import random +import unittest + +from tools.testing.test_selections import calculate_shards +from typing import Dict, List, Tuple + + +class TestCalculateShards(unittest.TestCase): + tests: List[str] = [ + 'super_long_test', + 'long_test1', + 'long_test2', + 'normal_test1', + 'normal_test2', + 'normal_test3', + 'short_test1', + 'short_test2', + 'short_test3', + 'short_test4', + 'short_test5', + ] + + test_times: Dict[str, float] = { + 'super_long_test': 55, + 'long_test1': 22, + 'long_test2': 18, + 'normal_test1': 9, + 'normal_test2': 7, + 'normal_test3': 5, + 'short_test1': 1, + 'short_test2': 0.6, + 'short_test3': 0.4, + 'short_test4': 0.3, + 'short_test5': 0.01, + } + + def assert_shards_equal( + self, + expected_shards: List[Tuple[float, List[str]]], + actual_shards: List[Tuple[float, List[str]]] + ) -> None: + for expected, actual in zip(expected_shards, actual_shards): + self.assertAlmostEqual(expected[0], actual[0]) + self.assertListEqual(expected[1], actual[1]) + + def test_calculate_2_shards_with_complete_test_times(self) -> None: + expected_shards = [ + (60, ['super_long_test', 'normal_test3']), + (58.31, ['long_test1', 'long_test2', 'normal_test1', 'normal_test2', 'short_test1', 'short_test2', + 'short_test3', 'short_test4', 'short_test5']) + ] + self.assert_shards_equal(expected_shards, calculate_shards(2, self.tests, self.test_times)) + + + def test_calculate_5_shards_with_complete_test_times(self) -> None: + expected_shards = [ + (55.0, ['super_long_test']), + (22.0, ['long_test1', ]), + (18.0, ['long_test2', ]), + (11.31, ['normal_test1', 'short_test1', 'short_test2', 'short_test3', 'short_test4', 'short_test5']), + (12.0, ['normal_test2', 'normal_test3']), + ] + self.assert_shards_equal(expected_shards, calculate_shards(5, self.tests, self.test_times)) + + + def test_calculate_2_shards_with_incomplete_test_times(self) -> None: + incomplete_test_times = {k: v for k, v in self.test_times.items() if 'test1' in k} + expected_shards = [ + (22.0, ['long_test1', 'long_test2', 'normal_test3', 'short_test3', 'short_test5']), + (10.0, ['normal_test1', 'short_test1', 'super_long_test', 'normal_test2', 'short_test2', 'short_test4']), + ] + self.assert_shards_equal(expected_shards, calculate_shards(2, self.tests, incomplete_test_times)) + + + def test_calculate_5_shards_with_incomplete_test_times(self) -> None: + incomplete_test_times = {k: v for k, v in self.test_times.items() if 'test1' in k} + expected_shards = [ + (22.0, ['long_test1', 'normal_test2', 'short_test5']), + (9.0, ['normal_test1', 'normal_test3']), + (1.0, ['short_test1', 'short_test2']), + (0.0, ['super_long_test', 'short_test3']), + (0.0, ['long_test2', 'short_test4']), + ] + self.assert_shards_equal(expected_shards, calculate_shards(5, self.tests, incomplete_test_times)) + + def test_calculate_2_shards_against_optimal_shards(self) -> None: + for _ in range(100): + random.seed(120) + random_times = {k: random.random() * 10 for k in self.tests} + # all test times except first two + rest_of_tests = [i for k, i in random_times.items() if k != 'super_long_test' and k != 'long_test1'] + sum_of_rest = sum(rest_of_tests) + random_times['super_long_test'] = max(sum_of_rest / 2, max(rest_of_tests)) + random_times['long_test1'] = sum_of_rest - random_times['super_long_test'] + # An optimal sharding would look like the below, but we don't need to compute this for the test: + # optimal_shards = [ + # (sum_of_rest, ['super_long_test', 'long_test1']), + # (sum_of_rest, [i for i in self.tests if i != 'super_long_test' and i != 'long_test1']), + # ] + calculated_shards = calculate_shards(2, self.tests, random_times) + max_shard_time = max(calculated_shards[0][0], calculated_shards[1][0]) + if sum_of_rest != 0: + # The calculated shard should not have a ratio worse than 7/6 for num_shards = 2 + self.assertGreaterEqual(7.0 / 6.0, max_shard_time / sum_of_rest) + sorted_tests = sorted(self.tests) + sorted_shard_tests = sorted(calculated_shards[0][1] + calculated_shards[1][1]) + # All the tests should be represented by some shard + self.assertEqual(sorted_tests, sorted_shard_tests) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test/test_trailing_newlines.py b/tools/test/test_trailing_newlines.py index ff6c9e6780c75..4f4b662b1036b 100644 --- a/tools/test/test_trailing_newlines.py +++ b/tools/test/test_trailing_newlines.py @@ -1,4 +1,4 @@ -from tools import trailing_newlines +from tools.linter import trailing_newlines import unittest import tempfile diff --git a/tools/test/test_translate_annotations.py b/tools/test/test_translate_annotations.py index a0fed6cac4149..867decc4af1aa 100644 --- a/tools/test/test_translate_annotations.py +++ b/tools/test/test_translate_annotations.py @@ -1,7 +1,7 @@ import re import unittest -from tools.translate_annotations import parse_annotation, parse_diff, translate +from tools.linter.translate_annotations import parse_annotation, parse_diff, translate flake8_regex \ = r'^(?P.*?):(?P\d+):(?P\d+): (?P\w+\d+) (?P.*)' diff --git a/tools/testing/__init__.py b/tools/testing/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/explicit_ci_jobs.py b/tools/testing/explicit_ci_jobs.py similarity index 100% rename from tools/explicit_ci_jobs.py rename to tools/testing/explicit_ci_jobs.py diff --git a/torch/testing/_internal/framework_utils.py b/tools/testing/test_selections.py similarity index 100% rename from torch/testing/_internal/framework_utils.py rename to tools/testing/test_selections.py